From 2ca86e67efbaa5f716b7e66d43786c70b96dfb12 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Fri, 23 Nov 2018 12:46:01 +0530 Subject: [PATCH] YARN-9037 --- .../yarn/util/resource/ResourceUtils.java | 26 +++++++++++++++ .../resource/DominantResourceCalculator.java | 32 +++++++++---------- .../hadoop/yarn/util/resource/Resources.java | 16 +++++----- ...AbstractPreemptableResourceCalculator.java | 2 +- .../capacity/TempQueuePerPartition.java | 2 +- .../scheduler/ClusterNodeTracker.java | 2 +- .../scheduler/SchedulerUtils.java | 6 ++-- .../scheduler/capacity/ParentQueue.java | 4 +-- .../scheduler/fair/ConfigurableResource.java | 2 +- .../fair/FairSchedulerConfiguration.java | 2 +- .../DominantResourceFairnessPolicy.java | 2 +- 11 files changed, 61 insertions(+), 35 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java index 20b64bd87ed..c6ceaaa48ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java @@ -60,6 +60,7 @@ public static final String TAGS = ".tags"; public static final String MINIMUM_ALLOCATION = ".minimum-allocation"; public static final String MAXIMUM_ALLOCATION = ".maximum-allocation"; + public static final String NON_COUNTABLE_RESOURCE = "system:csi-volume"; private static final String MEMORY = ResourceInformation.MEMORY_MB.getName(); private static final String VCORES = ResourceInformation.VCORES.getName(); @@ -74,10 +75,12 @@ private static final Map RESOURCE_NAME_TO_INDEX = new ConcurrentHashMap(); private static volatile Map resourceTypes; + private static volatile Map nonCountableResourceTypes; private static volatile ResourceInformation[] resourceTypesArray; private static volatile boolean initializedNodeResources = false; private static volatile Map readOnlyNodeResources; private static volatile int numKnownResourceTypes = -1; + private static volatile int numNonCountableResourceTypes = -1; static final Logger LOG = LoggerFactory.getLogger(ResourceUtils.class); @@ -290,15 +293,18 @@ static void initializeResourcesMap(Configuration conf) { public static void initializeResourcesFromResourceInformationMap( Map resourceInformationMap) { resourceTypes = Collections.unmodifiableMap(resourceInformationMap); + nonCountableResourceTypes = new HashMap<>(); updateKnownResources(); updateResourceTypeIndex(); initializedResources = true; numKnownResourceTypes = resourceTypes.size(); + numNonCountableResourceTypes = nonCountableResourceTypes.size(); } private static void updateKnownResources() { // Update resource names. resourceTypesArray = new ResourceInformation[resourceTypes.size()]; + List nonCountableResources = new ArrayList<>(); int index = 2; for (ResourceInformation resInfo : resourceTypes.values()) { @@ -309,10 +315,22 @@ private static void updateKnownResources() { resourceTypesArray[1] = ResourceInformation .newInstance(resourceTypes.get(VCORES)); } else { + if (resInfo.getTags() != null && resInfo.getTags() + .contains(NON_COUNTABLE_RESOURCE)) { + nonCountableResources.add(resInfo); + continue; + } resourceTypesArray[index] = ResourceInformation.newInstance(resInfo); index++; } } + + // Add all non-countable resource types to the end of the resource array. + for(ResourceInformation resInfo: nonCountableResources) { + resourceTypesArray[index] = ResourceInformation.newInstance(resInfo); + nonCountableResourceTypes.put(resInfo.getName(), resInfo); + index++; + } } private static void updateResourceTypeIndex() { @@ -355,6 +373,13 @@ public static int getNumberOfKnownResourceTypes() { return numKnownResourceTypes; } + public static int getNumberOfCountableResourceTypes() { + if (numKnownResourceTypes < 0) { + initializeResourceTypesIfNeeded(); + } + return numKnownResourceTypes - numNonCountableResourceTypes; + } + private static Map getResourceTypes( Configuration conf) { return getResourceTypes(conf, @@ -383,6 +408,7 @@ private static void initializeResourceTypesIfNeeded(Configuration conf, } } numKnownResourceTypes = resourceTypes.size(); + numNonCountableResourceTypes = nonCountableResourceTypes.size(); } private static Map getResourceTypes( diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 9aeb51cc2cc..d66e1a80e30 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -72,7 +72,7 @@ private int compare(Resource lhs, Resource rhs) { boolean rhsGreater = false; int ret = 0; - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation lhsResourceInformation = lhs .getResourceInformation(i); @@ -110,7 +110,7 @@ public int compare(Resource clusterResource, Resource lhs, Resource rhs, // resources and then look for which resource has the biggest // share overall. ResourceInformation[] clusterRes = clusterResource.getResources(); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); // If array creation shows up as a time sink, these arrays could be cached // because they're always the same length. @@ -183,7 +183,7 @@ private void calculateShares(ResourceInformation[] clusterRes, Resource first, ResourceInformation[] firstRes = first.getResources(); ResourceInformation[] secondRes = second.getResources(); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { firstShares[i] = calculateShare(clusterRes[i], firstRes[i]); secondShares[i] = calculateShare(clusterRes[i], secondRes[i]); @@ -274,7 +274,7 @@ private void calculateShares(ResourceInformation[] clusterRes, Resource first, max[0] = 0.0; max[1] = 0.0; - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { firstShares[i] = calculateShare(clusterRes[i], firstRes[i]); secondShares[i] = calculateShare(clusterRes[i], secondRes[i]); @@ -330,7 +330,7 @@ private double compareShares(double[] lhsShares, double[] rhsShares) { public long computeAvailableContainers(Resource available, Resource required) { long min = Long.MAX_VALUE; - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation availableResource = available .getResourceInformation(i); @@ -346,7 +346,7 @@ public long computeAvailableContainers(Resource available, @Override public float divide(Resource clusterResource, Resource numerator, Resource denominator) { - int nKnownResourceTypes = ResourceUtils.getNumberOfKnownResourceTypes(); + int nKnownResourceTypes = ResourceUtils.getNumberOfCountableResourceTypes(); ResourceInformation[] clusterRes = clusterResource.getResources(); // We have to provide the calculateShares() method with somewhere to store // the shares. We don't actually need these shares afterwards. @@ -375,7 +375,7 @@ public boolean isInvalidDivisor(Resource r) { @Override public float ratio(Resource a, Resource b) { float ratio = 0.0f; - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation aResourceInformation = a.getResourceInformation(i); ResourceInformation bResourceInformation = b.getResourceInformation(i); @@ -393,7 +393,7 @@ public Resource divideAndCeil(Resource numerator, int denominator) { public Resource divideAndCeil(Resource numerator, long denominator) { Resource ret = Resource.newInstance(numerator); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation resourceInformation = ret.getResourceInformation(i); resourceInformation @@ -414,7 +414,7 @@ public Resource divideAndCeil(Resource numerator, float denominator) { public Resource normalize(Resource r, Resource minimumResource, Resource maximumResource, Resource stepFactor) { Resource ret = Resource.newInstance(r); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation minimumResourceInformation = minimumResource @@ -448,7 +448,7 @@ public Resource roundDown(Resource r, Resource stepFactor) { private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) { Resource ret = Resource.newInstance(r); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation stepFactorResourceInformation = stepFactor @@ -473,7 +473,7 @@ private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) { public Resource multiplyAndNormalizeUp(Resource r, double[] by, Resource stepFactor) { Resource ret = Resource.newInstance(r); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation stepFactorResourceInformation = stepFactor @@ -502,7 +502,7 @@ public Resource multiplyAndNormalizeDown(Resource r, double by, private Resource multiplyAndNormalize(Resource r, double by, Resource stepFactor, boolean roundUp) { Resource ret = Resource.newInstance(r); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation stepFactorResourceInformation = stepFactor @@ -528,7 +528,7 @@ private Resource multiplyAndNormalize(Resource r, double by, @Override public boolean fitsIn(Resource smaller, Resource bigger) { - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation sResourceInformation = smaller .getResourceInformation(i); @@ -544,7 +544,7 @@ public boolean fitsIn(Resource smaller, Resource bigger) { @Override public Resource normalizeDown(Resource r, Resource stepFactor) { Resource ret = Resource.newInstance(r); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation rResourceInformation = r.getResourceInformation(i); ResourceInformation stepFactorResourceInformation = stepFactor @@ -564,7 +564,7 @@ public Resource normalizeDown(Resource r, Resource stepFactor) { @Override public boolean isAnyMajorResourceZeroOrNegative(Resource resource) { - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation resourceInformation = resource.getResourceInformation( i); @@ -577,7 +577,7 @@ public boolean isAnyMajorResourceZeroOrNegative(Resource resource) { @Override public boolean isAnyMajorResourceAboveZero(Resource resource) { - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation resourceInformation = resource.getResourceInformation( i); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 48c2c364ae9..bf1df8dbfa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -251,7 +251,7 @@ public static Resource clone(Resource res) { } public static Resource addTo(Resource lhs, Resource rhs) { - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -270,7 +270,7 @@ public static Resource add(Resource lhs, Resource rhs) { } public static Resource subtractFrom(Resource lhs, Resource rhs) { - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -325,7 +325,7 @@ public static Resource negate(Resource resource) { } public static Resource multiplyTo(Resource lhs, double by) { - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation lhsValue = lhs.getResourceInformation(i); @@ -348,7 +348,7 @@ public static Resource multiply(Resource lhs, double by) { */ public static Resource multiplyAndAddTo( Resource lhs, Resource rhs, double by) { - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -381,7 +381,7 @@ public static Resource multiplyAndNormalizeDown( public static Resource multiplyAndRoundDown(Resource lhs, double by) { Resource out = clone(lhs); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation lhsValue = lhs.getResourceInformation(i); @@ -490,7 +490,7 @@ public static Resource max( } public static boolean fitsIn(Resource smaller, Resource bigger) { - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = bigger.getResourceInformation(i); @@ -513,7 +513,7 @@ public static boolean fitsIn(ResourceCalculator rc, public static Resource componentwiseMin(Resource lhs, Resource rhs) { Resource ret = createResource(0); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); @@ -532,7 +532,7 @@ public static Resource componentwiseMin(Resource lhs, Resource rhs) { public static Resource componentwiseMax(Resource lhs, Resource rhs) { Resource ret = createResource(0); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { try { ResourceInformation rhsValue = rhs.getResourceInformation(i); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java index 5b8360a1c9e..c8f68a26a07 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java @@ -262,7 +262,7 @@ protected void initIdealAssignment(Resource totGuarant, private void resetCapacity(Resource clusterResource, Collection queues, boolean ignoreGuar) { Resource activeCap = Resource.newInstance(0, 0); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); if (ignoreGuar) { for (TempQueuePerPartition q : queues) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java index 4fb1862b887..57dc6395702 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java @@ -98,7 +98,7 @@ public TempQueuePerPartition(String queueName, Resource current, } this.normalizedGuarantee = new double[ResourceUtils - .getNumberOfKnownResourceTypes()]; + .getNumberOfCountableResourceTypes()]; this.children = new ArrayList<>(); this.apps = new ArrayList<>(); this.untouchableExtra = Resource.newInstance(0, 0); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java index 0f72c761e80..7e12aae5bed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ClusterNodeTracker.java @@ -73,7 +73,7 @@ private boolean reportedMaxAllocation = false; public ClusterNodeTracker() { - maxAllocation = new long[ResourceUtils.getNumberOfKnownResourceTypes()]; + maxAllocation = new long[ResourceUtils.getNumberOfCountableResourceTypes()]; Arrays.fill(maxAllocation, -1); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java index 9a02b6e75ca..d8415bfa779 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java @@ -355,7 +355,7 @@ private static void validateResourceRequest(ResourceRequest resReq, private static Map getZeroResources( Resource resource) { Map resourceInformations = Maps.newHashMap(); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation resourceInformation = @@ -372,7 +372,7 @@ private static void validateResourceRequest(ResourceRequest resReq, @VisibleForTesting static void checkResourceRequestAgainstAvailableResource(Resource reqResource, Resource availableResource) throws InvalidResourceRequestException { - for (int i = 0; i < ResourceUtils.getNumberOfKnownResourceTypes(); i++) { + for (int i = 0; i < ResourceUtils.getNumberOfCountableResourceTypes(); i++) { final ResourceInformation requestedRI = reqResource.getResourceInformation(i); final String reqResourceName = requestedRI.getName(); @@ -404,7 +404,7 @@ static void checkResourceRequestAgainstAvailableResource(Resource reqResource, } List invalidResources = Lists.newArrayList(); - for (int i = 0; i < ResourceUtils.getNumberOfKnownResourceTypes(); i++) { + for (int i = 0; i < ResourceUtils.getNumberOfCountableResourceTypes(); i++) { final ResourceInformation requestedRI = reqResource.getResourceInformation(i); final String reqResourceName = requestedRI.getName(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index e32130fc4ed..dc7a74f750f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -1027,7 +1027,7 @@ private void calculateEffectiveResourcesAndCapacity(String label, private Resource getMinResourceNormalized(String name, Map effectiveMinRatio, Resource minResource) { Resource ret = Resource.newInstance(minResource); - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation nResourceInformation = minResource .getResourceInformation(i); @@ -1051,7 +1051,7 @@ private Resource getMinResourceNormalized(String name, Map effect Resource configuredMinResources, Resource numeratorForMinRatio) { Map effectiveMinRatioPerResource = new HashMap<>(); if (numeratorForMinRatio != null) { - int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + int maxLength = ResourceUtils.getNumberOfCountableResourceTypes(); for (int i = 0; i < maxLength; i++) { ResourceInformation nResourceInformation = numeratorForMinRatio .getResourceInformation(i); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java index f772c4d1c0b..62bad4414b2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/ConfigurableResource.java @@ -63,7 +63,7 @@ public ConfigurableResource(Resource resource) { private static double[] getOneHundredPercentArray() { double[] resourcePercentages = - new double[ResourceUtils.getNumberOfKnownResourceTypes()]; + new double[ResourceUtils.getNumberOfCountableResourceTypes()]; Arrays.fill(resourcePercentages, 1.0); return resourcePercentages; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java index 8c4932bfe67..298498553ec 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java @@ -553,7 +553,7 @@ private static ConfigurableResource parseOldStyleResource(String value) private static double[] getResourcePercentage( String val) throws AllocationConfigurationException { int numberOfKnownResourceTypes = ResourceUtils - .getNumberOfKnownResourceTypes(); + .getNumberOfCountableResourceTypes(); double[] resourcePercentage = new double[numberOfKnownResourceTypes]; String[] strings = val.split(","); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java index 59635d987a0..e8203414081 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/DominantResourceFairnessPolicy.java @@ -48,7 +48,7 @@ public static final String NAME = "DRF"; private static final int NUM_RESOURCES = - ResourceUtils.getNumberOfKnownResourceTypes(); + ResourceUtils.getNumberOfCountableResourceTypes(); private static final DominantResourceFairnessComparator COMPARATORN = new DominantResourceFairnessComparatorN(); private static final DominantResourceFairnessComparator COMPARATOR2 = -- 2.17.2 (Apple Git-113)