From 327b0390939f8d1a2bdfa83bb8c5b78a274a32d8 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Mon, 20 Nov 2017 21:16:24 +0530 Subject: [PATCH] YARN-7538 --- .../scheduler/AbstractResourceUsage.java | 2 +- .../scheduler/QueueResourceQuotas.java | 38 +++++ .../resourcemanager/scheduler/ResourceUsage.java | 162 ++++++++++++++++++++- .../scheduler/capacity/LeafQueue.java | 4 +- .../scheduler/capacity/ParentQueue.java | 9 ++ 5 files changed, 209 insertions(+), 6 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java index c2953236f47..fd736e3c673 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java @@ -58,7 +58,7 @@ public AbstractResourceUsage() { USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), CACHED_PENDING( 5), AMLIMIT(6), MIN_RESOURCE(7), MAX_RESOURCE(8), EFF_MIN_RESOURCE( 9), EFF_MAX_RESOURCE( - 10), EFF_MIN_RESOURCE_UP(11), EFF_MAX_RESOURCE_UP(12); + 10), EFF_MIN_RESOURCE_DOWN(11), EFF_MAX_RESOURCE_DOWN(12); private int idx; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java index 08b4d04d54d..e131ef29363 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java @@ -112,4 +112,42 @@ public void setEffectiveMaxResource(Resource res) { public void setEffectiveMaxResource(String label, Resource res) { _set(label, ResourceType.EFF_MAX_RESOURCE, res); } + + /* + * Effective Minimum Resource + */ + public Resource getEffectiveMinResourceDown() { + return _get(NL, ResourceType.EFF_MIN_RESOURCE_DOWN); + } + + public Resource getEffectiveMinResourceDown(String label) { + return _get(label, ResourceType.EFF_MIN_RESOURCE_DOWN); + } + + public void setEffectiveMinResourceDown(String label, Resource res) { + _set(label, ResourceType.EFF_MIN_RESOURCE_DOWN, res); + } + + public void setEffectiveMinResourceDown(Resource res) { + _set(NL, ResourceType.EFF_MIN_RESOURCE_DOWN, res); + } + + /* + * Effective Maximum Resource + */ + public Resource getEffectiveMaxResourceDown() { + return getEffectiveMaxResource(NL); + } + + public Resource getEffectiveMaxResourceDown(String label) { + return _get(label, ResourceType.EFF_MAX_RESOURCE); + } + + public void setEffectiveMaxResourceDown(Resource res) { + setEffectiveMaxResource(NL, res); + } + + public void setEffectiveMaxResourceDown(String label, Resource res) { + _set(label, ResourceType.EFF_MAX_RESOURCE, res); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java index ede4aec1dd2..eee91029d9d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java @@ -28,6 +28,9 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractResourceUsage.ResourceType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractResourceUsage.UsageByLabel; import org.apache.hadoop.yarn.util.resource.Resources; /** @@ -39,14 +42,167 @@ * * And it is thread-safe */ -public class ResourceUsage extends AbstractResourceUsage { +public class ResourceUsage { // short for no-label :) private static final String NL = CommonNodeLabelsManager.NO_LABEL; + protected ReadLock readLock; + protected WriteLock writeLock; + protected Map usages; + public ResourceUsage() { - super(); + ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + readLock = lock.readLock(); + writeLock = lock.writeLock(); + + usages = new HashMap(); + usages.put(NL, new UsageByLabel(NL)); + } + + // Usage enum here to make implement cleaner + public enum ResourceType { + // CACHED_USED and CACHED_PENDING may be read by anyone, but must only + // be written by ordering policies + USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), CACHED_PENDING( + 5), AMLIMIT(6), MIN_RESOURCE(7), MAX_RESOURCE(8), EFF_MIN_RESOURCE( + 9), EFF_MAX_RESOURCE( + 10), EFF_MIN_RESOURCE_UP(11), EFF_MAX_RESOURCE_UP(12); + + private int idx; + + private ResourceType(int value) { + this.idx = value; + } + } + + public static class UsageByLabel { + // usage by label, contains all UsageType + private Resource[] resArr; + + public UsageByLabel(String label) { + resArr = new Resource[ResourceType.values().length]; + for (int i = 0; i < resArr.length; i++) { + resArr[i] = Resource.newInstance(0, 0); + }; + } + + public Resource getUsed() { + return resArr[ResourceType.USED.idx]; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{used=" + resArr[0] + "%, "); + sb.append("pending=" + resArr[1] + "%, "); + sb.append("am_used=" + resArr[2] + "%, "); + sb.append("reserved=" + resArr[3] + "%}"); + sb.append("min_eff=" + resArr[9] + "%, "); + sb.append("max_eff=" + resArr[10] + "%}"); + sb.append("min_effup=" + resArr[11] + "%, "); + return sb.toString(); + } + } + + private static Resource normalize(Resource res) { + if (res == null) { + return Resources.none(); + } + return res; + } + + protected Resource _get(String label, ResourceType type) { + if (label == null) { + label = RMNodeLabelsManager.NO_LABEL; + } + + try { + readLock.lock(); + UsageByLabel usage = usages.get(label); + if (null == usage) { + return Resources.none(); + } + return normalize(usage.resArr[type.idx]); + } finally { + readLock.unlock(); + } } + protected Resource _getAll(ResourceType type) { + try { + readLock.lock(); + Resource allOfType = Resources.createResource(0); + for (Map.Entry usageEntry : usages.entrySet()) { + //all usages types are initialized + Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]); + } + return allOfType; + } finally { + readLock.unlock(); + } + } + + private UsageByLabel getAndAddIfMissing(String label) { + if (label == null) { + label = RMNodeLabelsManager.NO_LABEL; + } + if (!usages.containsKey(label)) { + UsageByLabel u = new UsageByLabel(label); + usages.put(label, u); + return u; + } + + return usages.get(label); + } + + protected void _set(String label, ResourceType type, Resource res) { + try { + writeLock.lock(); + UsageByLabel usage = getAndAddIfMissing(label); + usage.resArr[type.idx] = res; + } finally { + writeLock.unlock(); + } + } + + protected void _inc(String label, ResourceType type, Resource res) { + try { + writeLock.lock(); + UsageByLabel usage = getAndAddIfMissing(label); + Resources.addTo(usage.resArr[type.idx], res); + } finally { + writeLock.unlock(); + } + } + + protected void _dec(String label, ResourceType type, Resource res) { + try { + writeLock.lock(); + UsageByLabel usage = getAndAddIfMissing(label); + Resources.subtractFrom(usage.resArr[type.idx], res); + } finally { + writeLock.unlock(); + } + } + + @Override + public String toString() { + try { + readLock.lock(); + return usages.toString(); + } finally { + readLock.unlock(); + } + } + + public Set getNodePartitionsSet() { + try { + readLock.lock(); + return usages.keySet(); + } finally { + readLock.unlock(); + } + } /* * Used */ @@ -78,7 +234,7 @@ public void setUsed(Resource res) { setUsed(NL, res); } - public void copyAllUsed(AbstractResourceUsage other) { + public void copyAllUsed(ResourceUsage other) { try { writeLock.lock(); for (Entry entry : other.usages.entrySet()) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 41ec4ba762f..4d2d011b680 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -1343,7 +1343,7 @@ private Resource getHeadroom(User user, currentPartitionResourceLimit = partition.equals(RMNodeLabelsManager.NO_LABEL) ? currentPartitionResourceLimit - : getQueueMaxResource(partition); + : getQueueResourceQuotas().getEffectiveMaxResource(partition); Resource headroom = Resources.componentwiseMin( Resources.subtract(userLimitResource, user.getUsed(partition)), @@ -1363,7 +1363,7 @@ private Resource getHeadroom(User user, clusterFreePartitionResource, headroom); return headroom; } - + private void setQueueResourceLimitsInfo( Resource clusterResource) { synchronized (queueResourceLimitsInfo) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index a427fb135ab..15f8f4047c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -997,6 +997,15 @@ private void calculateEffectiveResourcesAndCapacity(String label, .getAbsoluteMaximumCapacity(label))); } + childQueue.getQueueResourceQuotas().setEffectiveMinResourceDown( + Resources.normalizeDown(resourceCalculator, + getQueueResourceQuotas().getEffectiveMinResource(label), + minimumAllocation)); + childQueue.getQueueResourceQuotas().setEffectiveMaxResourceDown( + Resources.normalizeDown(resourceCalculator, + getQueueResourceQuotas().getEffectiveMaxResource(label), + minimumAllocation)); + if (LOG.isDebugEnabled()) { LOG.debug("Updating effective min resource for queue:" + childQueue.getQueueName() + " as effMinResource=" -- 2.13.6 (Apple Git-96)