diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java index cda5ec794e2..1be8a081bd6 100644 --- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java +++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java @@ -1152,4 +1152,35 @@ public static boolean equalsIgnoreCase(String s1, String s2) { return s1.equalsIgnoreCase(s2); } + /** + *

Checks if the String contains only unicode letters.

+ * + *

null will return false. + * An empty String (length()=0) will return true.

+ * + *
+   * StringUtils.isAlpha(null)   = false
+   * StringUtils.isAlpha("")     = true
+   * StringUtils.isAlpha("  ")   = false
+   * StringUtils.isAlpha("abc")  = true
+   * StringUtils.isAlpha("ab2c") = false
+   * StringUtils.isAlpha("ab-c") = false
+   * 
+ * + * @param str the String to check, may be null + * @return true if only contains letters, and is non-null + */ + public static boolean isAlpha(String str) { + if (str == null) { + return false; + } + int sz = str.length(); + for (int i = 0; i < sz; i++) { + if (Character.isLetter(str.charAt(i)) == false) { + return false; + } + } + return true; + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java index e25c8aa3f34..30096bf6a25 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java @@ -147,4 +147,80 @@ public static QueueConfigurations newInstance(float capacity, @Private @Unstable public abstract void setMaxAMPercentage(float maxAMPercentage); + + /** + * Get the effective minimum capacity of queue (from absolute resource). + * + * @return minimum resource capability + */ + @Public + @Unstable + public abstract Resource getEffectiveMinCapacity(); + + /** + * Set the effective minimum capacity of queue (from absolute resource). + * + * @param capacity + * minimum resource capability + */ + @Private + @Unstable + public abstract void setEffectiveMinCapacity(Resource capacity); + + /** + * Get the effective maximum capacity of queue (from absolute resource). + * + * @return maximum resource capability + */ + @Public + @Unstable + public abstract Resource getEffectiveMaxCapacity(); + + /** + * Set the effective maximum capacity of queue (from absolute resource). + * + * @param capacity + * maximum resource capability + */ + @Private + @Unstable + public abstract void setEffectiveMaxCapacity(Resource capacity); + + /** + * Get the configured minimum capacity of queue (from absolute resource). + * + * @return minimum resource capability + */ + @Public + @Unstable + public abstract Resource getConfiguredMinCapacity(); + + /** + * Set the configured minimum capacity of queue (from absolute resource). + * + * @param configuredMinResource + * minimum resource capability + */ + @Public + @Unstable + public abstract void setConfiguredMinCapacity(Resource configuredMinResource); + + /** + * Get the configured maximum capacity of queue (from absolute resource). + * + * @return maximum resource capability + */ + @Public + @Unstable + public abstract Resource getConfiguredMaxCapacity(); + + /** + * Set the configured maximum capacity of queue (from absolute resource). + * + * @param configuredMaxResource + * maximum resource capability + */ + @Public + @Unstable + public abstract void setConfiguredMaxCapacity(Resource configuredMaxResource); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java index 67592ccf067..a4c1f6c6de3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java @@ -225,6 +225,12 @@ public static ResourceInformation newInstance(String name, String units, Long.MAX_VALUE); } + public static ResourceInformation newInstance(String name, String units, + long minRes, long maxRes) { + return ResourceInformation.newInstance(name, units, 0L, + ResourceTypes.COUNTABLE, minRes, maxRes); + } + public static ResourceInformation newInstance(String name, long value) { return ResourceInformation .newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 7769c48fb7b..9f53704cfe4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -543,6 +543,10 @@ message QueueConfigurationsProto { optional float maxCapacity = 3; optional float absoluteMaxCapacity = 4; optional float maxAMPercentage = 5; + optional ResourceProto effectiveMinCapacity = 6; + optional ResourceProto effectiveMaxCapacity = 7; + optional ResourceProto configuredMinCapacity = 8; + optional ResourceProto configuredMaxCapacity = 9; } message QueueConfigurationsMapProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java index f308bcee490..80ef4b2aed2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java @@ -19,16 +19,22 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import org.apache.hadoop.yarn.api.records.QueueConfigurations; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import com.google.protobuf.TextFormat; public class QueueConfigurationsPBImpl extends QueueConfigurations { - QueueConfigurationsProto proto = - QueueConfigurationsProto.getDefaultInstance(); + QueueConfigurationsProto proto = QueueConfigurationsProto + .getDefaultInstance(); QueueConfigurationsProto.Builder builder = null; + Resource configuredMinResource = null; + Resource configuredMaxResource = null; + Resource effMinResource = null; + Resource effMaxResource = null; boolean viaProto = false; public QueueConfigurationsPBImpl() { @@ -41,11 +47,40 @@ public QueueConfigurationsPBImpl(QueueConfigurationsProto proto) { } public QueueConfigurationsProto getProto() { + mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; return proto; } + private void mergeLocalToProto() { + if (viaProto) { + maybeInitBuilder(); + } + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.effMinResource != null) { + builder + .setEffectiveMinCapacity(convertToProtoFormat(this.effMinResource)); + } + if (this.effMaxResource != null) { + builder + .setEffectiveMaxCapacity(convertToProtoFormat(this.effMaxResource)); + } + if (this.configuredMinResource != null) { + builder.setEffectiveMinCapacity( + convertToProtoFormat(this.configuredMinResource)); + } + if (this.configuredMaxResource != null) { + builder.setEffectiveMaxCapacity( + convertToProtoFormat(this.configuredMaxResource)); + } + } + @Override public float getCapacity() { QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; @@ -106,6 +141,58 @@ public void setMaxAMPercentage(float maxAMPercentage) { builder.setMaxAMPercentage(maxAMPercentage); } + @Override + public Resource getEffectiveMinCapacity() { + QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; + if (this.effMinResource != null) { + return this.effMinResource; + } + if (!p.hasEffectiveMinCapacity()) { + return null; + } + this.effMinResource = convertFromProtoFormat(p.getEffectiveMinCapacity()); + return this.effMinResource; + } + + @Override + public void setEffectiveMinCapacity(Resource capacity) { + maybeInitBuilder(); + if (capacity == null) { + builder.clearEffectiveMinCapacity(); + } + this.effMinResource = capacity; + } + + @Override + public Resource getEffectiveMaxCapacity() { + QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; + if (this.effMaxResource != null) { + return this.effMaxResource; + } + if (!p.hasEffectiveMaxCapacity()) { + return null; + } + this.effMaxResource = convertFromProtoFormat(p.getEffectiveMaxCapacity()); + return this.effMaxResource; + } + + @Override + public void setEffectiveMaxCapacity(Resource capacity) { + maybeInitBuilder(); + if (capacity == null) { + builder.clearEffectiveMaxCapacity(); + } + this.effMaxResource = capacity; + } + + private ResourcePBImpl convertFromProtoFormat(ResourceProto p) { + return new ResourcePBImpl(p); + } + + private ResourceProto convertToProtoFormat(Resource t) { + return ProtoUtils.convertToProtoFormat(t); + } + private void maybeInitBuilder() { if (viaProto || builder == null) { builder = QueueConfigurationsProto.newBuilder(proto); @@ -134,4 +221,49 @@ public boolean equals(Object other) { return false; } + @Override + public Resource getConfiguredMinCapacity() { + QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; + if (this.configuredMinResource != null) { + return this.configuredMinResource; + } + if (!p.hasConfiguredMinCapacity()) { + return null; + } + this.configuredMinResource = convertFromProtoFormat( + p.getConfiguredMinCapacity()); + return this.configuredMinResource; + } + + @Override + public void setConfiguredMinCapacity(Resource configuredMinResource) { + maybeInitBuilder(); + if (configuredMinResource == null) { + builder.clearConfiguredMinCapacity(); + } + this.configuredMinResource = configuredMinResource; + } + + @Override + public Resource getConfiguredMaxCapacity() { + QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; + if (this.configuredMaxResource != null) { + return this.configuredMaxResource; + } + if (!p.hasConfiguredMaxCapacity()) { + return null; + } + this.configuredMaxResource = convertFromProtoFormat( + p.getConfiguredMaxCapacity()); + return this.configuredMaxResource; + } + + @Override + public void setConfiguredMaxCapacity(Resource configuredMaxResource) { + maybeInitBuilder(); + if (configuredMaxResource == null) { + builder.clearConfiguredMaxCapacity(); + } + this.configuredMaxResource = configuredMaxResource; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java index 401e0c00d8c..6ebed6e0576 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java @@ -26,7 +26,6 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException; -import org.apache.hadoop.yarn.exceptions.YarnRuntimeException; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProtoOrBuilder; import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java index 7f155e7a40e..6375c4afd9c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java @@ -112,6 +112,14 @@ public Resource multiplyAndNormalizeUp(Resource r, double by, } @Override + public Resource multiplyAndNormalizeUp(Resource r, double[] by, + Resource stepFactor) { + return Resources.createResource( + roundUp((long) (r.getMemorySize() * by[0] + 0.5), + stepFactor.getMemorySize())); + } + + @Override public Resource multiplyAndNormalizeDown(Resource r, double by, Resource stepFactor) { return Resources.createResource( @@ -131,4 +139,10 @@ public boolean fitsIn(Resource smaller, Resource bigger) { public boolean isAnyMajorResourceZero(Resource resource) { return resource.getMemorySize() == 0f; } + + @Override + public Resource normalizeDown(Resource r, Resource stepFactor) { + return Resources.createResource( + roundDown((r.getMemorySize()), stepFactor.getMemorySize())); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java index 6b284e34f80..26ddc18cc51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java @@ -496,6 +496,27 @@ private Resource rounding(Resource r, Resource stepFactor, boolean roundUp) { } @Override + public Resource multiplyAndNormalizeUp(Resource r, double[] by, + Resource stepFactor) { + Resource ret = Resource.newInstance(r); + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { + ResourceInformation rResourceInformation = r.getResourceInformation(i); + ResourceInformation stepFactorResourceInformation = stepFactor + .getResourceInformation(i); + + long rValue = rResourceInformation.getValue(); + long stepFactorValue = UnitsConversionUtil.convert( + stepFactorResourceInformation.getUnits(), + rResourceInformation.getUnits(), + stepFactorResourceInformation.getValue()); + ret.setResourceValue(i, ResourceCalculator + .roundUp((long) Math.ceil(rValue * by[i]), stepFactorValue)); + } + return ret; + } + + @Override public Resource multiplyAndNormalizeUp(Resource r, double by, Resource stepFactor) { return this.multiplyAndNormalize(r, by, stepFactor, true); @@ -567,4 +588,29 @@ public boolean isAnyMajorResourceZero(Resource resource) { } return false; } + + @Override + public Resource normalizeDown(Resource r, Resource stepFactor) { + Resource ret = Resource.newInstance(r); + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { + ResourceInformation rResourceInformation = r.getResourceInformation(i); + ResourceInformation stepFactorResourceInformation = stepFactor + .getResourceInformation(i); + ResourceInformation tmp = ret.getResourceInformation(i); + + long rValue = rResourceInformation.getValue(); + long stepFactorValue = UnitsConversionUtil.convert( + stepFactorResourceInformation.getUnits(), + rResourceInformation.getUnits(), + stepFactorResourceInformation.getValue()); + + long value = rValue; + if (stepFactorValue != 0) { + value = roundDown(rValue, stepFactorValue); + } + tmp.setValue(value); + } + return ret; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java index d59560fa24d..dfd6684322d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java @@ -125,7 +125,19 @@ public abstract long computeAvailableContainers( */ public abstract Resource multiplyAndNormalizeUp( Resource r, double by, Resource stepFactor); - + + /** + * Multiply resource r by factor by + * and normalize up using step-factor stepFactor. + * + * @param r resource to be multiplied + * @param by multiplier array for all resource types + * @param stepFactor factor by which to normalize up + * @return resulting normalized resource + */ + public abstract Resource multiplyAndNormalizeUp( + Resource r, double[] by, Resource stepFactor); + /** * Multiply resource r by factor by * and normalize down using step-factor stepFactor. @@ -235,4 +247,16 @@ public abstract float divide( * @return returns true if any resource is zero. */ public abstract boolean isAnyMajorResourceZero(Resource resource); + + /** + * Get resource rand normalize down using step-factor + * stepFactor. + * + * @param r + * resource to be multiplied + * @param stepFactor + * factor by which to normalize down + * @return resulting normalized resource + */ + public abstract Resource normalizeDown(Resource r, Resource stepFactor); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java index 068e7f1cb3e..1c08844cf33 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java @@ -347,6 +347,11 @@ public static Resource multiplyAndAddTo( return lhs; } + public static Resource multiplyAndNormalizeUp(ResourceCalculator calculator, + Resource lhs, double[] by, Resource factor) { + return calculator.multiplyAndNormalizeUp(lhs, by, factor); + } + public static Resource multiplyAndNormalizeUp( ResourceCalculator calculator,Resource lhs, double by, Resource factor) { return calculator.multiplyAndNormalizeUp(lhs, by, factor); @@ -546,4 +551,9 @@ public static boolean isAnyMajorResourceZero(ResourceCalculator rc, Resource resource) { return rc.isAnyMajorResourceZero(resource); } + + public static Resource normalizeDown(ResourceCalculator calculator, + Resource resource, Resource factor) { + return calculator.normalizeDown(resource, factor); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java index a80f317bb4c..5196831e2ad 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/AbstractPreemptableResourceCalculator.java @@ -19,8 +19,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.PriorityUtilizationQueueOrderingPolicy; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import java.util.ArrayList; @@ -198,18 +201,33 @@ protected void computeFixpointAllocation(Resource totGuarant, private void resetCapacity(Resource clusterResource, Collection queues, boolean ignoreGuar) { Resource activeCap = Resource.newInstance(0, 0); + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); if (ignoreGuar) { for (TempQueuePerPartition q : queues) { - q.normalizedGuarantee = 1.0f / queues.size(); + for (int i = 0; i < maxLength; i++) { + q.normalizedGuarantee[i] = 1.0f / queues.size(); + } } } else { for (TempQueuePerPartition q : queues) { Resources.addTo(activeCap, q.getGuaranteed()); } for (TempQueuePerPartition q : queues) { - q.normalizedGuarantee = Resources.divide(rc, clusterResource, - q.getGuaranteed(), activeCap); + for (int i = 0; i < maxLength; i++) { + ResourceInformation nResourceInformation = q.getGuaranteed() + .getResourceInformation(i); + ResourceInformation dResourceInformation = activeCap + .getResourceInformation(i); + + long nValue = nResourceInformation.getValue(); + long dValue = UnitsConversionUtil.convert( + dResourceInformation.getUnits(), nResourceInformation.getUnits(), + dResourceInformation.getValue()); + if (dValue != 0) { + q.normalizedGuarantee[i] = (float) nValue / dValue; + } + } } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java index f843db402c4..748548a761e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/FifoCandidatesSelector.java @@ -140,10 +140,10 @@ // Can try preempting AMContainers (still saving atmost // maxAMCapacityForThisQueue AMResource's) if more resources are // required to be preemptionCandidates from this Queue. - Resource maxAMCapacityForThisQueue = Resources.multiply( - Resources.multiply(clusterResource, - leafQueue.getAbsoluteCapacity()), - leafQueue.getMaxAMResourcePerQueuePercent()); + Resource maxAMCapacityForThisQueue = Resources + .multiply( + leafQueue.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL), + leafQueue.getMaxAMResourcePerQueuePercent()); preemptAMContainers(clusterResource, selectedCandidates, skippedAMContainerlist, resToObtainByPartition, skippedAMSize, maxAMCapacityForThisQueue, @@ -199,7 +199,6 @@ private void preemptAMContainers(Resource clusterResource, * Given a target preemption for a specific application, select containers * to preempt (after unreserving all reservation for that app). */ - @SuppressWarnings("unchecked") private void preemptFrom(FiCaSchedulerApp app, Resource clusterResource, Map resToObtainByPartition, List skippedAMContainerlist, Resource skippedAMSize, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java index 2c072d2544f..8327cb906e4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicy.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; @@ -525,6 +526,13 @@ private TempQueuePerPartition cloneQueues(CSQueue curQueue, float absMaxCap = qc.getAbsoluteMaximumCapacity(partitionToLookAt); boolean preemptionDisabled = curQueue.getPreemptionDisabled(); + QueueResourceQuotas queueResourceQuotas = curQueue + .getQueueResourceQuotas(); + Resource effMinRes = queueResourceQuotas + .getEffectiveMinResource(partitionToLookAt); + Resource effMaxRes = queueResourceQuotas + .getEffectiveMaxResource(partitionToLookAt); + Resource current = Resources .clone(curQueue.getQueueResourceUsage().getUsed(partitionToLookAt)); Resource killable = Resources.none(); @@ -550,7 +558,7 @@ private TempQueuePerPartition cloneQueues(CSQueue curQueue, ret = new TempQueuePerPartition(queueName, current, preemptionDisabled, partitionToLookAt, killable, absCap, absMaxCap, partitionResource, - reserved, curQueue); + reserved, curQueue, effMinRes, effMaxRes); if (curQueue instanceof ParentQueue) { String configuredOrderingPolicy = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java index 89452f9c0d4..4d71223aaa3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TempQueuePerPartition.java @@ -22,9 +22,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.LinkedHashMap; import java.util.Map; @@ -46,7 +48,10 @@ Resource untouchableExtra; Resource preemptableExtra; - double normalizedGuarantee; + double[] normalizedGuarantee; + + private Resource effMinRes; + private Resource effMaxRes; final ArrayList children; private Collection apps; @@ -68,7 +73,8 @@ TempQueuePerPartition(String queueName, Resource current, boolean preemptionDisabled, String partition, Resource killable, float absCapacity, float absMaxCapacity, Resource totalPartitionResource, - Resource reserved, CSQueue queue) { + Resource reserved, CSQueue queue, Resource effMinRes, + Resource effMaxRes) { super(queueName, current, Resource.newInstance(0, 0), reserved, Resource.newInstance(0, 0)); @@ -84,7 +90,8 @@ pendingDeductReserved = Resources.createResource(0); } - this.normalizedGuarantee = Float.NaN; + this.normalizedGuarantee = new double[ResourceUtils + .getNumberOfKnownResourceTypes()]; this.children = new ArrayList<>(); this.apps = new ArrayList<>(); this.untouchableExtra = Resource.newInstance(0, 0); @@ -95,6 +102,8 @@ this.absCapacity = absCapacity; this.absMaxCapacity = absMaxCapacity; this.totalPartitionResource = totalPartitionResource; + this.effMinRes = effMinRes; + this.effMaxRes = effMaxRes; } public void setLeafQueue(LeafQueue l) { @@ -177,10 +186,18 @@ Resource offer(Resource avail, ResourceCalculator rc, } public Resource getGuaranteed() { + if(!effMinRes.equals(Resources.none())) { + return Resources.clone(effMinRes); + } + return Resources.multiply(totalPartitionResource, absCapacity); } public Resource getMax() { + if(!effMaxRes.equals(Resources.none())) { + return Resources.clone(effMaxRes); + } + return Resources.multiply(totalPartitionResource, absMaxCapacity); } @@ -226,8 +243,9 @@ public String toString() { sb.append(" NAME: " + queueName).append(" CUR: ").append(current) .append(" PEN: ").append(pending).append(" RESERVED: ").append(reserved) .append(" GAR: ").append(getGuaranteed()).append(" NORM: ") - .append(normalizedGuarantee).append(" IDEAL_ASSIGNED: ") - .append(idealAssigned).append(" IDEAL_PREEMPT: ").append(toBePreempted) + .append(Arrays.toString(normalizedGuarantee)) + .append(" IDEAL_ASSIGNED: ").append(idealAssigned) + .append(" IDEAL_PREEMPT: ").append(toBePreempted) .append(" ACTUAL_PREEMPT: ").append(getActuallyToBePreempted()) .append(" UNTOUCHABLE: ").append(untouchableExtra) .append(" PREEMPTABLE: ").append(preemptableExtra).append("\n"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java new file mode 100644 index 00000000000..c2953236f47 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java @@ -0,0 +1,198 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; +import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.util.resource.Resources; + +/** + * This class can be used to track resource usage in queue/user/app. + * + * And it is thread-safe + */ +public class AbstractResourceUsage { + protected ReadLock readLock; + protected WriteLock writeLock; + protected Map usages; + // short for no-label :) + private static final String NL = CommonNodeLabelsManager.NO_LABEL; + + public AbstractResourceUsage() { + ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + readLock = lock.readLock(); + writeLock = lock.writeLock(); + + usages = new HashMap(); + usages.put(NL, new UsageByLabel(NL)); + } + + // Usage enum here to make implement cleaner + public enum ResourceType { + // CACHED_USED and CACHED_PENDING may be read by anyone, but must only + // be written by ordering policies + USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), CACHED_PENDING( + 5), AMLIMIT(6), MIN_RESOURCE(7), MAX_RESOURCE(8), EFF_MIN_RESOURCE( + 9), EFF_MAX_RESOURCE( + 10), EFF_MIN_RESOURCE_UP(11), EFF_MAX_RESOURCE_UP(12); + + private int idx; + + private ResourceType(int value) { + this.idx = value; + } + } + + public static class UsageByLabel { + // usage by label, contains all UsageType + private Resource[] resArr; + + public UsageByLabel(String label) { + resArr = new Resource[ResourceType.values().length]; + for (int i = 0; i < resArr.length; i++) { + resArr[i] = Resource.newInstance(0, 0); + }; + } + + public Resource getUsed() { + return resArr[ResourceType.USED.idx]; + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append("{used=" + resArr[0] + "%, "); + sb.append("pending=" + resArr[1] + "%, "); + sb.append("am_used=" + resArr[2] + "%, "); + sb.append("reserved=" + resArr[3] + "%}"); + sb.append("min_eff=" + resArr[9] + "%, "); + sb.append("max_eff=" + resArr[10] + "%}"); + sb.append("min_effup=" + resArr[11] + "%, "); + return sb.toString(); + } + } + + private static Resource normalize(Resource res) { + if (res == null) { + return Resources.none(); + } + return res; + } + + protected Resource _get(String label, ResourceType type) { + if (label == null) { + label = RMNodeLabelsManager.NO_LABEL; + } + + try { + readLock.lock(); + UsageByLabel usage = usages.get(label); + if (null == usage) { + return Resources.none(); + } + return normalize(usage.resArr[type.idx]); + } finally { + readLock.unlock(); + } + } + + protected Resource _getAll(ResourceType type) { + try { + readLock.lock(); + Resource allOfType = Resources.createResource(0); + for (Map.Entry usageEntry : usages.entrySet()) { + //all usages types are initialized + Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]); + } + return allOfType; + } finally { + readLock.unlock(); + } + } + + private UsageByLabel getAndAddIfMissing(String label) { + if (label == null) { + label = RMNodeLabelsManager.NO_LABEL; + } + if (!usages.containsKey(label)) { + UsageByLabel u = new UsageByLabel(label); + usages.put(label, u); + return u; + } + + return usages.get(label); + } + + protected void _set(String label, ResourceType type, Resource res) { + try { + writeLock.lock(); + UsageByLabel usage = getAndAddIfMissing(label); + usage.resArr[type.idx] = res; + } finally { + writeLock.unlock(); + } + } + + protected void _inc(String label, ResourceType type, Resource res) { + try { + writeLock.lock(); + UsageByLabel usage = getAndAddIfMissing(label); + Resources.addTo(usage.resArr[type.idx], res); + } finally { + writeLock.unlock(); + } + } + + protected void _dec(String label, ResourceType type, Resource res) { + try { + writeLock.lock(); + UsageByLabel usage = getAndAddIfMissing(label); + Resources.subtractFrom(usage.resArr[type.idx], res); + } finally { + writeLock.unlock(); + } + } + + @Override + public String toString() { + try { + readLock.lock(); + return usages.toString(); + } finally { + readLock.unlock(); + } + } + + public Set getNodePartitionsSet() { + try { + readLock.lock(); + return usages.keySet(); + } finally { + readLock.unlock(); + } + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java new file mode 100644 index 00000000000..08b4d04d54d --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueResourceQuotas.java @@ -0,0 +1,115 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.scheduler; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; + +/** + * QueueResourceQuotas by Labels for following fields by label + * - EFFECTIVE_MIN_CAPACITY + * - EFFECTIVE_MAX_CAPACITY + * This class can be used to track resource usage in queue/user/app. + * + * And it is thread-safe + */ +public class QueueResourceQuotas extends AbstractResourceUsage { + // short for no-label :) + private static final String NL = CommonNodeLabelsManager.NO_LABEL; + + public QueueResourceQuotas() { + super(); + } + + /* + * Configured Minimum Resource + */ + public Resource getConfiguredMinResource() { + return _get(NL, ResourceType.MIN_RESOURCE); + } + + public Resource getConfiguredMinResource(String label) { + return _get(label, ResourceType.MIN_RESOURCE); + } + + public void setConfiguredMinResource(String label, Resource res) { + _set(label, ResourceType.MIN_RESOURCE, res); + } + + public void setConfiguredMinResource(Resource res) { + _set(NL, ResourceType.MIN_RESOURCE, res); + } + + /* + * Configured Maximum Resource + */ + public Resource getConfiguredMaxResource() { + return getConfiguredMaxResource(NL); + } + + public Resource getConfiguredMaxResource(String label) { + return _get(label, ResourceType.MAX_RESOURCE); + } + + public void setConfiguredMaxResource(Resource res) { + setConfiguredMaxResource(NL, res); + } + + public void setConfiguredMaxResource(String label, Resource res) { + _set(label, ResourceType.MAX_RESOURCE, res); + } + + /* + * Effective Minimum Resource + */ + public Resource getEffectiveMinResource() { + return _get(NL, ResourceType.EFF_MIN_RESOURCE); + } + + public Resource getEffectiveMinResource(String label) { + return _get(label, ResourceType.EFF_MIN_RESOURCE); + } + + public void setEffectiveMinResource(String label, Resource res) { + _set(label, ResourceType.EFF_MIN_RESOURCE, res); + } + + public void setEffectiveMinResource(Resource res) { + _set(NL, ResourceType.EFF_MIN_RESOURCE, res); + } + + /* + * Effective Maximum Resource + */ + public Resource getEffectiveMaxResource() { + return getEffectiveMaxResource(NL); + } + + public Resource getEffectiveMaxResource(String label) { + return _get(label, ResourceType.EFF_MAX_RESOURCE); + } + + public void setEffectiveMaxResource(Resource res) { + setEffectiveMaxResource(NL, res); + } + + public void setEffectiveMaxResource(String label, Resource res) { + _set(label, ResourceType.EFF_MAX_RESOURCE, res); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java index 6f0c7d20a80..ede4aec1dd2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java @@ -39,63 +39,12 @@ * * And it is thread-safe */ -public class ResourceUsage { - private ReadLock readLock; - private WriteLock writeLock; - private Map usages; +public class ResourceUsage extends AbstractResourceUsage { // short for no-label :) private static final String NL = CommonNodeLabelsManager.NO_LABEL; - private final UsageByLabel usageNoLabel; public ResourceUsage() { - ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - readLock = lock.readLock(); - writeLock = lock.writeLock(); - - usages = new HashMap(); - usageNoLabel = new UsageByLabel(NL); - usages.put(NL, usageNoLabel); - } - - // Usage enum here to make implement cleaner - private enum ResourceType { - //CACHED_USED and CACHED_PENDING may be read by anyone, but must only - //be written by ordering policies - USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4), - CACHED_PENDING(5), AMLIMIT(6); - - private int idx; - - private ResourceType(int value) { - this.idx = value; - } - } - - private static class UsageByLabel { - // usage by label, contains all UsageType - private Resource[] resArr; - - public UsageByLabel(String label) { - resArr = new Resource[ResourceType.values().length]; - for (int i = 0; i < resArr.length; i++) { - resArr[i] = Resource.newInstance(0, 0); - }; - } - - public Resource getUsed() { - return resArr[ResourceType.USED.idx]; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("{used=" + resArr[0] + "%, "); - sb.append("pending=" + resArr[1] + "%, "); - sb.append("am_used=" + resArr[2] + "%, "); - sb.append("reserved=" + resArr[3] + "%}"); - sb.append("am_limit=" + resArr[6] + "%, "); - return sb.toString(); - } + super(); } /* @@ -109,22 +58,6 @@ public Resource getUsed(String label) { return _get(label, ResourceType.USED); } - public Resource getCachedUsed() { - return _get(NL, ResourceType.CACHED_USED); - } - - public Resource getCachedUsed(String label) { - return _get(label, ResourceType.CACHED_USED); - } - - public Resource getCachedPending() { - return _get(NL, ResourceType.CACHED_PENDING); - } - - public Resource getCachedPending(String label) { - return _get(label, ResourceType.CACHED_PENDING); - } - public void incUsed(String label, Resource res) { _inc(label, ResourceType.USED, res); } @@ -145,7 +78,7 @@ public void setUsed(Resource res) { setUsed(NL, res); } - public void copyAllUsed(ResourceUsage other) { + public void copyAllUsed(AbstractResourceUsage other) { try { writeLock.lock(); for (Entry entry : other.usages.entrySet()) { @@ -160,22 +93,6 @@ public void setUsed(String label, Resource res) { _set(label, ResourceType.USED, res); } - public void setCachedUsed(String label, Resource res) { - _set(label, ResourceType.CACHED_USED, res); - } - - public void setCachedUsed(Resource res) { - _set(NL, ResourceType.CACHED_USED, res); - } - - public void setCachedPending(String label, Resource res) { - _set(label, ResourceType.CACHED_PENDING, res); - } - - public void setCachedPending(Resource res) { - _set(NL, ResourceType.CACHED_PENDING, res); - } - /* * Pending */ @@ -281,6 +198,47 @@ public void setAMUsed(String label, Resource res) { _set(label, ResourceType.AMUSED, res); } + public Resource getAllPending() { + return _getAll(ResourceType.PENDING); + } + + public Resource getAllUsed() { + return _getAll(ResourceType.USED); + } + + // Cache Used + public Resource getCachedUsed() { + return _get(NL, ResourceType.CACHED_USED); + } + + public Resource getCachedUsed(String label) { + return _get(label, ResourceType.CACHED_USED); + } + + public Resource getCachedPending() { + return _get(NL, ResourceType.CACHED_PENDING); + } + + public Resource getCachedPending(String label) { + return _get(label, ResourceType.CACHED_PENDING); + } + + public void setCachedUsed(String label, Resource res) { + _set(label, ResourceType.CACHED_USED, res); + } + + public void setCachedUsed(Resource res) { + _set(NL, ResourceType.CACHED_USED, res); + } + + public void setCachedPending(String label, Resource res) { + _set(label, ResourceType.CACHED_PENDING, res); + } + + public void setCachedPending(Resource res) { + _set(NL, ResourceType.CACHED_PENDING, res); + } + /* * AM-Resource Limit */ @@ -316,94 +274,6 @@ public void setAMLimit(String label, Resource res) { _set(label, ResourceType.AMLIMIT, res); } - private static Resource normalize(Resource res) { - if (res == null) { - return Resources.none(); - } - return res; - } - - private Resource _get(String label, ResourceType type) { - if (label == null || label.equals(NL)) { - return normalize(usageNoLabel.resArr[type.idx]); - } - try { - readLock.lock(); - UsageByLabel usage = usages.get(label); - if (null == usage) { - return Resources.none(); - } - return normalize(usage.resArr[type.idx]); - } finally { - readLock.unlock(); - } - } - - private Resource _getAll(ResourceType type) { - try { - readLock.lock(); - Resource allOfType = Resources.createResource(0); - for (Map.Entry usageEntry : usages.entrySet()) { - //all usages types are initialized - Resources.addTo(allOfType, usageEntry.getValue().resArr[type.idx]); - } - return allOfType; - } finally { - readLock.unlock(); - } - } - - public Resource getAllPending() { - return _getAll(ResourceType.PENDING); - } - - public Resource getAllUsed() { - return _getAll(ResourceType.USED); - } - - private UsageByLabel getAndAddIfMissing(String label) { - if (label == null || label.equals(NL)) { - return usageNoLabel; - } - if (!usages.containsKey(label)) { - UsageByLabel u = new UsageByLabel(label); - usages.put(label, u); - return u; - } - - return usages.get(label); - } - - private void _set(String label, ResourceType type, Resource res) { - try { - writeLock.lock(); - UsageByLabel usage = getAndAddIfMissing(label); - usage.resArr[type.idx] = res; - } finally { - writeLock.unlock(); - } - } - - private void _inc(String label, ResourceType type, Resource res) { - try { - writeLock.lock(); - UsageByLabel usage = getAndAddIfMissing(label); - Resources.addTo(usage.resArr[type.idx], res); - } finally { - writeLock.unlock(); - } - } - - private void _dec(String label, ResourceType type, Resource res) { - try { - writeLock.lock(); - UsageByLabel usage = getAndAddIfMissing(label); - Resources.subtractFrom(usage.resArr[type.idx], res); - } finally { - writeLock.unlock(); - } - } - public Resource getCachedDemand(String label) { try { readLock.lock(); @@ -415,23 +285,4 @@ public Resource getCachedDemand(String label) { readLock.unlock(); } } - - @Override - public String toString() { - try { - readLock.lock(); - return usages.toString(); - } finally { - readLock.unlock(); - } - } - - public Set getNodePartitionsSet() { - try { - readLock.lock(); - return usages.keySet(); - } finally { - readLock.unlock(); - } - } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 74c85ce2bf5..9caf5891000 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -56,6 +56,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.AbsoluteResourceType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; @@ -87,6 +89,7 @@ final ResourceCalculator resourceCalculator; Set accessibleLabels; + Set resourceTypes; final RMNodeLabelsManager labelManager; String defaultLabelExpression; @@ -102,6 +105,14 @@ // etc. QueueCapacities queueCapacities; + QueueResourceQuotas queueResourceQuotas; + + protected enum CapacityConfigType { + NONE, PERCENTAGE, ABSOLUTE_RESOURCE + }; + protected CapacityConfigType capacityConfigType = + CapacityConfigType.NONE; + private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null); protected CapacitySchedulerContext csContext; @@ -141,6 +152,9 @@ public AbstractCSQueue(CapacitySchedulerContext cs, // initialize QueueCapacities queueCapacities = new QueueCapacities(parent == null); + // initialize queueResourceQuotas + queueResourceQuotas = new QueueResourceQuotas(); + ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); readLock = lock.readLock(); writeLock = lock.writeLock(); @@ -276,6 +290,10 @@ void setupQueueConfigs(Resource clusterResource) this.defaultLabelExpression = csContext.getConfiguration().getDefaultNodeLabelExpression( getQueuePath()); + this.resourceTypes = new HashSet(); + for (AbsoluteResourceType type : AbsoluteResourceType.values()) { + resourceTypes.add(type.toString().toLowerCase()); + } // inherit from parent if labels not set if (this.accessibleLabels == null && parent != null) { @@ -292,6 +310,11 @@ void setupQueueConfigs(Resource clusterResource) // After we setup labels, we can setup capacities setupConfigurableCapacities(); + // Also fetch minimum/maximum resource constraint for this queue if + // configured. + capacityConfigType = CapacityConfigType.NONE; + updateConfigurableResourceRequirement(getQueuePath(), clusterResource); + this.maximumAllocation = csContext.getConfiguration().getMaximumAllocationPerQueue( getQueuePath()); @@ -364,6 +387,123 @@ void setupQueueConfigs(Resource clusterResource) return unionInheritedWeights; } + protected void updateConfigurableResourceRequirement(String queuePath, + Resource clusterResource) { + CapacitySchedulerConfiguration conf = csContext.getConfiguration(); + Set configuredNodelabels = conf.getConfiguredNodeLabels(queuePath); + + for (String label : configuredNodelabels) { + Resource minResource = conf.getMinimumResourceRequirement(label, + queuePath, resourceTypes); + Resource maxResource = conf.getMaximumResourceRequirement(label, + queuePath, resourceTypes); + + if (LOG.isDebugEnabled()) { + LOG.debug("capacityConfigType is '" + capacityConfigType + + "' for queue '" + getQueueName()); + } + if (this.capacityConfigType.equals(CapacityConfigType.NONE)) { + this.capacityConfigType = (!minResource.equals(Resources.none()) + && queueCapacities.getAbsoluteCapacity(label) == 0f) + ? CapacityConfigType.ABSOLUTE_RESOURCE + : CapacityConfigType.PERCENTAGE; + if (LOG.isDebugEnabled()) { + LOG.debug("capacityConfigType is updated as '" + capacityConfigType + + "' for queue '" + getQueueName()); + } + } + + validateAbsoluteVsPercentageCapacityConfig(minResource); + + // If min resource for a resource type is greater than its max resource, + // throw exception to handle such error configs. + if (!maxResource.equals(Resources.none()) && Resources.greaterThan( + resourceCalculator, clusterResource, minResource, maxResource)) { + throw new IllegalArgumentException("Min resource configuration " + + minResource + " is greater than its max value:" + maxResource + + " in queue:" + getQueueName()); + } + + // If parent's max resource is lesser to a specific child's max + // resource, throw exception to handle such error configs. + if (parent != null) { + Resource parentMaxRes = parent.getQueueResourceQuotas() + .getConfiguredMaxResource(label); + if (Resources.greaterThan(resourceCalculator, clusterResource, + parentMaxRes, Resources.none())) { + if (Resources.greaterThan(resourceCalculator, clusterResource, + maxResource, parentMaxRes)) { + throw new IllegalArgumentException("Max resource configuration " + + maxResource + " is greater than parents max value:" + + parentMaxRes + " in queue:" + getQueueName()); + } + + // If child's max resource is not set, but its parent max resource is + // set, we must set child max resource to its parent's. + if (maxResource.equals(Resources.none()) + && !minResource.equals(Resources.none())) { + maxResource = Resources.clone(parentMaxRes); + } + } + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Updating absolute resource configuration for queue:" + + getQueueName() + " as minResource=" + minResource + + " and maxResource=" + maxResource); + } + + queueResourceQuotas.setConfiguredMinResource(label, minResource); + queueResourceQuotas.setConfiguredMaxResource(label, maxResource); + } + } + + private void validateAbsoluteVsPercentageCapacityConfig( + Resource minResource) { + CapacityConfigType localType = CapacityConfigType.PERCENTAGE; + if (!minResource.equals(Resources.none())) { + localType = CapacityConfigType.ABSOLUTE_RESOURCE; + } + + if (!queueName.equals("root") + && !this.capacityConfigType.equals(localType)) { + throw new IllegalArgumentException("Queue '" + getQueueName() + + "' should use either percentage based capacity" + + " configuration or absolute resource."); + } + } + + @Override + public CapacityConfigType getCapacityConfigType() { + return capacityConfigType; + } + + @Override + public Resource getEffectiveCapacity(String label) { + return Resources + .clone(getQueueResourceQuotas().getEffectiveMinResource(label)); + } + + @Override + public Resource getEffectiveCapacityDown(String label, Resource factor) { + return Resources.normalizeDown(resourceCalculator, + getQueueResourceQuotas().getEffectiveMinResource(label), + minimumAllocation); + } + + @Override + public Resource getEffectiveMaxCapacity(String label) { + return Resources + .clone(getQueueResourceQuotas().getEffectiveMaxResource(label)); + } + + @Override + public Resource getEffectiveMaxCapacityDown(String label, Resource factor) { + return Resources.normalizeDown(resourceCalculator, + getQueueResourceQuotas().getEffectiveMaxResource(label), + minimumAllocation); + } + private void initializeQueueState(QueueState previousState, QueueState configuredState, QueueState parentState) { // verify that we can not any value for State other than RUNNING/STOPPED @@ -479,6 +619,14 @@ public QueueStatistics getQueueStatistics() { queueConfiguration.setMaxCapacity(maxCapacity); queueConfiguration.setAbsoluteMaxCapacity(absMaxCapacity); queueConfiguration.setMaxAMPercentage(maxAMPercentage); + queueConfiguration.setConfiguredMinCapacity( + queueResourceQuotas.getConfiguredMinResource(nodeLabel)); + queueConfiguration.setConfiguredMaxCapacity( + queueResourceQuotas.getConfiguredMaxResource(nodeLabel)); + queueConfiguration.setEffectiveMinCapacity( + queueResourceQuotas.getEffectiveMinResource(nodeLabel)); + queueConfiguration.setEffectiveMaxCapacity( + queueResourceQuotas.getEffectiveMaxResource(nodeLabel)); queueConfigurations.put(nodeLabel, queueConfiguration); } return queueConfigurations; @@ -555,6 +703,11 @@ public ResourceUsage getQueueResourceUsage() { } @Override + public QueueResourceQuotas getQueueResourceQuotas() { + return queueResourceQuotas; + } + + @Override public ReentrantReadWriteLock.ReadLock getReadLock() { return readLock; } @@ -604,7 +757,7 @@ private Resource getCurrentLimitResource(String nodePartition, * limit-set-by-parent) */ Resource queueMaxResource = - getQueueMaxResource(nodePartition, clusterResource); + getQueueMaxResource(nodePartition); return Resources.min(resourceCalculator, clusterResource, queueMaxResource, currentResourceLimits.getLimit()); @@ -617,11 +770,8 @@ private Resource getCurrentLimitResource(String nodePartition, return Resources.none(); } - Resource getQueueMaxResource(String nodePartition, Resource clusterResource) { - return Resources.multiplyAndNormalizeDown(resourceCalculator, - labelManager.getResourceByLabel(nodePartition, clusterResource), - queueCapacities.getAbsoluteMaximumCapacity(nodePartition), - minimumAllocation); + Resource getQueueMaxResource(String nodePartition) { + return getEffectiveMaxCapacity(nodePartition); } public boolean hasChildQueues() { @@ -782,7 +932,7 @@ public void incUsedResource(String nodeLabel, Resource resourceToInc, queueUsage.incUsed(nodeLabel, resourceToInc); CSQueueUtils.updateUsedCapacity(resourceCalculator, labelManager.getResourceByLabel(nodeLabel, Resources.none()), - nodeLabel, this); + Resources.none(), nodeLabel, this); if (null != parent) { parent.incUsedResource(nodeLabel, resourceToInc, null); } @@ -798,7 +948,7 @@ public void decUsedResource(String nodeLabel, Resource resourceToDec, queueUsage.decUsed(nodeLabel, resourceToDec); CSQueueUtils.updateUsedCapacity(resourceCalculator, labelManager.getResourceByLabel(nodeLabel, Resources.none()), - nodeLabel, this); + Resources.none(), nodeLabel, this); if (null != parent) { parent.decUsedResource(nodeLabel, resourceToDec, null); } @@ -904,7 +1054,7 @@ public boolean accept(Resource cluster, Resource maxResourceLimit; if (allocation.getSchedulingMode() == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY) { - maxResourceLimit = getQueueMaxResource(partition, cluster); + maxResourceLimit = getQueueMaxResource(partition); } else{ maxResourceLimit = labelManager.getResourceByLabel( schedulerContainer.getNodePartition(), cluster); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 43e7f532872..624fcc7c9ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -25,7 +25,6 @@ import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.security.AccessControlException; @@ -41,15 +40,19 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue.CapacityConfigType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet; +import com.google.common.annotations.VisibleForTesting; + /** * CSQueue represents a node in the tree of * hierarchical queues in the {@link CapacityScheduler}. @@ -355,4 +358,40 @@ public void validateSubmitApplication(ApplicationId applicationId, * @return map of usernames and corresponding weight */ Map getUserWeights(); + + /** + * Get QueueResourceQuotas associated with each queue. + * @return QueueResourceQuotas + */ + public QueueResourceQuotas getQueueResourceQuotas(); + + /** + * Get CapacityConfigType as PERCENTAGE or ABSOLUTE_RESOURCE + * @return CapacityConfigType + */ + public CapacityConfigType getCapacityConfigType(); + + /** + * Get effective capacity of queue. If min/max resource is configured, + * preference will be given to absolute configuration over normal capacity. + * Also round down the result to normalizeDown. + * + * @param label + * partition + * @return effective queue capacity + */ + Resource getEffectiveCapacity(String label); + Resource getEffectiveCapacityDown(String label, Resource factor); + + /** + * Get effective max capacity of queue. If min/max resource is configured, + * preference will be given to absolute configuration over normal capacity. + * Also round down the result to normalizeDown. + * + * @param label + * partition + * @return effective max queue capacity + */ + Resource getEffectiveMaxCapacity(String label); + Resource getEffectiveMaxCapacityDown(String label, Resource factor); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java index e1014c11fc8..81dec80d0eb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java @@ -150,7 +150,7 @@ private static void loadCapacitiesByLabelsFromConf(String queuePath, } } } - + // Set absolute capacities for {capacity, maximum-capacity} private static void updateAbsoluteCapacitiesByNodeLabels( QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) { @@ -180,8 +180,8 @@ private static void updateAbsoluteCapacitiesByNodeLabels( * used resource for all partitions of this queue. */ public static void updateUsedCapacity(final ResourceCalculator rc, - final Resource totalPartitionResource, String nodePartition, - AbstractCSQueue childQueue) { + final Resource totalPartitionResource, Resource clusterResource, + String nodePartition, AbstractCSQueue childQueue) { QueueCapacities queueCapacities = childQueue.getQueueCapacities(); CSQueueMetrics queueMetrics = childQueue.getMetrics(); ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage(); @@ -193,11 +193,8 @@ public static void updateUsedCapacity(final ResourceCalculator rc, if (Resources.greaterThan(rc, totalPartitionResource, totalPartitionResource, Resources.none())) { - // queueGuaranteed = totalPartitionedResource * - // absolute_capacity(partition) - Resource queueGuranteedResource = - Resources.multiply(totalPartitionResource, - queueCapacities.getAbsoluteCapacity(nodePartition)); + Resource queueGuranteedResource = childQueue + .getEffectiveCapacity(nodePartition); // make queueGuranteed >= minimum_allocation to avoid divided by 0. queueGuranteedResource = @@ -248,9 +245,7 @@ private static Resource getMaxAvailableResourceToQueue( for (String partition : nodeLabels) { // Calculate guaranteed resource for a label in a queue by below logic. // (total label resource) * (absolute capacity of label in that queue) - Resource queueGuranteedResource = Resources.multiply(nlm - .getResourceByLabel(partition, cluster), queue.getQueueCapacities() - .getAbsoluteCapacity(partition)); + Resource queueGuranteedResource = queue.getEffectiveCapacity(partition); // Available resource in queue for a specific label will be calculated as // {(guaranteed resource for a label in a queue) - @@ -289,15 +284,14 @@ public static void updateQueueStatistics( ResourceUsage queueResourceUsage = childQueue.getQueueResourceUsage(); if (nodePartition == null) { - for (String partition : Sets.union( - queueCapacities.getNodePartitionsSet(), + for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(), queueResourceUsage.getNodePartitionsSet())) { updateUsedCapacity(rc, nlm.getResourceByLabel(partition, cluster), - partition, childQueue); + cluster, partition, childQueue); } } else { updateUsedCapacity(rc, nlm.getResourceByLabel(nodePartition, cluster), - nodePartition, childQueue); + cluster, nodePartition, childQueue); } // Update queue metrics w.r.t node labels. In a generic way, we can diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java index 4515453c933..f81e097adfb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java @@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.FifoOrderingPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; import org.apache.hadoop.yarn.util.resource.Resources; @@ -60,6 +61,8 @@ import java.util.List; import java.util.Map; import java.util.Map.Entry; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import java.util.Set; import java.util.StringTokenizer; @@ -317,6 +320,21 @@ @Private public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1; + /** Configuring absolute min/max resources in a queue **/ + @Private + public static final String MINIMUM_RESOURCE = "min-resource"; + + @Private + public static final String MAXIMUM_RESOURCE = "max-resource"; + + public static final String DEFAULT_RESOURCE_TYPES = "memory,vcores"; + + public static final String PATTERN_FOR_ABSOLUTE_RESOURCE = "\\[([^\\]]+)"; + + public enum AbsoluteResourceType { + MEMORY, VCORES; + } + AppPriorityACLConfigurationParser priorityACLConfig = new AppPriorityACLConfigurationParser(); public CapacitySchedulerConfiguration() { @@ -394,7 +412,7 @@ public void setMaximumApplicationMasterResourcePerQueuePercent(String queue, public float getNonLabeledQueueCapacity(String queue) { float capacity = queue.equals("root") ? 100.0f : getFloat( - getQueuePrefix(queue) + CAPACITY, UNDEFINED); + getQueuePrefix(queue) + CAPACITY, 0f); if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) { throw new IllegalArgumentException("Illegal " + "capacity of " + capacity + " for queue " + queue); @@ -1676,4 +1694,163 @@ public void setAutoCreatedLeafQueueTemplateMaxCapacity(String queuePath, queuePath); setMaximumCapacity(leafQueueConfPrefix, val); } + + public static String getUnits(String resourceValue) { + String units; + for (int i = 0; i < resourceValue.length(); i++) { + if (Character.isAlphabetic(resourceValue.charAt(i))) { + units = resourceValue.substring(i); + if (StringUtils.isAlpha(units)) { + return units; + } + } + } + return ""; + } + + /** + * Get absolute minimum resource requirement for a queue. + * + * @param label + * NodeLabel + * @param queue + * queue path + * @param resourceTypes + * Resource types + * @return ResourceInformation + */ + public Resource getMinimumResourceRequirement(String label, String queue, + Set resourceTypes) { + return internalGetLabeledResourceRequirementForQueue(queue, label, + resourceTypes, MINIMUM_RESOURCE); + } + + /** + * Get absolute maximum resource requirement for a queue. + * + * @param label + * NodeLabel + * @param queue + * queue path + * @param resourceTypes + * Resource types + * @return Resource + */ + public Resource getMaximumResourceRequirement(String label, String queue, + Set resourceTypes) { + return internalGetLabeledResourceRequirementForQueue(queue, label, + resourceTypes, MAXIMUM_RESOURCE); + } + + @VisibleForTesting + public void setMinimumResourceRequirement(String label, String queue, + Resource resource) { + updateMinMaxResourceToConf(label, queue, resource, MINIMUM_RESOURCE); + } + + @VisibleForTesting + public void setMaximumResourceRequirement(String label, String queue, + Resource resource) { + updateMinMaxResourceToConf(label, queue, resource, MAXIMUM_RESOURCE); + } + + private void updateMinMaxResourceToConf(String label, String queue, + Resource resource, String type) { + if (queue.equals("root")) { + throw new IllegalArgumentException( + "Cannot set resource, root queue will take 100% of cluster capacity"); + } + + StringBuilder resourceString = new StringBuilder(); + resourceString + .append("[" + AbsoluteResourceType.MEMORY.toString().toLowerCase() + "=" + + resource.getMemorySize() + "," + + AbsoluteResourceType.VCORES.toString().toLowerCase() + "=" + + resource.getVirtualCores() + "]"); + + String prefix = getQueuePrefix(queue) + type; + if (!label.isEmpty()) { + prefix = getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + + DOT + type; + } + set(prefix, resourceString.toString()); + } + + private Resource internalGetLabeledResourceRequirementForQueue(String queue, + String label, Set resourceTypes, String suffix) { + String propertyName = getNodeLabelPrefix(queue, label) + suffix; + String resourceString = get(propertyName); + if (resourceString == null || resourceString.isEmpty()) { + return Resources.none(); + } + + // Define resource here. + Resource resource = Resource.newInstance(0l, 0); + Matcher matcher = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE) + .matcher(resourceString); + /* + * Absolute resource configuration for a queue will be grouped by "[]". + * Syntax of absolute resource config could be like below + * "memory=4Gi vcores=2". Ideally this means "4GB of memory and 2 vcores". + */ + if (matcher.find()) { + // Get the sub-group. + String subGroup = matcher.group(1); + if (subGroup.trim().isEmpty()) { + return Resources.none(); + } + + for (String kvPair : subGroup.trim().split(",")) { + String[] splits = kvPair.split("="); + + // Ensure that each sub string is key value pair separated by '='. + if (splits != null && splits.length > 1) { + updateResourceValuesFromConfig(resourceTypes, resource, splits); + } + } + } + + // Memory has to be configured always. + if (resource.getMemorySize() == 0l) { + return Resources.none(); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("CSConf - getAbsolueResourcePerQueue: prefix=" + + getNodeLabelPrefix(queue, label) + ", capacity=" + resource); + } + return resource; + } + + private void updateResourceValuesFromConfig(Set resourceTypes, + Resource resource, String[] splits) { + + // If key is not a valid type, skip it. + if (!resourceTypes.contains(splits[0])) { + return; + } + + String units = getUnits(splits[1]); + Long resourceValue = Long + .valueOf(splits[1].substring(0, splits[1].length() - units.length())); + + // Convert all incoming units to MB if units is configured. + if (!units.isEmpty()) { + resourceValue = UnitsConversionUtil.convert(units, "Mi", resourceValue); + } + + // map it based on key. + AbsoluteResourceType resType = AbsoluteResourceType + .valueOf(StringUtils.toUpperCase(splits[0].trim())); + switch (resType) { + case MEMORY : + resource.setMemorySize(resourceValue); + break; + case VCORES : + resource.setVirtualCores(resourceValue.intValue()); + break; + default : + break; + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index e8342d956ed..41ec4ba762f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -671,12 +671,7 @@ public Resource getUserAMResourceLimitPerPartition( 1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1)); effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f); - Resource queuePartitionResource = Resources - .multiplyAndNormalizeUp(resourceCalculator, - labelManager.getResourceByLabel(nodePartition, - lastClusterResource), - queueCapacities.getAbsoluteCapacity(nodePartition), - minimumAllocation); + Resource queuePartitionResource = getEffectiveCapacity(nodePartition); Resource userAMLimit = Resources.multiplyAndNormalizeUp( resourceCalculator, queuePartitionResource, @@ -705,11 +700,7 @@ public Resource calculateAndGetAMResourceLimitPerPartition( * non-labeled), * with per-partition am-resource-percent to get the max am * resource limit for this queue and partition. */ - Resource queuePartitionResource = Resources.multiplyAndNormalizeUp( - resourceCalculator, - labelManager.getResourceByLabel(nodePartition, lastClusterResource), - queueCapacities.getAbsoluteCapacity(nodePartition), - minimumAllocation); + Resource queuePartitionResource = getEffectiveCapacity(nodePartition); Resource queueCurrentLimit = Resources.none(); // For non-labeled partition, we need to consider the current queue @@ -965,6 +956,14 @@ private FiCaSchedulerApp getApplication( private void setPreemptionAllowed(ResourceLimits limits, String nodePartition) { // Set preemption-allowed: // For leaf queue, only under-utilized queue is allowed to preempt resources from other queues + if (!queueResourceQuotas.getEffectiveMinResource(nodePartition) + .equals(Resources.none())) { + limits.setIsAllowPreemption(Resources.lessThan(resourceCalculator, + csContext.getClusterResource(), queueUsage.getUsed(nodePartition), + queueResourceQuotas.getEffectiveMinResource(nodePartition))); + return; + } + float usedCapacity = queueCapacities.getAbsoluteUsedCapacity(nodePartition); float guaranteedCapacity = queueCapacities.getAbsoluteCapacity(nodePartition); limits.setIsAllowPreemption(usedCapacity < guaranteedCapacity); @@ -1344,7 +1343,7 @@ private Resource getHeadroom(User user, currentPartitionResourceLimit = partition.equals(RMNodeLabelsManager.NO_LABEL) ? currentPartitionResourceLimit - : getQueueMaxResource(partition, clusterResource); + : getQueueMaxResource(partition); Resource headroom = Resources.componentwiseMin( Resources.subtract(userLimitResource, user.getUsed(partition)), @@ -1716,12 +1715,8 @@ private void updateCurrentResourceLimits( // this. So need cap limits by queue's max capacity here. this.cachedResourceLimitsForHeadroom = new ResourceLimits(currentResourceLimits.getLimit()); - Resource queueMaxResource = - Resources.multiplyAndNormalizeDown(resourceCalculator, labelManager - .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource), - queueCapacities - .getAbsoluteMaximumCapacity(RMNodeLabelsManager.NO_LABEL), - minimumAllocation); + Resource queueMaxResource = getEffectiveMaxCapacityDown( + RMNodeLabelsManager.NO_LABEL, minimumAllocation); this.cachedResourceLimitsForHeadroom.setLimit(Resources.min( resourceCalculator, clusterResource, queueMaxResource, currentResourceLimits.getLimit())); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 959ca51eb2f..a427fb135ab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -25,6 +25,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; @@ -43,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.QueueState; import org.apache.hadoop.yarn.api.records.QueueUserACLInfo; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.AccessType; @@ -67,6 +69,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSet; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.CandidateNodeSetUtils; +import org.apache.hadoop.yarn.util.UnitsConversionUtil; +import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; @Private @@ -162,31 +167,78 @@ void setChildQueues(Collection childQueues) { writeLock.lock(); // Validate float childCapacities = 0; + Resource minResDefaultLabel = Resources.createResource(0, 0); for (CSQueue queue : childQueues) { childCapacities += queue.getCapacity(); + Resources.addTo(minResDefaultLabel, queue.getQueueResourceQuotas() + .getConfiguredMinResource()); + + // If any child queue is using percentage based capacity model vs parent + // queues' absolute configuration or vice versa, throw back an + // exception. + if (!queueName.equals("root") && getCapacity() != 0f + && !queue.getQueueResourceQuotas().getConfiguredMinResource() + .equals(Resources.none())) { + throw new IllegalArgumentException("Parent queue '" + getQueueName() + + "' and child queue '" + queue.getQueueName() + + "' should use either percentage based capacity" + + " configuration or absolute resource together."); + } } + float delta = Math.abs(1.0f - childCapacities); // crude way to check // allow capacities being set to 0, and enforce child 0 if parent is 0 - if (((queueCapacities.getCapacity() > 0) && (delta > PRECISION)) || ( - (queueCapacities.getCapacity() == 0) && (childCapacities > 0))) { - throw new IllegalArgumentException( - "Illegal" + " capacity of " + childCapacities - + " for children of queue " + queueName); + if ((minResDefaultLabel.equals(Resources.none()) + && (queueCapacities.getCapacity() > 0) && (delta > PRECISION)) + || ((queueCapacities.getCapacity() == 0) && (childCapacities > 0))) { + throw new IllegalArgumentException("Illegal" + " capacity of " + + childCapacities + " for children of queue " + queueName); } // check label capacities for (String nodeLabel : queueCapacities.getExistingNodeLabels()) { float capacityByLabel = queueCapacities.getCapacity(nodeLabel); // check children's labels float sum = 0; + Resource minRes = Resources.createResource(0, 0); + Resource resourceByLabel = labelManager.getResourceByLabel(nodeLabel, + scheduler.getClusterResource()); for (CSQueue queue : childQueues) { sum += queue.getQueueCapacities().getCapacity(nodeLabel); + + // If any child queue of a label is using percentage based capacity + // model vs parent queues' absolute configuration or vice versa, throw + // back an exception + if (!queueName.equals("root") && !this.capacityConfigType + .equals(queue.getCapacityConfigType())) { + throw new IllegalArgumentException("Parent queue '" + getQueueName() + + "' and child queue '" + queue.getQueueName() + + "' should use either percentage based capacity" + + "configuration or absolute resource together for label:" + + nodeLabel); + } + + // Accumulate all min/max resource configured for all child queues. + Resources.addTo(minRes, queue.getQueueResourceQuotas() + .getConfiguredMinResource(nodeLabel)); } - if ((capacityByLabel > 0 && Math.abs(1.0f - sum) > PRECISION) + if ((minResDefaultLabel.equals(Resources.none()) && capacityByLabel > 0 + && Math.abs(1.0f - sum) > PRECISION) || (capacityByLabel == 0) && (sum > 0)) { throw new IllegalArgumentException( "Illegal" + " capacity of " + sum + " for children of queue " + queueName + " for label=" + nodeLabel); } + + // Ensure that for each parent queue: parent.min-resource >= + // Σ(child.min-resource). + Resource parentMinResource = queueResourceQuotas + .getConfiguredMinResource(nodeLabel); + if (!parentMinResource.equals(Resources.none()) && Resources.lessThan( + resourceCalculator, resourceByLabel, parentMinResource, minRes)) { + throw new IllegalArgumentException("Parent Queues" + " capacity: " + + parentMinResource + " is less than" + " to its children:" + + minRes + " for queue:" + queueName); + } } this.childQueues.clear(); @@ -687,11 +739,8 @@ private ResourceLimits getResourceLimitsOfChild(CSQueue child, child.getQueueResourceUsage().getUsed(nodePartition)); // Get child's max resource - Resource childConfiguredMaxResource = Resources.multiplyAndNormalizeDown( - resourceCalculator, - labelManager.getResourceByLabel(nodePartition, clusterResource), - child.getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition), - minimumAllocation); + Resource childConfiguredMaxResource = getEffectiveMaxCapacityDown( + nodePartition, minimumAllocation); // Child's limit should be capped by child configured max resource childLimit = @@ -827,6 +876,14 @@ public void updateClusterResource(Resource clusterResource, ResourceLimits resourceLimits) { try { writeLock.lock(); + + // Update effective capacity in all parent queue. + Set configuredNodelabels = csContext.getConfiguration() + .getConfiguredNodeLabels(getQueuePath()); + for (String label : configuredNodelabels) { + calculateEffectiveResourcesAndCapacity(label, clusterResource); + } + // Update all children for (CSQueue childQueue : childQueues) { // Get ResourceLimits of child queue before assign containers @@ -848,6 +905,211 @@ public boolean hasChildQueues() { return true; } + private void calculateEffectiveResourcesAndCapacity(String label, + Resource clusterResource) { + + // For root queue, ensure that max/min resource is updated to latest + // cluster resource. + Resource resourceByLabel = labelManager.getResourceByLabel(label, + clusterResource); + if (getQueueName().equals("root")) { + queueResourceQuotas.setConfiguredMinResource(label, resourceByLabel); + queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel); + queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel); + queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel); + queueCapacities.setAbsoluteCapacity(label, 1.0f); + } + + // Total configured min resources of direct children of this given parent + // queue. + Resource configuredMinResources = Resource.newInstance(0L, 0); + for (CSQueue childQueue : getChildQueues()) { + Resources.addTo(configuredMinResources, + childQueue.getQueueResourceQuotas().getConfiguredMinResource(label)); + } + + // Factor to scale down effective resource: When cluster has sufficient + // resources, effective_min_resources will be same as configured + // min_resources. + Resource numeratorForMinRatio = null; + ResourceCalculator rc = this.csContext.getResourceCalculator(); + if (getQueueName().equals("root")) { + if (!resourceByLabel.equals(Resources.none()) && Resources.lessThan(rc, + clusterResource, resourceByLabel, configuredMinResources)) { + numeratorForMinRatio = resourceByLabel; + } + } else { + if (Resources.lessThan(rc, clusterResource, + queueResourceQuotas.getEffectiveMinResource(label), + configuredMinResources)) { + numeratorForMinRatio = queueResourceQuotas + .getEffectiveMinResource(label); + } + } + + Map effectiveMinRatioPerResource = getEffectiveMinRatioPerResource( + configuredMinResources, numeratorForMinRatio); + + // loop and do this for all child queues + for (CSQueue childQueue : getChildQueues()) { + Resource minResource = childQueue.getQueueResourceQuotas() + .getConfiguredMinResource(label); + + // Update effective resource (min/max) to each child queue. + if (childQueue.getCapacityConfigType() + .equals(CapacityConfigType.ABSOLUTE_RESOURCE)) { + childQueue.getQueueResourceQuotas().setEffectiveMinResource(label, + getMinResourceNormalized(childQueue.getQueueName(), effectiveMinRatioPerResource, + minResource)); + + // Max resource of a queue should be a minimum of {configuredMaxRes, + // parentMaxRes}. parentMaxRes could be configured value. But if not + // present could also be taken from effective max resource of parent. + Resource parentMaxRes = queueResourceQuotas + .getConfiguredMaxResource(label); + if (parent != null && parentMaxRes.equals(Resources.none())) { + parentMaxRes = parent.getQueueResourceQuotas() + .getEffectiveMaxResource(label); + } + + // Minimum of {childMaxResource, parentMaxRes}. However if + // childMaxResource is empty, consider parent's max resource alone. + Resource childMaxResource = childQueue.getQueueResourceQuotas() + .getConfiguredMaxResource(label); + Resource effMaxResource = Resources.min(resourceCalculator, + resourceByLabel, childMaxResource.equals(Resources.none()) + ? parentMaxRes + : childMaxResource, + parentMaxRes); + childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label, + Resources.clone(effMaxResource)); + + // In cases where we still need to update some units based on + // percentage, we have to calculate percentage and update. + deriveCapacityFromAbsoluteConfigurations(label, clusterResource, rc, + childQueue); + } else { + childQueue.getQueueResourceQuotas().setEffectiveMinResource(label, + Resources.multiply(resourceByLabel, + childQueue.getQueueCapacities().getAbsoluteCapacity(label))); + childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label, + Resources.multiply(resourceByLabel, childQueue.getQueueCapacities() + .getAbsoluteMaximumCapacity(label))); + } + + if (LOG.isDebugEnabled()) { + LOG.debug("Updating effective min resource for queue:" + + childQueue.getQueueName() + " as effMinResource=" + + childQueue.getQueueResourceQuotas().getEffectiveMinResource(label) + + "and Updating effective max resource as effMaxResource=" + + childQueue.getQueueResourceQuotas() + .getEffectiveMaxResource(label)); + } + } + } + + private Resource getMinResourceNormalized(String name, Map effectiveMinRatio, + Resource minResource) { + Resource ret = Resource.newInstance(minResource); + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { + ResourceInformation nResourceInformation = minResource + .getResourceInformation(i); + + Float ratio = effectiveMinRatio.get(nResourceInformation.getName()); + if (ratio != null) { + ret.setResourceValue(i, + (long) (nResourceInformation.getValue() * ratio.floatValue())); + if (LOG.isDebugEnabled()) { + LOG.debug("Updating min resource for Queue: " + name + " as " + + ret.getResourceInformation(i) + ", Actual resource: " + + nResourceInformation.getValue() + ", ratio: " + + ratio.floatValue()); + } + } + } + return ret; + } + + private Map getEffectiveMinRatioPerResource( + Resource configuredMinResources, Resource numeratorForMinRatio) { + Map effectiveMinRatioPerResource = new HashMap<>(); + if (numeratorForMinRatio != null) { + int maxLength = ResourceUtils.getNumberOfKnownResourceTypes(); + for (int i = 0; i < maxLength; i++) { + ResourceInformation nResourceInformation = numeratorForMinRatio + .getResourceInformation(i); + ResourceInformation dResourceInformation = configuredMinResources + .getResourceInformation(i); + + long nValue = nResourceInformation.getValue(); + long dValue = UnitsConversionUtil.convert( + dResourceInformation.getUnits(), nResourceInformation.getUnits(), + dResourceInformation.getValue()); + if (dValue != 0) { + effectiveMinRatioPerResource.put(nResourceInformation.getName(), + (float) nValue / dValue); + } + } + } + return effectiveMinRatioPerResource; + } + + private void deriveCapacityFromAbsoluteConfigurations(String label, + Resource clusterResource, ResourceCalculator rc, CSQueue childQueue) { + + /* + * In case when queues are configured with absolute resources, it is better + * to update capacity/max-capacity etc w.r.t absolute resource as well. In + * case of computation, these values wont be used any more. However for + * metrics and UI, its better these values are pre-computed here itself. + */ + + // 1. Update capacity as a float based on parent's minResource + childQueue.getQueueCapacities().setCapacity(label, + rc.divide(clusterResource, + childQueue.getQueueResourceQuotas().getEffectiveMinResource(label), + getQueueResourceQuotas().getEffectiveMinResource(label))); + + // 2. Update max-capacity as a float based on parent's maxResource + childQueue.getQueueCapacities().setMaximumCapacity(label, + rc.divide(clusterResource, + childQueue.getQueueResourceQuotas().getEffectiveMaxResource(label), + getQueueResourceQuotas().getEffectiveMaxResource(label))); + + // 3. Update absolute capacity as a float based on parent's minResource and + // cluster resource. + childQueue.getQueueCapacities().setAbsoluteCapacity(label, + (float) childQueue.getQueueCapacities().getCapacity() + / getQueueCapacities().getAbsoluteCapacity(label)); + + // 4. Update absolute max-capacity as a float based on parent's maxResource + // and cluster resource. + childQueue.getQueueCapacities().setAbsoluteMaximumCapacity(label, + (float) childQueue.getQueueCapacities().getMaximumCapacity(label) + / getQueueCapacities().getAbsoluteMaximumCapacity(label)); + + // Re-visit max applications for a queue based on absolute capacity if + // needed. + if (childQueue instanceof LeafQueue) { + LeafQueue leafQueue = (LeafQueue) childQueue; + CapacitySchedulerConfiguration conf = csContext.getConfiguration(); + int maxApplications = (int) (conf.getMaximumSystemApplications() + * childQueue.getQueueCapacities().getAbsoluteCapacity(label)); + leafQueue.setMaxApplications(maxApplications); + + int maxApplicationsPerUser = Math.min(maxApplications, + (int) (maxApplications + * (leafQueue.getUsersManager().getUserLimit() / 100.0f) + * leafQueue.getUsersManager().getUserLimitFactor())); + leafQueue.setMaxApplicationsPerUser(maxApplicationsPerUser); + LOG.info("LeafQueue:" + leafQueue.getQueueName() + ", maxApplications=" + + maxApplications + ", maxApplicationsPerUser=" + + maxApplicationsPerUser + ", Abs Cap:" + + childQueue.getQueueCapacities().getAbsoluteCapacity(label)); + } + } + @Override public List getChildQueues() { try { @@ -980,9 +1242,21 @@ void allocateResource(Resource clusterResource, * When this happens, we have to preempt killable container (on same or different * nodes) of parent queue to avoid violating parent's max resource. */ - if (getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition) - < getQueueCapacities().getAbsoluteUsedCapacity(nodePartition)) { - killContainersToEnforceMaxQueueCapacity(nodePartition, clusterResource); + if (!queueResourceQuotas.getEffectiveMaxResource(nodePartition) + .equals(Resources.none())) { + if (Resources.lessThan(resourceCalculator, clusterResource, + queueResourceQuotas.getEffectiveMaxResource(nodePartition), + queueUsage.getUsed(nodePartition))) { + killContainersToEnforceMaxQueueCapacity(nodePartition, + clusterResource); + } + } else { + if (getQueueCapacities() + .getAbsoluteMaximumCapacity(nodePartition) < getQueueCapacities() + .getAbsoluteUsedCapacity(nodePartition)) { + killContainersToEnforceMaxQueueCapacity(nodePartition, + clusterResource); + } } } finally { writeLock.unlock(); @@ -999,8 +1273,7 @@ private void killContainersToEnforceMaxQueueCapacity(String partition, Resource partitionResource = labelManager.getResourceByLabel(partition, null); - Resource maxResource = Resources.multiply(partitionResource, - getQueueCapacities().getAbsoluteMaximumCapacity(partition)); + Resource maxResource = getEffectiveMaxCapacity(partition); while (Resources.greaterThan(resourceCalculator, partitionResource, queueUsage.getUsed(partition), maxResource)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java index 33f30b00412..7287c5b7c3b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java @@ -686,10 +686,7 @@ private Resource computeUserLimit(String userName, Resource clusterResource, * * If we're running over capacity, then its (usedResources + required) * (which extra resources we are allocating) */ - Resource queueCapacity = Resources.multiplyAndNormalizeUp( - resourceCalculator, partitionResource, - lQueue.getQueueCapacities().getAbsoluteCapacity(nodePartition), - lQueue.getMinimumAllocation()); + Resource queueCapacity = lQueue.getEffectiveCapacity(nodePartition); /* * Assume we have required resource equals to minimumAllocation, this can diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java index 054438765e7..4985a1adc0b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/PriorityUtilizationQueueOrderingPolicy.java @@ -20,9 +20,11 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.StringUtils; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration; +import org.apache.hadoop.yarn.util.resource.Resources; import java.util.ArrayList; import java.util.Collections; @@ -121,6 +123,15 @@ public int compare(CSQueue q1, CSQueue q2) { // For queue with same used ratio / priority, queue with higher configured // capacity goes first if (0 == rc) { + Resource minEffRes1 = q1.getQueueResourceQuotas() + .getConfiguredMinResource(p); + Resource minEffRes2 = q2.getQueueResourceQuotas() + .getConfiguredMinResource(p); + if (!minEffRes1.equals(Resources.none()) + && !minEffRes2.equals(Resources.none())) { + return minEffRes2.compareTo(minEffRes1); + } + float abs1 = q1.getQueueCapacities().getAbsoluteCapacity(p); float abs2 = q2.getQueueCapacities().getAbsoluteCapacity(p); return Float.compare(abs2, abs1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 40405fc11ae..502157fffed 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest; @@ -903,6 +904,7 @@ protected void getActivedAppDiagnosticMessage( StringBuilder diagnosticMessage) { LeafQueue queue = getCSLeafQueue(); QueueCapacities queueCapacities = queue.getQueueCapacities(); + QueueResourceQuotas queueResourceQuotas = queue.getQueueResourceQuotas(); diagnosticMessage.append(" Details : AM Partition = "); diagnosticMessage.append(appAMNodePartitionName.isEmpty() ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName); @@ -924,6 +926,18 @@ protected void getActivedAppDiagnosticMessage( queueCapacities.getAbsoluteMaximumCapacity(appAMNodePartitionName) * 100); diagnosticMessage.append(" % ; "); + diagnosticMessage.append("Queue's capacity (absolute resource) = "); + diagnosticMessage.append( + queueResourceQuotas.getEffectiveMinResource(appAMNodePartitionName)); + diagnosticMessage.append(" ; "); + diagnosticMessage.append("Queue's used capacity (absolute resource) = "); + diagnosticMessage + .append(queue.getQueueResourceUsage().getUsed(appAMNodePartitionName)); + diagnosticMessage.append(" ; "); + diagnosticMessage.append("Queue's max capacity (absolute resource) = "); + diagnosticMessage.append( + queueResourceQuotas.getEffectiveMaxResource(appAMNodePartitionName)); + diagnosticMessage.append(" ; "); } /** @@ -987,13 +1001,10 @@ public ApplicationResourceUsageReport getResourceUsageReport() { ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator(); if (!calc.isInvalidDivisor(totalPartitionRes)) { - float queueAbsMaxCapPerPartition = - ((AbstractCSQueue) getQueue()).getQueueCapacities() - .getAbsoluteCapacity(getAppAMNodePartitionName()); + Resource effCap = ((AbstractCSQueue) getQueue()) + .getEffectiveCapacity(getAppAMNodePartitionName()); float queueUsagePerc = calc.divide(totalPartitionRes, - report.getUsedResources(), - Resources.multiply(totalPartitionRes, queueAbsMaxCapPerPartition)) - * 100; + report.getUsedResources(), effCap) * 100; report.setQueueUsagePercentage(queueUsagePerc); } return report; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 74b4e799d81..3f68e07ddb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -155,21 +155,36 @@ private void renderQueueCapacityInfo(ResponseInfo ri, String label) { ? new ResourceInfo(Resources.none()) : resourceUsages.getAmUsed(); ri. - __("Used Capacity:", percent(capacities.getUsedCapacity() / 100)). - __("Configured Capacity:", percent(capacities.getCapacity() / 100)). - __("Configured Max Capacity:", percent(capacities.getMaxCapacity() / 100)). - __("Absolute Used Capacity:", percent(capacities.getAbsoluteUsedCapacity() / 100)). - __("Absolute Configured Capacity:", percent(capacities.getAbsoluteCapacity() / 100)). - __("Absolute Configured Max Capacity:", percent(capacities.getAbsoluteMaxCapacity() / 100)). - __("Used Resources:", resourceUsages.getUsed().toString()). - __("Configured Max Application Master Limit:", StringUtils.format("%.1f", - capacities.getMaxAMLimitPercentage())). - __("Max Application Master Resources:", - resourceUsages.getAMLimit().toString()). - __("Used Application Master Resources:", - amUsed.toString()). - __("Max Application Master Resources Per User:", - userAMResourceLimit.toString()); + __("Used Capacity:", + appendPercent(resourceUsages.getUsed().toString(), + capacities.getUsedCapacity() / 100)) + .__("Configured Capacity:", + capacities.getConfiguredMinResource().toString()) + .__("Configured Max Capacity:", + capacities.getConfiguredMaxResource().getResource() + .equals(Resources.none()) + ? "unlimited" + : capacities.getConfiguredMaxResource().toString()) + .__("Effective Capacity:", + appendPercent(capacities.getEffectiveMinResource().toString(), + capacities.getCapacity() / 100)) + .__("Effective Max Capacity:", + appendPercent(capacities.getEffectiveMaxResource().toString(), + capacities.getMaxCapacity() / 100)) + .__("Absolute Used Capacity:", + percent(capacities.getAbsoluteUsedCapacity() / 100)) + .__("Absolute Configured Capacity:", + percent(capacities.getAbsoluteCapacity() / 100)) + .__("Absolute Configured Max Capacity:", + percent(capacities.getAbsoluteMaxCapacity() / 100)) + .__("Used Resources:", resourceUsages.getUsed().toString()) + .__("Configured Max Application Master Limit:", + StringUtils.format("%.1f", capacities.getMaxAMLimitPercentage())) + .__("Max Application Master Resources:", + resourceUsages.getAMLimit().toString()) + .__("Used Application Master Resources:", amUsed.toString()) + .__("Max Application Master Resources Per User:", + userAMResourceLimit.toString()); } private void renderCommonLeafQueueInfo(ResponseInfo ri) { @@ -615,6 +630,10 @@ public void render(HtmlBlock.Block html) { return QueuesBlock.class; } + static String appendPercent(String message, float f) { + return message + " (" + StringUtils.formatPercent(f, 1) + ")"; + } + static String percent(float f) { return StringUtils.formatPercent(f, 1); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 32e4ac5bb36..ca420419c89 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -59,7 +59,8 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { max = 1f; this.maxCapacity = max * 100; - capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), false); + capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), + parent.getQueueResourceQuotas(), false); queues = getQueues(parent); health = new CapacitySchedulerHealthInfo(cs); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index 7dcdf582225..343c78e8ed1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @@ -89,8 +90,9 @@ protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { } @Override - protected void populateQueueCapacities(QueueCapacities qCapacities) { - capacities = new QueueCapacitiesInfo(qCapacities); + protected void populateQueueCapacities(QueueCapacities qCapacities, + QueueResourceQuotas qResQuotas) { + capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas); } public int getNumActiveApplications() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 22705cc6c56..d4de9aeff9f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; @@ -62,6 +63,8 @@ protected long pendingContainers; protected QueueCapacitiesInfo capacities; protected ResourcesInfo resources; + protected ResourceInfo minEffectiveCapacity; + protected ResourceInfo maxEffectiveCapacity; CapacitySchedulerQueueInfo() { }; @@ -101,18 +104,26 @@ Collections.sort(nodeLabels); } QueueCapacities qCapacities = q.getQueueCapacities(); - populateQueueCapacities(qCapacities); + QueueResourceQuotas qResQuotas = q.getQueueResourceQuotas(); + populateQueueCapacities(qCapacities, qResQuotas); ResourceUsage queueResourceUsage = q.getQueueResourceUsage(); populateQueueResourceUsage(queueResourceUsage); + + minEffectiveCapacity = new ResourceInfo( + q.getQueueResourceQuotas().getEffectiveMinResource()); + maxEffectiveCapacity = new ResourceInfo( + q.getQueueResourceQuotas().getEffectiveMaxResource()); } protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { resources = new ResourcesInfo(queueResourceUsage, false); } - protected void populateQueueCapacities(QueueCapacities qCapacities) { - capacities = new QueueCapacitiesInfo(qCapacities, false); + protected void populateQueueCapacities(QueueCapacities qCapacities, + QueueResourceQuotas qResQuotas) { + capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas, + false); } public float getCapacity() { @@ -200,4 +211,12 @@ public QueueCapacitiesInfo getCapacities() { public ResourcesInfo getResources() { return resources; } + + public ResourceInfo getMinEffectiveCapacity(){ + return minEffectiveCapacity; + } + + public ResourceInfo getMaxEffectiveCapacity(){ + return maxEffectiveCapacity; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java index 5e298f9dc81..2a155028aa8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java @@ -21,6 +21,9 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; + /** * This class represents queue capacities for a given partition */ @@ -36,13 +39,19 @@ private float absoluteUsedCapacity; private float absoluteMaxCapacity = 100; private float maxAMLimitPercentage; + private ResourceInfo configuredMinResource; + private ResourceInfo configuredMaxResource; + private ResourceInfo effectiveMinResource; + private ResourceInfo effectiveMaxResource; public PartitionQueueCapacitiesInfo() { } public PartitionQueueCapacitiesInfo(String partitionName, float capacity, float usedCapacity, float maxCapacity, float absCapacity, - float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage) { + float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage, + Resource confMinRes, Resource confMaxRes, Resource effMinRes, + Resource effMaxRes) { super(); this.partitionName = partitionName; this.capacity = capacity; @@ -52,6 +61,10 @@ public PartitionQueueCapacitiesInfo(String partitionName, float capacity, this.absoluteUsedCapacity = absUsedCapacity; this.absoluteMaxCapacity = absMaxCapacity; this.maxAMLimitPercentage = maxAMLimitPercentage; + this.configuredMinResource = new ResourceInfo(confMinRes); + this.configuredMaxResource = new ResourceInfo(confMaxRes); + this.effectiveMinResource = new ResourceInfo(effMinRes); + this.effectiveMaxResource = new ResourceInfo(effMaxRes); } public float getCapacity() { @@ -117,4 +130,23 @@ public float getMaxAMLimitPercentage() { public void setMaxAMLimitPercentage(float maxAMLimitPercentage) { this.maxAMLimitPercentage = maxAMLimitPercentage; } + + public ResourceInfo getConfiguredMinResource() { + return configuredMinResource; + } + + public ResourceInfo getConfiguredMaxResource() { + if (configuredMaxResource.getResource().equals(Resources.none())) { + return null; + } + return configuredMaxResource; + } + + public ResourceInfo getEffectiveMinResource() { + return effectiveMinResource; + } + + public ResourceInfo getEffectiveMaxResource() { + return effectiveMaxResource; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java index 9a3e4392eb4..35c80d2ea4a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java @@ -24,6 +24,7 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; /** @@ -39,6 +40,7 @@ public QueueCapacitiesInfo() { } public QueueCapacitiesInfo(QueueCapacities capacities, + QueueResourceQuotas resourceQuotas, boolean considerAMUsage) { if (capacities == null) { return; @@ -68,12 +70,17 @@ public QueueCapacitiesInfo(QueueCapacities capacities, queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo( partitionName, capacity, usedCapacity, maxCapacity, absCapacity, absUsedCapacity, absMaxCapacity, - considerAMUsage ? maxAMLimitPercentage : 0f)); + considerAMUsage ? maxAMLimitPercentage : 0f, + resourceQuotas.getConfiguredMinResource(partitionName), + resourceQuotas.getConfiguredMaxResource(partitionName), + resourceQuotas.getEffectiveMinResource(partitionName), + resourceQuotas.getEffectiveMaxResource(partitionName))); } } - public QueueCapacitiesInfo(QueueCapacities capacities) { - this(capacities, true); + public QueueCapacitiesInfo(QueueCapacities capacities, + QueueResourceQuotas resourceQuotas) { + this(capacities, resourceQuotas, true); } public void add(PartitionQueueCapacitiesInfo partitionQueueCapacitiesInfo) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java index b5c20f4d84b..909d02319bb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNM.java @@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse; import org.apache.hadoop.yarn.server.api.records.AppCollectorData; +import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest; import org.apache.hadoop.yarn.server.api.records.MasterKey; import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus; import org.apache.hadoop.yarn.server.api.records.NodeStatus; @@ -142,6 +143,13 @@ public void addRegisteringCollector(ApplicationId appId, return this.registeringCollectors; } + public void unRegisterNode() throws Exception { + UnRegisterNodeManagerRequest request = Records + .newRecord(UnRegisterNodeManagerRequest.class); + request.setNodeId(nodeId); + resourceTracker.unRegisterNodeManager(request); + } + public RegisterNodeManagerResponse registerNode() throws Exception { return registerNode(null, null); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java index d128b0205fb..d982d2f3349 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java @@ -822,6 +822,12 @@ public SubmitApplicationResponse run() throws IOException, YarnException { return rmApp; } + public MockNM unRegisterNode(MockNM nm) throws Exception { + nm.unRegisterNode(); + drainEventsImplicitly(); + return nm; + } + public MockNM registerNode(String nodeIdStr, int memory) throws Exception { MockNM nm = new MockNM(nodeIdStr, memory, getResourceTrackerService()); nm.registerNode(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java index 0bc5cb5d4af..ca43a955857 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/ProportionalCapacityPreemptionPolicyMockFramework.java @@ -28,12 +28,14 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Priority; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceInformation; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy; @@ -55,6 +57,7 @@ import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator; import org.apache.hadoop.yarn.util.resource.ResourceCalculator; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.Assert; import org.junit.Before; @@ -532,6 +535,18 @@ private Resource parseResourceFromString(String p) { } else { res = Resources.createResource(Integer.valueOf(resource[0]), Integer.valueOf(resource[1])); + if (resource.length > 2) { + // Using the same order of resources from ResourceUtils, set resource + // informations. + ResourceInformation[] storedResourceInfo = ResourceUtils + .getResourceTypesArray(); + for (int i = 2; i < resource.length; i++) { + res.setResourceInformation(storedResourceInfo[i].getName(), + ResourceInformation.newInstance(storedResourceInfo[i].getName(), + storedResourceInfo[i].getUnits(), + Integer.valueOf(resource[i]))); + } + } } return res; } @@ -644,9 +659,11 @@ private void setupQueue(CSQueue queue, String q, String[] queueExprArray, QueueCapacities qc = new QueueCapacities(0 == myLevel); ResourceUsage ru = new ResourceUsage(); + QueueResourceQuotas qr = new QueueResourceQuotas(); when(queue.getQueueCapacities()).thenReturn(qc); when(queue.getQueueResourceUsage()).thenReturn(ru); + when(queue.getQueueResourceQuotas()).thenReturn(qr); LOG.debug("Setup queue, name=" + queue.getQueueName() + " path=" + queue.getQueuePath()); @@ -679,7 +696,17 @@ private void setupQueue(CSQueue queue, String q, String[] queueExprArray, qc.setAbsoluteMaximumCapacity(partitionName, absMax); qc.setAbsoluteUsedCapacity(partitionName, absUsed); qc.setUsedCapacity(partitionName, used); + qr.setEffectiveMaxResource(parseResourceFromString(values[1].trim())); + qr.setEffectiveMinResource(parseResourceFromString(values[0].trim())); + qr.setEffectiveMaxResource(partitionName, + parseResourceFromString(values[1].trim())); + qr.setEffectiveMinResource(partitionName, + parseResourceFromString(values[0].trim())); when(queue.getUsedCapacity()).thenReturn(used); + when(queue.getEffectiveCapacity(partitionName)) + .thenReturn(parseResourceFromString(values[0].trim())); + when(queue.getEffectiveMaxCapacity(partitionName)) + .thenReturn(parseResourceFromString(values[1].trim())); ru.setPending(partitionName, pending); // Setup reserved resource if it contained by input config Resource reserved = Resources.none(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java index 694be09ee87..dd950f3bf1f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicy.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.monitor.SchedulingMonitor; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.policy.QueueOrderingPolicy; import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey; @@ -48,7 +49,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerPreemptEvent; -import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.OrderingPolicy; import org.apache.hadoop.yarn.util.Clock; @@ -437,8 +437,8 @@ public void testPerQueueDisablePreemptionBroadHierarchical() { policy.editSchedule(); // queueF(appD) wants resources, Verify that resources come from queueE(appC) // because it's a sibling and queueB(appA) because queueA is over capacity. - verify(mDisp, times(28)).handle(argThat(new IsPreemptionRequestFor(appA))); - verify(mDisp, times(22)).handle(argThat(new IsPreemptionRequestFor(appC))); + verify(mDisp, times(27)).handle(argThat(new IsPreemptionRequestFor(appA))); + verify(mDisp, times(23)).handle(argThat(new IsPreemptionRequestFor(appC))); // Need to call setup() again to reset mDisp setup(); @@ -1202,6 +1202,17 @@ ParentQueue mockNested(Resource[] abs, int[] maxCap, Resource[] used, when(root.getQueuePath()).thenReturn(CapacitySchedulerConfiguration.ROOT); boolean preemptionDisabled = mockPreemptionStatus("root"); when(root.getPreemptionDisabled()).thenReturn(preemptionDisabled); + QueueResourceQuotas rootQr = new QueueResourceQuotas(); + rootQr.setEffectiveMaxResource(Resource.newInstance(maxCap[0], 0)); + rootQr.setEffectiveMinResource(abs[0]); + rootQr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL, + Resource.newInstance(maxCap[0], 0)); + rootQr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[0]); + when(root.getQueueResourceQuotas()).thenReturn(rootQr); + when(root.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL)) + .thenReturn(abs[0]); + when(root.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL)) + .thenReturn(Resource.newInstance(maxCap[0], 0)); for (int i = 1; i < queues.length; ++i) { final CSQueue q; @@ -1232,6 +1243,18 @@ ParentQueue mockNested(Resource[] abs, int[] maxCap, Resource[] used, qc.setAbsoluteMaximumCapacity(maxCap[i] / (float) tot.getMemorySize()); when(q.getQueueCapacities()).thenReturn(qc); + QueueResourceQuotas qr = new QueueResourceQuotas(); + qr.setEffectiveMaxResource(Resource.newInstance(maxCap[i], 0)); + qr.setEffectiveMinResource(abs[i]); + qr.setEffectiveMaxResource(RMNodeLabelsManager.NO_LABEL, + Resource.newInstance(maxCap[i], 0)); + qr.setEffectiveMinResource(RMNodeLabelsManager.NO_LABEL, abs[i]); + when(q.getQueueResourceQuotas()).thenReturn(qr); + when(q.getEffectiveCapacity(RMNodeLabelsManager.NO_LABEL)) + .thenReturn(abs[i]); + when(q.getEffectiveMaxCapacity(RMNodeLabelsManager.NO_LABEL)) + .thenReturn(Resource.newInstance(maxCap[i], 0)); + String parentPathName = p.getQueuePath(); parentPathName = (parentPathName == null) ? "root" : parentPathName; String queuePathName = (parentPathName + "." + queueName).replace("/", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java index 1fd455a607a..67c09cd7e5d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyForNodePartitions.java @@ -18,11 +18,17 @@ package org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity; +import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes; +import org.apache.hadoop.yarn.api.records.ResourceInformation; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity.TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor; +import org.apache.hadoop.yarn.util.resource.ResourceUtils; import org.junit.Before; import org.junit.Test; import java.io.IOException; +import java.util.HashMap; +import java.util.Map; import static org.mockito.Matchers.argThat; import static org.mockito.Mockito.never; @@ -613,4 +619,74 @@ public void testNodePartitionPreemptionWithVCoreResource() throws IOException { verify(mDisp, never()).handle( argThat(new IsPreemptionRequestFor(getAppAttemptId(3)))); } + + @Test + public void testNormalizeGuaranteeWithMultipleResource() throws IOException { + // Initialize resource map + Map riMap = new HashMap<>(); + String RESOURCE_1 = "res1"; + + // Initialize mandatory resources + ResourceInformation memory = ResourceInformation.newInstance( + ResourceInformation.MEMORY_MB.getName(), + ResourceInformation.MEMORY_MB.getUnits(), + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + ResourceInformation vcores = ResourceInformation.newInstance( + ResourceInformation.VCORES.getName(), + ResourceInformation.VCORES.getUnits(), + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + riMap.put(ResourceInformation.MEMORY_URI, memory); + riMap.put(ResourceInformation.VCORES_URI, vcores); + riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0, + ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE)); + + ResourceUtils.initializeResourcesFromResourceInformationMap(riMap); + + /** + * Queue structure is: + * + *
+     *           root
+     *           /  \
+     *          a    b
+     *        /  \  /  \
+     *       a1  a2 b1  b2
+     * 
+ * + * a1 and b2 are using most of resources. + * a2 and b1 needs more resources. Both are under served. + * hence demand will consider both queue's need while trying to + * do preemption. + */ + String labelsConfig = + "=100,true;"; + String nodesConfig = + "n1=;"; // n1 is default partition + String queuesConfig = + // guaranteed,max,used,pending + "root(=[100:100:10 100:100:10 100:100:10 100:100:10]);" + //root + "-a(=[50:80:4 100:100:10 80:90:10 30:20:4]);" + // a + "--a1(=[25:30:2 100:50:10 80:90:10 0]);" + // a1 + "--a2(=[25:50:2 100:50:10 0 30:20:4]);" + // a2 + "-b(=[50:20:6 100:100:10 20:10 40:50:8]);" + // b + "--b1(=[25:5:4 100:20:10 0 20:10:4]);" + // b1 + "--b2(=[25:15:2 100:20:10 20:10 20:10:4])"; // b2 + String appsConfig= + //queueName\t(priority,resource,host,expression,#repeat,reserved) + "a1\t" // app1 in a1 + + "(1,8:9:1,n1,,10,false);" + + "b2\t" // app2 in b2 + + "(1,2:1,n1,,10,false)"; // 80 of y + + buildEnv(labelsConfig, nodesConfig, queuesConfig, appsConfig); + policy.editSchedule(); + + verify(mDisp, times(7)).handle( + argThat(new IsPreemptionRequestFor(getAppAttemptId(1)))); + + riMap.remove(RESOURCE_1); + ResourceUtils.initializeResourcesFromResourceInformationMap(riMap); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java index 7784549a34a..a1d89d74dab 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/monitor/capacity/TestProportionalCapacityPreemptionPolicyIntraQueueWithDRF.java @@ -67,9 +67,9 @@ public void testSimpleIntraQueuePreemptionWithVCoreResource() conf.set(CapacitySchedulerConfiguration.INTRAQUEUE_PREEMPTION_ORDER_POLICY, "priority_first"); - String labelsConfig = "=100:200,true;"; + String labelsConfig = "=100:50,true;"; String nodesConfig = // n1 has no label - "n1= res=100:200"; + "n1= res=100:50"; String queuesConfig = // guaranteed,max,used,pending,reserved "root(=[100:50 100:50 80:40 120:60 0]);" + // root @@ -105,7 +105,7 @@ public void testSimpleIntraQueuePreemptionWithVCoreResource() verify(mDisp, times(1)).handle(argThat( new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( getAppAttemptId(4)))); - verify(mDisp, times(7)).handle(argThat( + verify(mDisp, times(3)).handle(argThat( new TestProportionalCapacityPreemptionPolicy.IsPreemptionRequestFor( getAppAttemptId(3)))); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java new file mode 100644 index 00000000000..5a66281b8d5 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestAbsoluteResourceConfiguration.java @@ -0,0 +1,516 @@ +/** +* Licensed to the Apache Software Foundation (ASF) under one +* or more contributor license agreements. See the NOTICE file +* distributed with this work for additional information +* regarding copyright ownership. The ASF licenses this file +* to you under the Apache License, Version 2.0 (the +* "License"); you may not use this file except in compliance +* with the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.resourcemanager.MockNM; +import org.apache.hadoop.yarn.server.resourcemanager.MockRM; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.util.resource.Resources; +import org.junit.Assert; +import org.junit.Test; + +public class TestAbsoluteResourceConfiguration { + + private static final int GB = 1024; + + private static final String QUEUEA = "queueA"; + private static final String QUEUEB = "queueB"; + private static final String QUEUEC = "queueC"; + private static final String QUEUEA1 = "queueA1"; + private static final String QUEUEA2 = "queueA2"; + private static final String QUEUEB1 = "queueB1"; + + private static final String QUEUEA_FULL = CapacitySchedulerConfiguration.ROOT + + "." + QUEUEA; + private static final String QUEUEB_FULL = CapacitySchedulerConfiguration.ROOT + + "." + QUEUEB; + private static final String QUEUEC_FULL = CapacitySchedulerConfiguration.ROOT + + "." + QUEUEC; + private static final String QUEUEA1_FULL = QUEUEA_FULL + "." + QUEUEA1; + private static final String QUEUEA2_FULL = QUEUEA_FULL + "." + QUEUEA2; + private static final String QUEUEB1_FULL = QUEUEB_FULL + "." + QUEUEB1; + + private static final Resource QUEUE_A_MINRES = Resource.newInstance(100 * GB, + 10); + private static final Resource QUEUE_A_MAXRES = Resource.newInstance(200 * GB, + 30); + private static final Resource QUEUE_A1_MINRES = Resource.newInstance(50 * GB, + 5); + private static final Resource QUEUE_A2_MINRES = Resource.newInstance(50 * GB, + 5); + private static final Resource QUEUE_B_MINRES = Resource.newInstance(50 * GB, + 10); + private static final Resource QUEUE_B1_MINRES = Resource.newInstance(40 * GB, + 10); + private static final Resource QUEUE_B_MAXRES = Resource.newInstance(150 * GB, + 30); + private static final Resource QUEUE_C_MINRES = Resource.newInstance(50 * GB, + 10); + private static final Resource QUEUE_C_MAXRES = Resource.newInstance(150 * GB, + 20); + private static final Resource QUEUEA_REDUCED = Resource.newInstance(64000, 6); + private static final Resource QUEUEB_REDUCED = Resource.newInstance(32000, 6); + private static final Resource QUEUEC_REDUCED = Resource.newInstance(32000, 6); + private static final Resource QUEUEMAX_REDUCED = Resource.newInstance(128000, + 20); + + private static Set resourceTypes = new HashSet<>( + Arrays.asList("memory", "vcores")); + + private CapacitySchedulerConfiguration setupSimpleQueueConfiguration( + boolean isCapacityNeeded) { + CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[]{QUEUEA, QUEUEB, QUEUEC}); + + // Set default capacities like normal configuration. + if (isCapacityNeeded) { + csConf.setCapacity(QUEUEA_FULL, 50f); + csConf.setCapacity(QUEUEB_FULL, 25f); + csConf.setCapacity(QUEUEC_FULL, 25f); + } + + return csConf; + } + + private CapacitySchedulerConfiguration setupComplexQueueConfiguration( + boolean isCapacityNeeded) { + CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); + csConf.setQueues(CapacitySchedulerConfiguration.ROOT, + new String[]{QUEUEA, QUEUEB, QUEUEC}); + csConf.setQueues(QUEUEA_FULL, new String[]{QUEUEA1, QUEUEA2}); + csConf.setQueues(QUEUEB_FULL, new String[]{QUEUEB1}); + + // Set default capacities like normal configuration. + if (isCapacityNeeded) { + csConf.setCapacity(QUEUEA_FULL, 50f); + csConf.setCapacity(QUEUEB_FULL, 25f); + csConf.setCapacity(QUEUEC_FULL, 25f); + csConf.setCapacity(QUEUEA1_FULL, 50f); + csConf.setCapacity(QUEUEA2_FULL, 50f); + csConf.setCapacity(QUEUEB1_FULL, 100f); + } + + return csConf; + } + + private CapacitySchedulerConfiguration setupMinMaxResourceConfiguration( + CapacitySchedulerConfiguration csConf) { + // Update min/max resource to queueA/B/C + csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES); + + csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES); + csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES); + csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES); + + return csConf; + } + + private CapacitySchedulerConfiguration setupComplexMinMaxResourceConfig( + CapacitySchedulerConfiguration csConf) { + // Update min/max resource to queueA/B/C + csConf.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEA1_FULL, QUEUE_A1_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEA2_FULL, QUEUE_A2_MINRES); + csConf.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES); + + csConf.setMaximumResourceRequirement("", QUEUEA_FULL, QUEUE_A_MAXRES); + csConf.setMaximumResourceRequirement("", QUEUEB_FULL, QUEUE_B_MAXRES); + csConf.setMaximumResourceRequirement("", QUEUEC_FULL, QUEUE_C_MAXRES); + + return csConf; + } + + @Test + public void testSimpleMinMaxResourceConfigurartionPerQueue() { + + CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration(true); + setupMinMaxResourceConfiguration(csConf); + + Assert.assertEquals("Min resource configured for QUEUEA is not correct", + QUEUE_A_MINRES, + csConf.getMinimumResourceRequirement("", QUEUEA_FULL, resourceTypes)); + Assert.assertEquals("Max resource configured for QUEUEA is not correct", + QUEUE_A_MAXRES, + csConf.getMaximumResourceRequirement("", QUEUEA_FULL, resourceTypes)); + Assert.assertEquals("Min resource configured for QUEUEB is not correct", + QUEUE_B_MINRES, + csConf.getMinimumResourceRequirement("", QUEUEB_FULL, resourceTypes)); + Assert.assertEquals("Max resource configured for QUEUEB is not correct", + QUEUE_B_MAXRES, + csConf.getMaximumResourceRequirement("", QUEUEB_FULL, resourceTypes)); + Assert.assertEquals("Min resource configured for QUEUEC is not correct", + QUEUE_C_MINRES, + csConf.getMinimumResourceRequirement("", QUEUEC_FULL, resourceTypes)); + Assert.assertEquals("Max resource configured for QUEUEC is not correct", + QUEUE_C_MAXRES, + csConf.getMaximumResourceRequirement("", QUEUEC_FULL, resourceTypes)); + } + + @Test + public void testEffectiveMinMaxResourceConfigurartionPerQueue() + throws Exception { + // create conf with basic queue configuration. + CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration( + false); + setupMinMaxResourceConfiguration(csConf); + + csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + @SuppressWarnings("resource") + MockRM rm = new MockRM(csConf); + rm.start(); + + // Add few nodes + rm.registerNode("127.0.0.1:1234", 250 * GB, 40); + + // Get queue object to verify min/max resource configuration. + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA); + Assert.assertNotNull(qA); + Assert.assertEquals("Min resource configured for QUEUEA is not correct", + QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource()); + Assert.assertEquals("Max resource configured for QUEUEA is not correct", + QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource()); + Assert.assertEquals("Effective Min resource for QUEUEA is not correct", + QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEA is not correct", + QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource()); + + LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB); + Assert.assertNotNull(qB); + Assert.assertEquals("Min resource configured for QUEUEB is not correct", + QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource()); + Assert.assertEquals("Max resource configured for QUEUEB is not correct", + QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource()); + Assert.assertEquals("Effective Min resource for QUEUEB is not correct", + QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEB is not correct", + QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource()); + + LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); + Assert.assertNotNull(qC); + Assert.assertEquals("Min resource configured for QUEUEC is not correct", + QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource()); + Assert.assertEquals("Max resource configured for QUEUEC is not correct", + QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource()); + Assert.assertEquals("Effective Min resource for QUEUEC is not correct", + QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEC is not correct", + QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); + + rm.stop(); + } + + @Test + public void testSimpleValidateAbsoluteResourceConfig() throws Exception { + /** + * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1 + * + * Test below cases 1) Configure percentage based capacity and absolute + * resource together. 2) As per above tree structure, ensure all values + * could be retrieved. 3) Validate whether min resource cannot be more than + * max resources. 4) Validate whether max resource of queue cannot be more + * than its parent max resource. + */ + // create conf with basic queue configuration. + CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration( + false); + setupMinMaxResourceConfiguration(csConf); + csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + @SuppressWarnings("resource") + MockRM rm = new MockRM(csConf); + rm.start(); + + // Add few nodes + rm.registerNode("127.0.0.1:1234", 250 * GB, 40); + + // Get queue object to verify min/max resource configuration. + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + // 1. Create a new config with capcity and min/max together. Ensure an + // exception is thrown. + CapacitySchedulerConfiguration csConf1 = setupSimpleQueueConfiguration( + true); + setupMinMaxResourceConfiguration(csConf1); + + try { + cs.reinitialize(csConf1, rm.getRMContext()); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e instanceof IOException); + Assert.assertEquals( + "Failed to re-init queues : Queue 'queueA' should use either" + + " percentage based capacity configuration or absolute resource.", + e.getMessage()); + } + rm.stop(); + + // 2. Create a new config with min/max alone with a complex queue config. + // Check all values could be fetched correctly. + CapacitySchedulerConfiguration csConf2 = setupComplexQueueConfiguration( + false); + setupComplexMinMaxResourceConfig(csConf2); + + rm = new MockRM(csConf2); + rm.start(); + rm.registerNode("127.0.0.1:1234", 250 * GB, 40); + cs = (CapacityScheduler) rm.getResourceScheduler(); + + LeafQueue qA1 = (LeafQueue) cs.getQueue(QUEUEA1); + Assert.assertEquals("Effective Min resource for QUEUEA1 is not correct", + QUEUE_A1_MINRES, qA1.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEA1 is not correct", + QUEUE_A_MAXRES, qA1.queueResourceQuotas.getEffectiveMaxResource()); + + LeafQueue qA2 = (LeafQueue) cs.getQueue(QUEUEA2); + Assert.assertEquals("Effective Min resource for QUEUEA2 is not correct", + QUEUE_A2_MINRES, qA2.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEA2 is not correct", + QUEUE_A_MAXRES, qA2.queueResourceQuotas.getEffectiveMaxResource()); + + LeafQueue qB1 = (LeafQueue) cs.getQueue(QUEUEB1); + Assert.assertNotNull(qB1); + Assert.assertEquals("Min resource configured for QUEUEB1 is not correct", + QUEUE_B1_MINRES, qB1.queueResourceQuotas.getConfiguredMinResource()); + Assert.assertEquals("Max resource configured for QUEUEB1 is not correct", + QUEUE_B_MAXRES, qB1.queueResourceQuotas.getConfiguredMaxResource()); + Assert.assertEquals("Effective Min resource for QUEUEB1 is not correct", + QUEUE_B1_MINRES, qB1.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEB1 is not correct", + QUEUE_B_MAXRES, qB1.queueResourceQuotas.getEffectiveMaxResource()); + + LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); + Assert.assertNotNull(qC); + Assert.assertEquals("Min resource configured for QUEUEC is not correct", + QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource()); + Assert.assertEquals("Max resource configured for QUEUEC is not correct", + QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource()); + Assert.assertEquals("Effective Min resource for QUEUEC is not correct", + QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEC is not correct", + QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); + + // 3. Create a new config and make sure one queue's min resource is more + // than its max resource configured. + CapacitySchedulerConfiguration csConf3 = setupComplexQueueConfiguration( + false); + setupComplexMinMaxResourceConfig(csConf3); + + csConf3.setMinimumResourceRequirement("", QUEUEB1_FULL, QUEUE_B_MAXRES); + csConf3.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_B1_MINRES); + + try { + cs.reinitialize(csConf3, rm.getRMContext()); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e instanceof IOException); + Assert.assertEquals( + "Failed to re-init queues : Min resource configuration " + + " is greater than its " + + "max value: in queue:queueB1", + e.getMessage()); + } + + // 4. Create a new config and make sure one queue's max resource is more + // than its preant's max resource configured. + CapacitySchedulerConfiguration csConf4 = setupComplexQueueConfiguration( + false); + setupComplexMinMaxResourceConfig(csConf4); + + csConf4.setMaximumResourceRequirement("", QUEUEB1_FULL, QUEUE_A_MAXRES); + + try { + cs.reinitialize(csConf4, rm.getRMContext()); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e instanceof IOException); + Assert + .assertEquals( + "Failed to re-init queues : Max resource configuration " + + " is greater than parents max value:" + + " in queue:queueB1", + e.getMessage()); + } + rm.stop(); + } + + @Test + public void testComplexValidateAbsoluteResourceConfig() throws Exception { + /** + * Queue structure is as follows. root / | \ a b c / \ | a1 a2 b1 + * + * Test below cases: 1) Parent and its child queues must use either + * percentage based or absolute resource configuration. 2) Parent's min + * resource must be more than sum of child's min resource. + */ + + // create conf with basic queue configuration. + CapacitySchedulerConfiguration csConf = setupComplexQueueConfiguration( + false); + setupComplexMinMaxResourceConfig(csConf); + csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + @SuppressWarnings("resource") + MockRM rm = new MockRM(csConf); + rm.start(); + + // Add few nodes + rm.registerNode("127.0.0.1:1234", 250 * GB, 40); + + // 1. Explicitly set percentage based config for parent queues. This will + // make Queue A,B and C with percentage based and A1,A2 or B1 with absolute + // resource. + csConf.setCapacity(QUEUEA_FULL, 50f); + csConf.setCapacity(QUEUEB_FULL, 25f); + csConf.setCapacity(QUEUEC_FULL, 25f); + + // Also unset resource based config. + csConf.setMinimumResourceRequirement("", QUEUEA_FULL, Resources.none()); + csConf.setMinimumResourceRequirement("", QUEUEB_FULL, Resources.none()); + csConf.setMinimumResourceRequirement("", QUEUEC_FULL, Resources.none()); + + // Get queue object to verify min/max resource configuration. + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + try { + cs.reinitialize(csConf, rm.getRMContext()); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e instanceof IOException); + Assert.assertEquals( + "Failed to re-init queues : Parent queue 'queueA' " + + "and child queue 'queueA1' should use either percentage based" + + " capacity configuration or absolute resource together.", + e.getMessage()); + } + + // 2. Create a new config and make sure one queue's min resource is more + // than its max resource configured. + CapacitySchedulerConfiguration csConf1 = setupComplexQueueConfiguration( + false); + setupComplexMinMaxResourceConfig(csConf1); + + // Configure QueueA with lesser resource than its children. + csConf1.setMinimumResourceRequirement("", QUEUEA_FULL, QUEUE_A1_MINRES); + + try { + cs.reinitialize(csConf1, rm.getRMContext()); + Assert.fail(); + } catch (IOException e) { + Assert.assertTrue(e instanceof IOException); + Assert.assertEquals("Failed to re-init queues : Parent Queues capacity: " + + " is less than to its children:" + + " for queue:queueA", e.getMessage()); + } + } + + @Test + public void testEffectiveResourceAfterReducingClusterResource() + throws Exception { + // create conf with basic queue configuration. + CapacitySchedulerConfiguration csConf = setupSimpleQueueConfiguration( + false); + setupMinMaxResourceConfiguration(csConf); + + csConf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, + ResourceScheduler.class); + + @SuppressWarnings("resource") + MockRM rm = new MockRM(csConf); + rm.start(); + + // Add few nodes + MockNM nm1 = rm.registerNode("127.0.0.1:1234", 125 * GB, 20); + rm.registerNode("127.0.0.2:1234", 125 * GB, 20); + + // Get queue object to verify min/max resource configuration. + CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler(); + + LeafQueue qA = (LeafQueue) cs.getQueue(QUEUEA); + Assert.assertNotNull(qA); + Assert.assertEquals("Min resource configured for QUEUEA is not correct", + QUEUE_A_MINRES, qA.queueResourceQuotas.getConfiguredMinResource()); + Assert.assertEquals("Max resource configured for QUEUEA is not correct", + QUEUE_A_MAXRES, qA.queueResourceQuotas.getConfiguredMaxResource()); + Assert.assertEquals("Effective Min resource for QUEUEA is not correct", + QUEUE_A_MINRES, qA.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEA is not correct", + QUEUE_A_MAXRES, qA.queueResourceQuotas.getEffectiveMaxResource()); + + LeafQueue qB = (LeafQueue) cs.getQueue(QUEUEB); + Assert.assertNotNull(qB); + Assert.assertEquals("Min resource configured for QUEUEB is not correct", + QUEUE_B_MINRES, qB.queueResourceQuotas.getConfiguredMinResource()); + Assert.assertEquals("Max resource configured for QUEUEB is not correct", + QUEUE_B_MAXRES, qB.queueResourceQuotas.getConfiguredMaxResource()); + Assert.assertEquals("Effective Min resource for QUEUEB is not correct", + QUEUE_B_MINRES, qB.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEB is not correct", + QUEUE_B_MAXRES, qB.queueResourceQuotas.getEffectiveMaxResource()); + + LeafQueue qC = (LeafQueue) cs.getQueue(QUEUEC); + Assert.assertNotNull(qC); + Assert.assertEquals("Min resource configured for QUEUEC is not correct", + QUEUE_C_MINRES, qC.queueResourceQuotas.getConfiguredMinResource()); + Assert.assertEquals("Max resource configured for QUEUEC is not correct", + QUEUE_C_MAXRES, qC.queueResourceQuotas.getConfiguredMaxResource()); + Assert.assertEquals("Effective Min resource for QUEUEC is not correct", + QUEUE_C_MINRES, qC.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEC is not correct", + QUEUE_C_MAXRES, qC.queueResourceQuotas.getEffectiveMaxResource()); + + // unregister one NM. + rm.unRegisterNode(nm1); + + // After loosing one NM, effective min res of queueA will become just + // above half. Hence A's min will be 60Gi and 6 cores and max will be + // 128GB and 20 cores. + Assert.assertEquals("Effective Min resource for QUEUEA is not correct", + QUEUEA_REDUCED, qA.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEA is not correct", + QUEUEMAX_REDUCED, qA.queueResourceQuotas.getEffectiveMaxResource()); + + Assert.assertEquals("Effective Min resource for QUEUEB is not correct", + QUEUEB_REDUCED, qB.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEB is not correct", + QUEUEMAX_REDUCED, qB.queueResourceQuotas.getEffectiveMaxResource()); + + Assert.assertEquals("Effective Min resource for QUEUEC is not correct", + QUEUEC_REDUCED, qC.queueResourceQuotas.getEffectiveMinResource()); + Assert.assertEquals("Effective Max resource for QUEUEC is not correct", + QUEUEMAX_REDUCED, qC.queueResourceQuotas.getEffectiveMaxResource()); + + rm.stop(); + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 8aca235f8aa..24ae244a969 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState; import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; @@ -86,6 +87,7 @@ final static int GB = 1024; LeafQueue queue; + CSQueue root; private final ResourceCalculator resourceCalculator = new DefaultResourceCalculator(); @@ -100,7 +102,7 @@ public void setUp() throws IOException { setupQueueConfiguration(csConf); rmContext = TestUtils.getMockRMContext(); - + Resource clusterResource = Resources.createResource(10 * 16 * GB, 10 * 32); CapacitySchedulerContext csContext = mock(CapacitySchedulerContext.class); when(csContext.getConfiguration()).thenReturn(csConf); @@ -110,10 +112,11 @@ public void setUp() throws IOException { when(csContext.getMaximumResourceCapability()). thenReturn(Resources.createResource(16*GB, 32)); when(csContext.getClusterResource()). - thenReturn(Resources.createResource(10 * 16 * GB, 10 * 32)); + thenReturn(clusterResource); when(csContext.getResourceCalculator()). thenReturn(resourceCalculator); when(csContext.getRMContext()).thenReturn(rmContext); + when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager(conf); @@ -122,13 +125,17 @@ public void setUp() throws IOException { containerTokenSecretManager); Map queues = new HashMap(); - CSQueue root = CapacitySchedulerQueueManager + root = CapacitySchedulerQueueManager .parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); - queue = spy(new LeafQueue(csContext, A, root, null)); + QueueResourceQuotas queueResourceQuotas = ((LeafQueue) queues.get(A)) + .getQueueResourceQuotas(); + doReturn(queueResourceQuotas).when(queue).getQueueResourceQuotas(); // Stub out ACL checks doReturn(true). @@ -189,6 +196,8 @@ public void testAMResourceLimit() throws Exception { // when there is only 1 user, and drops to 2G (the userlimit) when there // is a second user Resource clusterResource = Resource.newInstance(80 * GB, 40); + root.updateClusterResource(clusterResource, new ResourceLimits( + clusterResource)); queue.updateClusterResource(clusterResource, new ResourceLimits( clusterResource)); @@ -287,6 +296,8 @@ public void testLimitsComputation() throws Exception { CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); LeafQueue queue = (LeafQueue)queues.get(A); @@ -357,6 +368,8 @@ public void testLimitsComputation() throws Exception { csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); clusterResource = Resources.createResource(100 * 16 * GB); + root.updateClusterResource(clusterResource, new ResourceLimits( + clusterResource)); queue = (LeafQueue)queues.get(A); @@ -378,6 +391,8 @@ public void testLimitsComputation() throws Exception { root = CapacitySchedulerQueueManager.parseQueue( csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); + root.updateClusterResource(clusterResource, new ResourceLimits( + clusterResource)); queue = (LeafQueue)queues.get(A); assertEquals(9999, (int)csConf.getMaximumApplicationsPerQueue(queue.getQueuePath())); @@ -393,7 +408,7 @@ public void testActiveApplicationLimits() throws Exception { final String user_0 = "user_0"; final String user_1 = "user_1"; final String user_2 = "user_2"; - + assertEquals(Resource.newInstance(16 * GB, 1), queue.calculateAndGetAMResourceLimit()); assertEquals(Resource.newInstance(8 * GB, 1), @@ -578,6 +593,7 @@ public void testHeadroom() throws Exception { thenReturn(Resources.createResource(16*GB)); when(csContext.getResourceCalculator()).thenReturn(resourceCalculator); when(csContext.getRMContext()).thenReturn(rmContext); + when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); // Say cluster has 100 nodes of 16G each Resource clusterResource = Resources.createResource(100 * 16 * GB); @@ -586,6 +602,8 @@ public void testHeadroom() throws Exception { Map queues = new HashMap(); CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); + rootQueue.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); ResourceUsage queueCapacities = rootQueue.getQueueResourceUsage(); when(csContext.getClusterResourceUsage()) @@ -693,6 +711,8 @@ public void testHeadroom() throws Exception { // Now reduce cluster size and check for the smaller headroom clusterResource = Resources.createResource(90*16*GB); + rootQueue.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Any change is cluster resource needs to enforce user-limit recomputation. // In existing code, LeafQueue#updateClusterResource handled this. However diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java index 0aac2ef23da..d73f1c84373 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java @@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.preemption.PreemptionManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator; @@ -600,6 +601,7 @@ public void testHeadroom() throws Exception { RMContext spyRMContext = spy(rmContext); when(spyRMContext.getNodeLabelManager()).thenReturn(mgr); when(csContext.getRMContext()).thenReturn(spyRMContext); + when(csContext.getPreemptionManager()).thenReturn(new PreemptionManager()); mgr.activateNode(NodeId.newInstance("h0", 0), Resource.newInstance(160 * GB, 16)); // default Label @@ -615,6 +617,8 @@ public void testHeadroom() throws Exception { Map queues = new HashMap(); CSQueue rootQueue = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, "root", queues, queues, TestUtils.spyHook); + rootQueue.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); ResourceUsage queueResUsage = rootQueue.getQueueResourceUsage(); when(csContext.getClusterResourceUsage()) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java index 1edb0dab0b8..ebc517425c4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java @@ -4306,7 +4306,7 @@ public void testCSReservationWithRootUnblocked() throws Exception { null, null, NULL_UPDATE_REQUESTS); CapacityScheduler.schedule(cs); } - assertEquals("P2 Used Resource should be 8 GB", 8 * GB, + assertEquals("P2 Used Resource should be 7 GB", 7 * GB, cs.getQueue("p2").getUsedResources().getMemorySize()); //Free a container from X1 diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java index b386c18ce48..38768e57be7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerWithMultiResourceTypes.java @@ -61,11 +61,20 @@ public void testBasicCapacitySchedulerWithProfile() throws Exception { Map riMap = new HashMap<>(); // Initialize mandatory resources - riMap.put(ResourceInformation.MEMORY_URI, ResourceInformation.MEMORY_MB); - riMap.put(ResourceInformation.VCORES_URI, ResourceInformation.VCORES); - riMap.put(RESOURCE_1, ResourceInformation - .newInstance(RESOURCE_1, "", 0, ResourceTypes.COUNTABLE, 0, - Integer.MAX_VALUE)); + ResourceInformation memory = ResourceInformation.newInstance( + ResourceInformation.MEMORY_MB.getName(), + ResourceInformation.MEMORY_MB.getUnits(), + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB); + ResourceInformation vcores = ResourceInformation.newInstance( + ResourceInformation.VCORES.getName(), + ResourceInformation.VCORES.getUnits(), + YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES, + YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES); + riMap.put(ResourceInformation.MEMORY_URI, memory); + riMap.put(ResourceInformation.VCORES_URI, vcores); + riMap.put(RESOURCE_1, ResourceInformation.newInstance(RESOURCE_1, "", 0, + ResourceTypes.COUNTABLE, 0, Integer.MAX_VALUE)); ResourceUtils.initializeResourcesFromResourceInformationMap(riMap); @@ -107,7 +116,7 @@ protected ResourceProfilesManager createResourceProfileManager() { RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default"); MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1); - Assert.assertEquals(Resource.newInstance(1 * GB, 0), + Assert.assertEquals(Resource.newInstance(1 * GB, 1), leafQueue.getUsedResources()); RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId()); @@ -123,9 +132,9 @@ protected ResourceProfilesManager createResourceProfileManager() { // Do node heartbeats 1 time and check container allocated. cs.handle(new NodeUpdateSchedulerEvent(rmNode1)); - // Now used resource = + + // Now used resource = + Assert.assertEquals( - TestUtils.createResource(3 * GB, 2, ImmutableMap.of(RESOURCE_1, 2)), + TestUtils.createResource(3 * GB, 3, ImmutableMap.of(RESOURCE_1, 2)), leafQueue.getUsedResources()); // Acquire container diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java index 0fcc86d3928..fcfa0dd1796 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java @@ -242,6 +242,8 @@ public void testSortedQueues() throws Exception { Resources.createResource(numNodes * (memoryPerNode*GB), numNodes * coresPerNode); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Start testing CSQueue a = queues.get(A); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java index a32352b3af2..1426e881385 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java @@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueStateManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; @@ -191,6 +192,8 @@ private void setUpInternal(ResourceCalculator rC) throws Exception { CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); + root.updateClusterResource(Resources.createResource(100 * 16 * GB, 100 * 32), + new ResourceLimits(Resources.createResource(100 * 16 * GB, 100 * 32))); ResourceUsage queueResUsage = root.getQueueResourceUsage(); when(csContext.getClusterResourceUsage()) @@ -307,13 +310,11 @@ public void testInitializeQueue() throws Exception { // Verify the value for getAMResourceLimit for queues with < .1 maxcap Resource clusterResource = Resource.newInstance(50 * GB, 50); - a.updateClusterResource(clusterResource, + root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); assertEquals(Resource.newInstance(1 * GB, 1), a.calculateAndGetAMResourceLimit()); - b.updateClusterResource(clusterResource, - new ResourceLimits(clusterResource)); assertEquals(Resource.newInstance(5 * GB, 1), b.calculateAndGetAMResourceLimit()); } @@ -358,6 +359,8 @@ public void testSingleQueueOneUserMetrics() throws Exception { Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); @@ -556,6 +559,8 @@ public void testSingleQueueWithOneUser() throws Exception { Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); @@ -630,6 +635,8 @@ public void testSingleQueueWithOneUser() throws Exception { // Test max-capacity // Now - no more allocs since we are at max-cap a.setMaxCapacity(0.5f); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), @@ -699,6 +706,8 @@ public void testDRFUsageRatioRounding() throws Exception { Resource clusterResource = Resources.createResource(numNodes * (80 * GB), numNodes * 100); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Set user-limit. Need a small queue within a large cluster. b.setUserLimit(50); @@ -779,6 +788,8 @@ public void testDRFUserLimits() throws Exception { Resources.createResource(numNodes * (8 * GB), numNodes * 100); when(csContext.getNumClusterNodes()).thenReturn(numNodes); when(csContext.getClusterResource()).thenReturn(clusterResource); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests so that one application is memory dominant // and other application is vcores dominant @@ -891,6 +902,8 @@ public void testUserLimits() throws Exception { Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); @@ -915,6 +928,8 @@ public void testUserLimits() throws Exception { // Set user-limit a.setUserLimit(50); a.setUserLimitFactor(2); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // There're two active users assertEquals(2, a.getAbstractUsersManager().getNumActiveUsers()); @@ -940,7 +955,7 @@ public void testUserLimits() throws Exception { assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize()); // Allocate one container to app_0, before allocating this container, - // user-limit = ceil((4 + 1) / 2) = 3G. app_0's used resource (3G) <= + // user-limit = floor((5 + 1) / 2) = 3G. app_0's used resource (3G) <= // user-limit. applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_1, @@ -1012,6 +1027,8 @@ public void testUserSpecificUserLimits() throws Exception { Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests // app_0 asks for 3 3-GB containers @@ -1072,10 +1089,10 @@ public void testUserSpecificUserLimits() throws Exception { assertEquals(8*GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize()); - assertEquals(4*GB, + assertEquals(4 * GB, app_0.getTotalPendingRequestsPerPartition().get("").getMemorySize()); - assertEquals(1*GB, + assertEquals(1 * GB, app_1.getTotalPendingRequestsPerPartition().get("").getMemorySize()); } @@ -1100,6 +1117,8 @@ public void testComputeUserLimitAndSetHeadroom() throws IOException { final int numNodes = 2; Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); CapacitySchedulerQueueManager mockCapacitySchedulerQueueManager = mock(CapacitySchedulerQueueManager.class); @@ -1122,6 +1141,8 @@ public void testComputeUserLimitAndSetHeadroom() throws IOException { qb.setUserLimit(100); qb.setUserLimitFactor(1); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); final ApplicationAttemptId appAttemptId_0 = TestUtils.getMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0 = @@ -1256,106 +1277,6 @@ public void testComputeUserLimitAndSetHeadroom() throws IOException { } @Test - public void testUserHeadroomMultiApp() throws Exception { - // Mock the queue - LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); - //unset maxCapacity - a.setMaxCapacity(1.0f); - - // Users - final String user_0 = "user_0"; - final String user_1 = "user_1"; - - // Submit applications - final ApplicationAttemptId appAttemptId_0 = - TestUtils.getMockApplicationAttemptId(0, 0); - FiCaSchedulerApp app_0 = - new FiCaSchedulerApp(appAttemptId_0, user_0, a, - a.getAbstractUsersManager(), spyRMContext); - a.submitApplicationAttempt(app_0, user_0); - - final ApplicationAttemptId appAttemptId_1 = - TestUtils.getMockApplicationAttemptId(1, 0); - FiCaSchedulerApp app_1 = - new FiCaSchedulerApp(appAttemptId_1, user_0, a, - a.getAbstractUsersManager(), spyRMContext); - a.submitApplicationAttempt(app_1, user_0); // same user - - final ApplicationAttemptId appAttemptId_2 = - TestUtils.getMockApplicationAttemptId(2, 0); - FiCaSchedulerApp app_2 = - new FiCaSchedulerApp(appAttemptId_2, user_1, a, - a.getAbstractUsersManager(), spyRMContext); - a.submitApplicationAttempt(app_2, user_1); - - // Setup some nodes - String host_0 = "127.0.0.1"; - FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, - 0, 16*GB); - String host_1 = "127.0.0.2"; - FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, - 0, 16*GB); - - Map apps = ImmutableMap.of( - app_0.getApplicationAttemptId(), app_0, app_1.getApplicationAttemptId(), - app_1, app_2.getApplicationAttemptId(), app_2); - Map nodes = ImmutableMap.of(node_0.getNodeID(), - node_0, node_1.getNodeID(), node_1); - - final int numNodes = 2; - Resource clusterResource = Resources.createResource(numNodes * (16*GB), 1); - when(csContext.getNumClusterNodes()).thenReturn(numNodes); - - Priority priority = TestUtils.createMockPriority(1); - - app_0.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true, - priority, recordFactory))); - - applyCSAssignment(clusterResource, - a.assignContainers(clusterResource, node_0, - new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps); - assertEquals(1*GB, a.getUsedResources().getMemorySize()); - assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize()); - assertEquals(0*GB, app_1.getCurrentConsumption().getMemorySize()); - //Now, headroom is the same for all apps for a given user + queue combo - //and a change to any app's headroom is reflected for all the user's apps - //once those apps are active/have themselves calculated headroom for - //allocation at least one time - assertEquals(2*GB, app_0.getHeadroom().getMemorySize()); - assertEquals(0*GB, app_1.getHeadroom().getMemorySize());//not yet active - assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active - - app_1.updateResourceRequests(Collections.singletonList( - TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 2, true, - priority, recordFactory))); - - applyCSAssignment(clusterResource, - a.assignContainers(clusterResource, node_0, - new ResourceLimits(clusterResource), - SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), a, nodes, apps); - assertEquals(2*GB, a.getUsedResources().getMemorySize()); - assertEquals(1*GB, app_0.getCurrentConsumption().getMemorySize()); - assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize()); - assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); - assertEquals(1*GB, app_1.getHeadroom().getMemorySize());//now active - assertEquals(0*GB, app_2.getHeadroom().getMemorySize());//not yet active - - //Complete container and verify that headroom is updated, for both apps - //for the user - RMContainer rmContainer = app_0.getLiveContainers().iterator().next(); - a.completedContainer(clusterResource, app_0, node_0, rmContainer, - ContainerStatus.newInstance(rmContainer.getContainerId(), - ContainerState.COMPLETE, "", - ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), - RMContainerEventType.KILL, null, true); - - assertEquals(2*GB, app_0.getHeadroom().getMemorySize()); - assertEquals(2*GB, app_1.getHeadroom().getMemorySize()); - } - - @Test public void testHeadroomWithMaxCap() throws Exception { // Mock the queue LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A)); @@ -1403,7 +1324,7 @@ public void testHeadroomWithMaxCap() throws Exception { final int numNodes = 2; Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1); when(csContext.getNumClusterNodes()).thenReturn(numNodes); - + // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); app_0.updateResourceRequests(Collections.singletonList( @@ -1422,6 +1343,11 @@ public void testHeadroomWithMaxCap() throws Exception { a.setUserLimit(50); a.setUserLimitFactor(2); + ParentQueue root = (ParentQueue) queues + .get(CapacitySchedulerConfiguration.ROOT); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + // Now, only user_0 should be active since he is the only one with // outstanding requests assertEquals("There should only be 1 active user!", @@ -1438,7 +1364,7 @@ public void testHeadroomWithMaxCap() throws Exception { // TODO, fix headroom in the future patch assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); // User limit = 2G, 2 in use - assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); + assertEquals(1*GB, app_1.getHeadroom().getMemorySize()); // the application is not yet active // Again one to user_0 since he hasn't exceeded user limit yet @@ -1449,11 +1375,13 @@ public void testHeadroomWithMaxCap() throws Exception { assertEquals(3*GB, a.getUsedResources().getMemorySize()); assertEquals(2*GB, app_0.getCurrentConsumption().getMemorySize()); assertEquals(1*GB, app_1.getCurrentConsumption().getMemorySize()); - assertEquals(1*GB, app_0.getHeadroom().getMemorySize()); // 4G - 3G - assertEquals(1*GB, app_1.getHeadroom().getMemorySize()); // 4G - 3G - + assertEquals(0*GB, app_0.getHeadroom().getMemorySize()); + assertEquals(0*GB, app_1.getHeadroom().getMemorySize()); + // Submit requests for app_1 and set max-cap a.setMaxCapacity(.1f); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); app_2.updateResourceRequests(Collections.singletonList( TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 1, true, priority, recordFactory))); @@ -1485,6 +1413,108 @@ public void testHeadroomWithMaxCap() throws Exception { } @Test + public void testUserHeadroomMultiApp() throws Exception { + // Mock the queue + LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A)); + // unset maxCapacity + a.setMaxCapacity(1.0f); + + // Users + final String user_0 = "user_0"; + final String user_1 = "user_1"; + + // Submit applications + final ApplicationAttemptId appAttemptId_0 = TestUtils + .getMockApplicationAttemptId(0, 0); + FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a, + a.getAbstractUsersManager(), spyRMContext); + a.submitApplicationAttempt(app_0, user_0); + + final ApplicationAttemptId appAttemptId_1 = TestUtils + .getMockApplicationAttemptId(1, 0); + FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a, + a.getAbstractUsersManager(), spyRMContext); + a.submitApplicationAttempt(app_1, user_0); // same user + + final ApplicationAttemptId appAttemptId_2 = TestUtils + .getMockApplicationAttemptId(2, 0); + FiCaSchedulerApp app_2 = new FiCaSchedulerApp(appAttemptId_2, user_1, a, + a.getAbstractUsersManager(), spyRMContext); + a.submitApplicationAttempt(app_2, user_1); + + // Setup some nodes + String host_0 = "127.0.0.1"; + FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0, + 16 * GB); + String host_1 = "127.0.0.2"; + FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0, + 16 * GB); + + Map apps = ImmutableMap.of( + app_0.getApplicationAttemptId(), app_0, app_1.getApplicationAttemptId(), + app_1, app_2.getApplicationAttemptId(), app_2); + Map nodes = ImmutableMap.of(node_0.getNodeID(), + node_0, node_1.getNodeID(), node_1); + + final int numNodes = 2; + Resource clusterResource = Resources.createResource(numNodes * (16 * GB), + 1); + when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + + Priority priority = TestUtils.createMockPriority(1); + + app_0.updateResourceRequests( + Collections.singletonList(TestUtils.createResourceRequest( + ResourceRequest.ANY, 1 * GB, 1, true, priority, recordFactory))); + + applyCSAssignment(clusterResource, + a.assignContainers(clusterResource, node_0, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), + a, nodes, apps); + assertEquals(1 * GB, a.getUsedResources().getMemorySize()); + assertEquals(1 * GB, app_0.getCurrentConsumption().getMemorySize()); + assertEquals(0 * GB, app_1.getCurrentConsumption().getMemorySize()); + // Now, headroom is the same for all apps for a given user + queue combo + // and a change to any app's headroom is reflected for all the user's apps + // once those apps are active/have themselves calculated headroom for + // allocation at least one time + assertEquals(2 * GB, app_0.getHeadroom().getMemorySize()); + assertEquals(2 * GB, app_1.getHeadroom().getMemorySize());// not yet active + assertEquals(3 * GB, app_2.getHeadroom().getMemorySize());// not yet active + + app_1.updateResourceRequests( + Collections.singletonList(TestUtils.createResourceRequest( + ResourceRequest.ANY, 1 * GB, 2, true, priority, recordFactory))); + + applyCSAssignment(clusterResource, + a.assignContainers(clusterResource, node_0, + new ResourceLimits(clusterResource), + SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY), + a, nodes, apps); + assertEquals(2 * GB, a.getUsedResources().getMemorySize()); + assertEquals(1 * GB, app_0.getCurrentConsumption().getMemorySize()); + assertEquals(1 * GB, app_1.getCurrentConsumption().getMemorySize()); + assertEquals(1 * GB, app_0.getHeadroom().getMemorySize()); + assertEquals(1 * GB, app_1.getHeadroom().getMemorySize());// now active + assertEquals(3 * GB, app_2.getHeadroom().getMemorySize());// not yet active + + // Complete container and verify that headroom is updated, for both apps + // for the user + RMContainer rmContainer = app_0.getLiveContainers().iterator().next(); + a.completedContainer(clusterResource, app_0, node_0, rmContainer, + ContainerStatus.newInstance(rmContainer.getContainerId(), + ContainerState.COMPLETE, "", + ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), + RMContainerEventType.KILL, null, true); + + assertEquals(2 * GB, app_0.getHeadroom().getMemorySize()); + assertEquals(2 * GB, app_1.getHeadroom().getMemorySize()); + } + + @Test public void testSingleQueueWithMultipleUsers() throws Exception { // Mock the queue @@ -1542,6 +1572,8 @@ public void testSingleQueueWithMultipleUsers() throws Exception { Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); when(csContext.getClusterResource()).thenReturn(clusterResource); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); @@ -1624,6 +1656,8 @@ public void testSingleQueueWithMultipleUsers() throws Exception { // Test max-capacity // Now - no more allocs since we are at max-cap a.setMaxCapacity(0.5f); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), @@ -1638,6 +1672,8 @@ public void testSingleQueueWithMultipleUsers() throws Exception { // Now, allocations should goto app_3 since it's under user-limit a.setMaxCapacity(1.0f); a.setUserLimitFactor(1); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); applyCSAssignment(clusterResource, a.assignContainers(clusterResource, node_0, new ResourceLimits(clusterResource), @@ -1743,6 +1779,8 @@ public void testReservation() throws Exception { Resource clusterResource = Resources.createResource(numNodes * (4*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priority = TestUtils.createMockPriority(1); @@ -1880,6 +1918,8 @@ public void testReservationExchange() throws Exception { final int numNodes = 3; Resource clusterResource = Resources.createResource(numNodes * (4*GB), numNodes * 16); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); when(csContext.getMaximumResourceCapability()).thenReturn( Resources.createResource(4*GB, 16)); @@ -2051,6 +2091,8 @@ public void testLocalityScheduling() throws Exception { Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests and submit Priority priority = TestUtils.createMockPriority(1); @@ -2237,11 +2279,10 @@ public void testRackLocalityDelayScheduling() throws Exception { CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); - queues = newQueues; root.reinitialize(newRoot, cs.getClusterResource()); // Manipulate queue 'b' - LeafQueue a = stubLeafQueue((LeafQueue) queues.get(B)); + LeafQueue a = stubLeafQueue((LeafQueue) newQueues.get(B)); // Check locality parameters. assertEquals(2, a.getNodeLocalityDelay()); @@ -2277,6 +2318,8 @@ public void testRackLocalityDelayScheduling() throws Exception { Resource clusterResource = Resources.createResource(numNodes * (8 * GB), numNodes * 16); when(spyRMContext.getScheduler().getNumClusterNodes()).thenReturn(numNodes); + newRoot.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests and submit Priority priority = TestUtils.createMockPriority(1); @@ -2412,6 +2455,8 @@ public void testApplicationPriorityScheduling() throws Exception { Resource clusterResource = Resources.createResource(numNodes * (8*GB), 1); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests and submit List app_0_requests_0 = new ArrayList(); @@ -2545,6 +2590,8 @@ public void testSchedulingConstraints() throws Exception { Resource clusterResource = Resources.createResource( numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests and submit Priority priority = TestUtils.createMockPriority(1); @@ -2660,17 +2707,14 @@ public void testActivateApplicationAfterQueueRefresh() throws Exception { assertEquals(2, e.getNumActiveApplications()); assertEquals(1, e.getNumPendingApplications()); - csConf.setDouble(CapacitySchedulerConfiguration - .MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, - CapacitySchedulerConfiguration - .DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT * 2); + csConf.setDouble( + CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + CapacitySchedulerConfiguration.DEFAULT_MAXIMUM_APPLICATIONMASTERS_RESOURCE_PERCENT + * 2); Map newQueues = new HashMap(); - CSQueue newRoot = - CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, - CapacitySchedulerConfiguration.ROOT, - newQueues, queues, - TestUtils.spyHook); - queues = newQueues; + CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, + csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, + TestUtils.spyHook); root.reinitialize(newRoot, csContext.getClusterResource()); // after reinitialization @@ -2697,7 +2741,6 @@ public void testLocalityDelaysAfterQueueRefresh() throws Exception { CapacitySchedulerConfiguration.ROOT, newQueues, queues, TestUtils.spyHook); - queues = newQueues; root.reinitialize(newRoot, cs.getClusterResource()); // after reinitialization @@ -2745,7 +2788,7 @@ public void testActivateApplicationByUpdatingClusterResource() assertEquals(1, e.getNumPendingApplications()); Resource clusterResource = Resources.createResource(200 * 16 * GB, 100 * 32); - e.updateClusterResource(clusterResource, + root.updateClusterResource(clusterResource, new ResourceLimits(clusterResource)); // after updating cluster resource @@ -2837,6 +2880,9 @@ public void testLocalityConstraints() throws Exception { numNodes * (8*GB), numNodes * 1); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + // Setup resource-requests // resourceName: // host_0_0: < 1, 1GB, 1, true > @@ -3036,36 +3082,44 @@ public void testLocalityConstraints() throws Exception { @Test public void testMaxAMResourcePerQueuePercentAfterQueueRefresh() throws Exception { + Map queues = new HashMap(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); - Resource clusterResource = Resources - .createResource(100 * 16 * GB, 100 * 32); + final String newRootName = "root" + System.currentTimeMillis(); + setupQueueConfiguration(csConf, newRootName); + + Resource clusterResource = Resources.createResource(100 * 16 * GB, + 100 * 32); CapacitySchedulerContext csContext = mockCSContext(csConf, clusterResource); when(csContext.getRMContext()).thenReturn(rmContext); - csConf.setFloat(CapacitySchedulerConfiguration. - MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.1f); - ParentQueue root = new ParentQueue(csContext, - CapacitySchedulerConfiguration.ROOT, null, null); - csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80); - LeafQueue a = new LeafQueue(csContext, A, root, null); - assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f); - assertEquals(a.calculateAndGetAMResourceLimit(), - Resources.createResource(160 * GB, 1)); - - csConf.setFloat(CapacitySchedulerConfiguration. - MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f); - LeafQueue newA = new LeafQueue(csContext, A, root, null); - a.reinitialize(newA, clusterResource); - assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f); - assertEquals(a.calculateAndGetAMResourceLimit(), - Resources.createResource(320 * GB, 1)); + csConf.setFloat( + CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + 0.1f); + + CSQueue root; + root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, null, + CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); - Resource newClusterResource = Resources.createResource(100 * 20 * GB, - 100 * 32); - a.updateClusterResource(newClusterResource, - new ResourceLimits(newClusterResource)); - // 100 * 20 * 0.2 = 400 - assertEquals(a.calculateAndGetAMResourceLimit(), - Resources.createResource(400 * GB, 1)); + // Manipulate queue 'a' + LeafQueue b = stubLeafQueue((LeafQueue) queues.get(B)); + assertEquals(0.1f, b.getMaxAMResourcePerQueuePercent(), 1e-3f); + assertEquals(b.calculateAndGetAMResourceLimit(), + Resources.createResource(159 * GB, 1)); + + csConf.setFloat( + CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, + 0.2f); + clusterResource = Resources.createResource(100 * 20 * GB, 100 * 32); + Map newQueues = new HashMap(); + CSQueue newRoot = CapacitySchedulerQueueManager.parseQueue(csContext, + csConf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues, + TestUtils.spyHook); + root.reinitialize(newRoot, clusterResource); + + b = stubLeafQueue((LeafQueue) newQueues.get(B)); + assertEquals(b.calculateAndGetAMResourceLimit(), + Resources.createResource(320 * GB, 1)); } @Test @@ -3142,6 +3196,8 @@ public void testFifoAssignment() throws Exception { Resource clusterResource = Resources.createResource(numNodes * (16 * GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); String user_0 = "user_0"; @@ -3308,6 +3364,8 @@ public void testFairAssignment() throws Exception { Resource clusterResource = Resources.createResource( numNodes * (16*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); String user_0 = "user_0"; @@ -3435,6 +3493,8 @@ public void testLocalityDelaySkipsApplication() throws Exception { Resource clusterResource = Resources.createResource(numNodes * (8*GB), numNodes * 16); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests and submit // App0 has node local request for host_0/host_1, and app1 has node local @@ -3533,6 +3593,8 @@ public void testGetTotalPendingResourcesConsideringUserLimitOneUser() Resource clusterResource = Resources.createResource(numNodes * (100*GB), numNodes * 128); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Pending resource requests for app_0 and app_1 total 5GB. Priority priority = TestUtils.createMockPriority(1); @@ -3699,6 +3761,8 @@ public void testGetTotalPendingResourcesConsideringUserLimitTwoUsers() Resource clusterResource = Resources.createResource(numNodes * (100*GB), numNodes * 128); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Pending resource requests for user_0: app_0 and app_1 total 3GB. Priority priority = TestUtils.createMockPriority(1); @@ -3921,7 +3985,7 @@ public void testApplicationQueuePercent() // Queue "test" consumes 100% of the cluster, so its capacity and absolute // capacity are both 1.0f. - Queue queue = createQueue("test", null, 1.0f, 1.0f); + Queue queue = createQueue("test", null, 1.0f, 1.0f, res); final String user = "user1"; FiCaSchedulerApp app = new FiCaSchedulerApp(appAttId, user, queue, @@ -3938,7 +4002,8 @@ public void testApplicationQueuePercent() // Queue "test2" is a child of root and its capacity is 50% of root. As a // child of root, its absolute capaicty is also 50%. - queue = createQueue("test2", null, 0.5f, 0.5f); + queue = createQueue("test2", null, 0.5f, 0.5f, + Resources.divideAndCeil(dominantResourceCalculator, res, 2)); app = new FiCaSchedulerApp(appAttId, user, queue, queue.getAbstractUsersManager(), rmContext); app.getAppAttemptResourceUsage().incUsed(requestedResource); @@ -3950,7 +4015,8 @@ public void testApplicationQueuePercent() // Queue "test2.1" is 50% of queue "test2", which is 50% of the cluster. // Therefore, "test2.1" capacity is 50% and absolute capacity is 25%. - AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f); + AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f, + Resources.divideAndCeil(dominantResourceCalculator, res, 4)); app = new FiCaSchedulerApp(appAttId, user, qChild, qChild.getAbstractUsersManager(), rmContext); app.getAppAttemptResourceUsage().incUsed(requestedResource); @@ -3969,7 +4035,7 @@ private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) { } private AbstractCSQueue createQueue(String name, Queue parent, float capacity, - float absCap) { + float absCap, Resource res) { CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf()); QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null, null, QueueState.RUNNING, null, "", null, false); @@ -3981,6 +4047,13 @@ private AbstractCSQueue createQueue(String name, Queue parent, float capacity, QueueCapacities qCaps = mock(QueueCapacities.class); when(qCaps.getAbsoluteCapacity(any())).thenReturn(absCap); when(queue.getQueueCapacities()).thenReturn(qCaps); + QueueResourceQuotas qQuotas = mock(QueueResourceQuotas.class); + when(qQuotas.getConfiguredMinResource(any())).thenReturn(res); + when(qQuotas.getConfiguredMaxResource(any())).thenReturn(res); + when(qQuotas.getEffectiveMinResource(any())).thenReturn(res); + when(qQuotas.getEffectiveMaxResource(any())).thenReturn(res); + when(queue.getQueueResourceQuotas()).thenReturn(qQuotas); + when(queue.getEffectiveCapacity(any())).thenReturn(res); return queue; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java index a9196d109fc..fe66aba05ef 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java @@ -68,6 +68,11 @@ public class TestParentQueue { + private static final Resource QUEUE_B_RESOURCE = Resource + .newInstance(14 * 1024, 22); + private static final Resource QUEUE_A_RESOURCE = Resource + .newInstance(6 * 1024, 10); + private static final Log LOG = LogFactory.getLog(TestParentQueue.class); RMContext rmContext; @@ -118,6 +123,23 @@ private void setupSingleLevelQueues(CapacitySchedulerConfiguration conf) { LOG.info("Setup top-level queues a and b"); } + private void setupSingleLevelQueuesWithAbsoluteResource( + CapacitySchedulerConfiguration conf) { + + // Define top-level queues + conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[]{A, B}); + + final String Q_A = CapacitySchedulerConfiguration.ROOT + "." + A; + conf.setMinimumResourceRequirement("", Q_A, + QUEUE_A_RESOURCE); + + final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B; + conf.setMinimumResourceRequirement("", Q_B, + QUEUE_B_RESOURCE); + + LOG.info("Setup top-level queues a and b with absolute resource"); + } + private FiCaSchedulerApp getMockApplication(int appId, String user) { FiCaSchedulerApp application = mock(FiCaSchedulerApp.class); doReturn(user).when(application).getUser(); @@ -249,6 +271,8 @@ public void testSingleLevelQueues() throws Exception { Resources.createResource(numNodes * (memoryPerNode*GB), numNodes * coresPerNode); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Start testing LeafQueue a = (LeafQueue)queues.get(A); @@ -494,6 +518,8 @@ public void testMultiLevelQueues() throws Exception { Resources.createResource(numNodes * (memoryPerNode*GB), numNodes * coresPerNode); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Start testing CSQueue a = queues.get(A); @@ -710,6 +736,8 @@ public void testOffSwitchScheduling() throws Exception { Resources.createResource(numNodes * (memoryPerNode*GB), numNodes * coresPerNode); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Start testing LeafQueue a = (LeafQueue)queues.get(A); @@ -790,6 +818,8 @@ public void testOffSwitchSchedulingMultiLevelQueues() throws Exception { Resources.createResource(numNodes * (memoryPerNode*GB), numNodes * coresPerNode); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Start testing LeafQueue b3 = (LeafQueue)queues.get(B3); @@ -923,6 +953,78 @@ public void testQueueAcl() throws Exception { reset(c); } + @Test + public void testAbsoluteResourceWithChangeInClusterResource() + throws Exception { + // Setup queue configs + setupSingleLevelQueuesWithAbsoluteResource(csConf); + + Map queues = new HashMap(); + CSQueue root = CapacitySchedulerQueueManager.parseQueue(csContext, csConf, + null, CapacitySchedulerConfiguration.ROOT, queues, queues, + TestUtils.spyHook); + + // Setup some nodes + final int memoryPerNode = 10; + int coresPerNode = 16; + int numNodes = 2; + + Resource clusterResource = Resources.createResource( + numNodes * (memoryPerNode * GB), numNodes * coresPerNode); + when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + + // Start testing + LeafQueue a = (LeafQueue) queues.get(A); + LeafQueue b = (LeafQueue) queues.get(B); + + assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(), + QUEUE_A_RESOURCE); + assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(), + QUEUE_B_RESOURCE); + assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(), + QUEUE_A_RESOURCE); + assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(), + QUEUE_B_RESOURCE); + + numNodes = 1; + clusterResource = Resources.createResource(numNodes * (memoryPerNode * GB), + numNodes * coresPerNode); + when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + + Resource QUEUE_B_RESOURCE_HALF = Resource.newInstance(7 * 1024, 11); + Resource QUEUE_A_RESOURCE_HALF = Resource.newInstance(3 * 1024, 5); + assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(), + QUEUE_A_RESOURCE); + assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(), + QUEUE_B_RESOURCE); + assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(), + QUEUE_A_RESOURCE_HALF); + assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(), + QUEUE_B_RESOURCE_HALF); + + coresPerNode = 40; + clusterResource = Resources.createResource(numNodes * (memoryPerNode * GB), + numNodes * coresPerNode); + when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + + Resource QUEUE_B_RESOURCE_70PERC = Resource.newInstance(7 * 1024, 27); + Resource QUEUE_A_RESOURCE_30PERC = Resource.newInstance(3 * 1024, 12); + assertEquals(a.getQueueResourceQuotas().getConfiguredMinResource(), + QUEUE_A_RESOURCE); + assertEquals(b.getQueueResourceQuotas().getConfiguredMinResource(), + QUEUE_B_RESOURCE); + assertEquals(a.getQueueResourceQuotas().getEffectiveMinResource(), + QUEUE_A_RESOURCE_30PERC); + assertEquals(b.getQueueResourceQuotas().getEffectiveMinResource(), + QUEUE_B_RESOURCE_70PERC); + } + @After public void tearDown() throws Exception { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java index 32f022f7156..08557203f61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java @@ -267,6 +267,8 @@ public void testReservation() throws Exception { final int numNodes = 3; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -454,6 +456,8 @@ public void testReservationLimitOtherUsers() throws Exception { final int numNodes = 3; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -600,6 +604,8 @@ public void testReservationNoContinueLook() throws Exception { final int numNodes = 3; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -782,6 +788,8 @@ public void testAssignContainersNeedToUnreserve() throws Exception { final int numNodes = 2; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -896,6 +904,10 @@ public void testGetAppToUnreserve() throws Exception { String host_1 = "host_1"; FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0, 8 * GB); + + Resource clusterResource = Resources.createResource(2 * 8 * GB); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests @@ -1068,6 +1080,8 @@ public void testAssignToQueue() throws Exception { final int numNodes = 2; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -1256,6 +1270,8 @@ public void testAssignToUser() throws Exception { final int numNodes = 2; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); @@ -1418,6 +1434,9 @@ public void testReservationsNoneAvailable() throws Exception { final int numNodes = 3; Resource clusterResource = Resources.createResource(numNodes * (8 * GB)); when(csContext.getNumClusterNodes()).thenReturn(numNodes); + root.updateClusterResource(clusterResource, + new ResourceLimits(clusterResource)); + // Setup resource-requests Priority priorityAM = TestUtils.createMockPriority(1); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java index e3c108a581e..b9d5b82d464 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/policy/TestPriorityUtilizationQueueOrderingPolicy.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.ImmutableTable; import org.apache.hadoop.yarn.api.records.Priority; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.junit.Assert; @@ -52,6 +53,8 @@ when(q.getQueueCapacities()).thenReturn(qc); when(q.getPriority()).thenReturn(Priority.newInstance(priorities[i])); + QueueResourceQuotas qr = new QueueResourceQuotas(); + when(q.getQueueResourceQuotas()).thenReturn(qr); list.add(q); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index 1108f1add7b..013234888e7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -354,10 +354,10 @@ private void verifyClusterSchedulerGeneric(String type, float usedCapacity, private void verifySubQueue(JSONObject info, String q, float parentAbsCapacity, float parentAbsMaxCapacity) throws JSONException, Exception { - int numExpectedElements = 18; + int numExpectedElements = 20; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 31; + numExpectedElements = 33; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length());