From 10fc68be24eed38eaca1b8eca96a6a81ae16aa7a Mon Sep 17 00:00:00 2001 From: Sunil G Date: Tue, 3 Oct 2017 23:43:31 +0530 Subject: [PATCH] YARN-7254 --- .../yarn/api/records/QueueConfigurations.java | 38 +++++++++++ .../src/main/proto/yarn_protos.proto | 2 + .../records/impl/pb/QueueConfigurationsPBImpl.java | 73 +++++++++++++++++++++- .../scheduler/capacity/AbstractCSQueue.java | 10 ++- .../scheduler/capacity/CSQueue.java | 4 +- .../scheduler/capacity/LeafQueue.java | 4 +- .../scheduler/capacity/ParentQueue.java | 18 +++++- .../scheduler/capacity/UsersManager.java | 2 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 23 +++++-- .../webapp/dao/CapacitySchedulerInfo.java | 3 +- .../webapp/dao/CapacitySchedulerLeafQueueInfo.java | 6 +- .../webapp/dao/CapacitySchedulerQueueInfo.java | 10 ++- .../webapp/dao/PartitionQueueCapacitiesInfo.java | 17 ++++- .../webapp/dao/QueueCapacitiesInfo.java | 13 +++- 14 files changed, 194 insertions(+), 29 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java index e25c8aa..dfe515a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueConfigurations.java @@ -147,4 +147,42 @@ public static QueueConfigurations newInstance(float capacity, @Private @Unstable public abstract void setMaxAMPercentage(float maxAMPercentage); + + /** + * Get the effective minimum capacity of queue (from absolute resource). + * + * @return minimum resource capability + */ + @Public + @Unstable + public abstract Resource getEffectiveMinCapacity(); + + /** + * Set the effective minimum capacity of queue (from absolute resource). + * + * @param capacity + * minimum resource capability + */ + @Private + @Unstable + public abstract void setEffectiveMinCapacity(Resource capacity); + + /** + * Get the effective maximum capacity of queue (from absolute resource). + * + * @return maximum resource capability + */ + @Public + @Unstable + public abstract Resource getEffectiveMaxCapacity(); + + /** + * Set the effective maximum capacity of queue (from absolute resource). + * + * @param capacity + * maximum resource capability + */ + @Private + @Unstable + public abstract void setEffectiveMaxCapacity(Resource capacity); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto index 81ebd79..e9c8724 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto @@ -472,6 +472,8 @@ message QueueConfigurationsProto { optional float maxCapacity = 3; optional float absoluteMaxCapacity = 4; optional float maxAMPercentage = 5; + optional ResourceProto effectiveMinCapacity = 6; + optional ResourceProto effectiveMaxCapacity = 7; } message QueueConfigurationsMapProto { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java index f308bce..743f3d4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java @@ -19,8 +19,10 @@ package org.apache.hadoop.yarn.api.records.impl.pb; import org.apache.hadoop.yarn.api.records.QueueConfigurations; +import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProto; import org.apache.hadoop.yarn.proto.YarnProtos.QueueConfigurationsProtoOrBuilder; +import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto; import com.google.protobuf.TextFormat; @@ -29,6 +31,8 @@ QueueConfigurationsProto proto = QueueConfigurationsProto.getDefaultInstance(); QueueConfigurationsProto.Builder builder = null; + Resource minResource = null; + Resource maxResource = null; boolean viaProto = false; public QueueConfigurationsPBImpl() { @@ -41,11 +45,29 @@ public QueueConfigurationsPBImpl(QueueConfigurationsProto proto) { } public QueueConfigurationsProto getProto() { + mergeLocalToProto(); proto = viaProto ? proto : builder.build(); viaProto = true; return proto; } + private void mergeLocalToProto() { + if (viaProto) + maybeInitBuilder(); + mergeLocalToBuilder(); + proto = builder.build(); + viaProto = true; + } + + private void mergeLocalToBuilder() { + if (this.minResource != null) { + builder.setEffectiveMinCapacity(convertToProtoFormat(this.minResource)); + } + if (this.maxResource != null) { + builder.setEffectiveMaxCapacity(convertToProtoFormat(this.maxResource)); + } + } + @Override public float getCapacity() { QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; @@ -106,6 +128,56 @@ public void setMaxAMPercentage(float maxAMPercentage) { builder.setMaxAMPercentage(maxAMPercentage); } + @Override + public Resource getEffectiveMinCapacity() { + QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; + if (this.minResource != null) { + return this.minResource; + } + if (!p.hasEffectiveMinCapacity()) { + return null; + } + this.minResource = convertFromProtoFormat(p.getEffectiveMinCapacity()); + return this.minResource; + } + + @Override + public void setEffectiveMinCapacity(Resource capacity) { + maybeInitBuilder(); + if (capacity == null) + builder.clearEffectiveMinCapacity(); + this.minResource = capacity; + } + + @Override + public Resource getEffectiveMaxCapacity() { + QueueConfigurationsProtoOrBuilder p = viaProto ? proto : builder; + if (this.maxResource != null) { + return this.maxResource; + } + if (!p.hasEffectiveMaxCapacity()) { + return null; + } + this.maxResource = convertFromProtoFormat(p.getEffectiveMaxCapacity()); + return this.maxResource; + } + + @Override + public void setEffectiveMaxCapacity(Resource capacity) { + maybeInitBuilder(); + if (capacity == null) + builder.clearEffectiveMaxCapacity(); + this.maxResource = capacity; + } + + private ResourcePBImpl convertFromProtoFormat(ResourceProto p) { + return new ResourcePBImpl(p); + } + + private ResourceProto convertToProtoFormat(Resource t) { + return ProtoUtils.convertToProtoFormat(t); + } + private void maybeInitBuilder() { if (viaProto || builder == null) { builder = QueueConfigurationsProto.newBuilder(proto); @@ -133,5 +205,4 @@ public boolean equals(Object other) { } return false; } - } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 39ec57a..ceeb112 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -390,6 +390,10 @@ protected void updateConfigurableResourceRequirement(String queuePath, Resource maxResource = conf.getMaximumResourceRequirement(label, queuePath, resourceTypes); + if (LOG.isDebugEnabled()) { + LOG.debug("capacityConfigType is '" + capacityConfigType + + "' for queue '" + getQueueName()); + } if (this.capacityConfigType.equals(CapacityConfigType.NONE)) { this.capacityConfigType = (!minResource.equals(Resources.none()) && queueCapacities.getAbsoluteCapacity(label) == 0f) @@ -473,12 +477,6 @@ public Resource getEffectiveCapacity(String label) { } @Override - public Resource getEffectiveCapacityUp(String label) { - return Resources - .clone(getQueueResourceQuotas().getEffectiveMinResourceUp(label)); - } - - @Override public Resource getEffectiveCapacityDown(String label, Resource factor) { return Resources.normalizeDown(resourceCalculator, getQueueResourceQuotas().getEffectiveMinResource(label), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index a93d74e..2a292f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -25,7 +25,6 @@ import java.util.Set; import java.util.concurrent.locks.ReentrantReadWriteLock; -import com.google.common.annotations.VisibleForTesting; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceStability.Stable; import org.apache.hadoop.security.AccessControlException; @@ -52,6 +51,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet; +import com.google.common.annotations.VisibleForTesting; + /** * CSQueue represents a node in the tree of * hierarchical queues in the {@link CapacityScheduler}. @@ -379,7 +380,6 @@ public void validateSubmitApplication(ApplicationId applicationId, * @return effective queue capacity */ Resource getEffectiveCapacity(String label); - Resource getEffectiveCapacityUp(String label); Resource getEffectiveCapacityDown(String label, Resource factor); /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index 23d5088..1078a45 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -656,7 +656,7 @@ public Resource getUserAMResourceLimitPerPartition( 1.0f / Math.max(getAbstractUsersManager().getNumActiveUsers(), 1)); effectiveUserLimit = Math.min(effectiveUserLimit * userWeight, 1.0f); - Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition); + Resource queuePartitionResource = getEffectiveCapacity(nodePartition); Resource userAMLimit = Resources.multiplyAndNormalizeUp( resourceCalculator, queuePartitionResource, @@ -685,7 +685,7 @@ public Resource calculateAndGetAMResourceLimitPerPartition( * non-labeled), * with per-partition am-resource-percent to get the max am * resource limit for this queue and partition. */ - Resource queuePartitionResource = getEffectiveCapacityUp(nodePartition); + Resource queuePartitionResource = getEffectiveCapacity(nodePartition); Resource queueCurrentLimit = Resources.none(); // For non-labeled partition, we need to consider the current queue diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java index 5b529d6..5a12eb8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java @@ -915,6 +915,7 @@ private void calculateEffectiveResourcesAndCapacity(String label, queueResourceQuotas.setConfiguredMaxResource(label, resourceByLabel); queueResourceQuotas.setEffectiveMinResource(label, resourceByLabel); queueResourceQuotas.setEffectiveMaxResource(label, resourceByLabel); + queueCapacities.setAbsoluteCapacity(label, 1.0f); } // Total configured min resources of direct children of queue @@ -961,7 +962,7 @@ private void calculateEffectiveResourcesAndCapacity(String label, // present could also be taken from effective max resource of parent. Resource parentMaxRes = queueResourceQuotas .getConfiguredMaxResource(label); - if (parentMaxRes.equals(Resources.none())) { + if (parent != null && parentMaxRes.equals(Resources.none())) { parentMaxRes = parent.getQueueResourceQuotas() .getEffectiveMaxResource(label); } @@ -977,6 +978,21 @@ private void calculateEffectiveResourcesAndCapacity(String label, parentMaxRes); childQueue.getQueueResourceQuotas().setEffectiveMaxResource(label, Resources.clone(effMaxResource)); + + // In cases where we still need to update some units based on + // percentage, we have to calculate an absolute value. + Resource parentMinRes = queueResourceQuotas + .getConfiguredMinResource(label); + if (parent != null && parentMinRes.equals(Resources.none())) { + parentMinRes = parent.getQueueResourceQuotas() + .getEffectiveMinResource(label); + } + float absCap = rc.divide(clusterResource, + childQueue.getQueueResourceQuotas().getEffectiveMinResource(), + parentMinRes); + childQueue.getQueueCapacities().setAbsoluteCapacity(label, + (float) absCap + / getQueueCapacities().getAbsoluteCapacity(label)); } else { childQueue.getQueueResourceQuotas().setEffectiveMinResource(label, Resources.multiply(resourceByLabel, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java index a066a35..7a97a8c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UsersManager.java @@ -686,7 +686,7 @@ private Resource computeUserLimit(String userName, Resource clusterResource, * * If we're running over capacity, then its (usedResources + required) * (which extra resources we are allocating) */ - Resource queueCapacity = lQueue.getEffectiveCapacityUp(nodePartition); + Resource queueCapacity = lQueue.getEffectiveCapacity(nodePartition); /* * Assume we have required resource equals to minimumAllocation, this can diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 17bb104..6dec704 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractUsersManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest; @@ -907,6 +908,7 @@ protected void getActivedAppDiagnosticMessage( StringBuilder diagnosticMessage) { LeafQueue queue = getCSLeafQueue(); QueueCapacities queueCapacities = queue.getQueueCapacities(); + QueueResourceQuotas queueResourceQuotas = queue.getQueueResourceQuotas(); diagnosticMessage.append(" Details : AM Partition = "); diagnosticMessage.append(appAMNodePartitionName.isEmpty() ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName); @@ -928,6 +930,18 @@ protected void getActivedAppDiagnosticMessage( queueCapacities.getAbsoluteMaximumCapacity(appAMNodePartitionName) * 100); diagnosticMessage.append(" % ; "); + diagnosticMessage.append("Queue's capacity (absolute resource) = "); + diagnosticMessage.append( + queueResourceQuotas.getEffectiveMinResource(appAMNodePartitionName)); + diagnosticMessage.append(" ; "); + diagnosticMessage.append("Queue's used capacity (absolute resource) = "); + diagnosticMessage + .append(queue.getQueueResourceUsage().getUsed(appAMNodePartitionName)); + diagnosticMessage.append(" ; "); + diagnosticMessage.append("Queue's max capacity (absolute resource) = "); + diagnosticMessage.append( + queueResourceQuotas.getEffectiveMaxResource(appAMNodePartitionName)); + diagnosticMessage.append(" ; "); } /** @@ -991,13 +1005,10 @@ public ApplicationResourceUsageReport getResourceUsageReport() { ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator(); if (!calc.isInvalidDivisor(totalPartitionRes)) { - float queueAbsMaxCapPerPartition = - ((AbstractCSQueue) getQueue()).getQueueCapacities() - .getAbsoluteCapacity(getAppAMNodePartitionName()); + Resource effCap = ((AbstractCSQueue) getQueue()) + .getEffectiveCapacity(getAppAMNodePartitionName()); float queueUsagePerc = calc.divide(totalPartitionRes, - report.getUsedResources(), - Resources.multiply(totalPartitionRes, queueAbsMaxCapPerPartition)) - * 100; + report.getUsedResources(), effCap) * 100; report.setQueueUsagePercentage(queueUsagePerc); } return report; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 32e4ac5..ca42041 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -59,7 +59,8 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { max = 1f; this.maxCapacity = max * 100; - capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), false); + capacities = new QueueCapacitiesInfo(parent.getQueueCapacities(), + parent.getQueueResourceQuotas(), false); queues = getQueues(parent); health = new CapacitySchedulerHealthInfo(cs); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index 7dcdf58..343c78e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -25,6 +25,7 @@ import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @@ -89,8 +90,9 @@ protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { } @Override - protected void populateQueueCapacities(QueueCapacities qCapacities) { - capacities = new QueueCapacitiesInfo(qCapacities); + protected void populateQueueCapacities(QueueCapacities qCapacities, + QueueResourceQuotas qResQuotas) { + capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas); } public int getNumActiveApplications() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 86b2fea..d4de9ae 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -28,6 +28,7 @@ import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; @@ -103,7 +104,8 @@ Collections.sort(nodeLabels); } QueueCapacities qCapacities = q.getQueueCapacities(); - populateQueueCapacities(qCapacities); + QueueResourceQuotas qResQuotas = q.getQueueResourceQuotas(); + populateQueueCapacities(qCapacities, qResQuotas); ResourceUsage queueResourceUsage = q.getQueueResourceUsage(); populateQueueResourceUsage(queueResourceUsage); @@ -118,8 +120,10 @@ protected void populateQueueResourceUsage(ResourceUsage queueResourceUsage) { resources = new ResourcesInfo(queueResourceUsage, false); } - protected void populateQueueCapacities(QueueCapacities qCapacities) { - capacities = new QueueCapacitiesInfo(qCapacities, false); + protected void populateQueueCapacities(QueueCapacities qCapacities, + QueueResourceQuotas qResQuotas) { + capacities = new QueueCapacitiesInfo(qCapacities, qResQuotas, + false); } public float getCapacity() { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java index 5e298f9..3eea06f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionQueueCapacitiesInfo.java @@ -21,6 +21,8 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.api.records.Resource; + /** * This class represents queue capacities for a given partition */ @@ -36,13 +38,16 @@ private float absoluteUsedCapacity; private float absoluteMaxCapacity = 100; private float maxAMLimitPercentage; + private ResourceInfo minResource; + private ResourceInfo maxResource; public PartitionQueueCapacitiesInfo() { } public PartitionQueueCapacitiesInfo(String partitionName, float capacity, float usedCapacity, float maxCapacity, float absCapacity, - float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage) { + float absUsedCapacity, float absMaxCapacity, float maxAMLimitPercentage, + Resource minRes, Resource maxRes) { super(); this.partitionName = partitionName; this.capacity = capacity; @@ -52,6 +57,8 @@ public PartitionQueueCapacitiesInfo(String partitionName, float capacity, this.absoluteUsedCapacity = absUsedCapacity; this.absoluteMaxCapacity = absMaxCapacity; this.maxAMLimitPercentage = maxAMLimitPercentage; + this.minResource = new ResourceInfo(minRes); + this.maxResource = new ResourceInfo(maxRes); } public float getCapacity() { @@ -117,4 +124,12 @@ public float getMaxAMLimitPercentage() { public void setMaxAMLimitPercentage(float maxAMLimitPercentage) { this.maxAMLimitPercentage = maxAMLimitPercentage; } + + public ResourceInfo getMinResource() { + return minResource; + } + + public ResourceInfo getMaxResource() { + return maxResource; + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java index 9a3e439..35bcffb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/QueueCapacitiesInfo.java @@ -24,7 +24,10 @@ import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueResourceQuotas; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; +import org.apache.hadoop.yarn.util.resource.Resources; /** * DAO which wraps PartitionQueueCapacitiesInfo applicable for a queue @@ -39,6 +42,7 @@ public QueueCapacitiesInfo() { } public QueueCapacitiesInfo(QueueCapacities capacities, + QueueResourceQuotas resourceQuotas, boolean considerAMUsage) { if (capacities == null) { return; @@ -68,12 +72,15 @@ public QueueCapacitiesInfo(QueueCapacities capacities, queueCapacitiesByPartition.add(new PartitionQueueCapacitiesInfo( partitionName, capacity, usedCapacity, maxCapacity, absCapacity, absUsedCapacity, absMaxCapacity, - considerAMUsage ? maxAMLimitPercentage : 0f)); + considerAMUsage ? maxAMLimitPercentage : 0f, + resourceQuotas.getEffectiveMinResource(partitionName), + resourceQuotas.getEffectiveMaxResource(partitionName))); } } - public QueueCapacitiesInfo(QueueCapacities capacities) { - this(capacities, true); + public QueueCapacitiesInfo(QueueCapacities capacities, + QueueResourceQuotas resourceQuotas) { + this(capacities, resourceQuotas, true); } public void add(PartitionQueueCapacitiesInfo partitionQueueCapacitiesInfo) { -- 2.10.1 (Apple Git-78)