diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 47cea19..929d44c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -560,7 +560,7 @@ boolean hasPendingResourceRequest(String nodePartition, queueUsage, nodePartition, cluster, schedulingMode); } - boolean accessibleToPartition(String nodePartition) { + public boolean accessibleToPartition(String nodePartition) { // if queue's label is *, it can access any node if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 2eeda66..496eeac 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -22,13 +22,18 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerHealth; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo; @@ -61,6 +66,7 @@ static class CSQInfo { CapacitySchedulerInfo csinfo; CapacitySchedulerQueueInfo qinfo; + //String label; } static class LeafQueueInfoBlock extends HtmlBlock { @@ -190,10 +196,30 @@ public void render(Block html) { static class QueuesBlock extends HtmlBlock { final CapacityScheduler cs; final CSQInfo csqinfo; + private List nodeLabelsInfo; - @Inject QueuesBlock(ResourceManager rm, CSQInfo info) { + @Inject + QueuesBlock(ResourceManager rm, CSQInfo info) { cs = (CapacityScheduler) rm.getResourceScheduler(); csqinfo = info; + RMNodeLabelsManager nodeLabelManager = + rm.getRMContext().getNodeLabelManager(); + nodeLabelsInfo = nodeLabelManager.pullRMNodeLabelsInfo(); +// RMNodeLabel rmNodelabel; +// Resource emptyResource = Resource.newInstance(0, 0); +// if (null != nodeLabelsInfo && nodeLabelsInfo.size() > 0) { +// for (int i = 0; i < nodeLabelsInfo.size();) { +// rmNodelabel = nodeLabelsInfo.get(i); +// if (rmNodelabel.getIsExclusive() +// && !emptyResource.equals(rmNodelabel.getResource())) { +// // Keep only the labels which has resources hence we display only +// // schedule-able labels +// i++; +// continue; +// } +// nodeLabelsInfo.remove(i); +// } +// } } @Override @@ -246,12 +272,6 @@ public void render(Block html) { span().$style(Q_END)._("100% ")._(). span(".q", "default")._()._(); } else { - CSQueue root = cs.getRootQueue(); - CapacitySchedulerInfo sinfo = new CapacitySchedulerInfo(root, cs); - csqinfo.csinfo = sinfo; - csqinfo.qinfo = null; - - float used = sinfo.getUsedCapacity() / 100; ul. li().$style("margin-bottom: 1em"). span().$style("font-weight: bold")._("Legend:")._(). @@ -263,8 +283,20 @@ public void render(Block html) { _("Used (over capacity)")._(). span().$class("qlegend ui-corner-all ui-state-default"). _("Max Capacity")._(). - _(). - li(). + _(); + + float used = 0; + if (null == nodeLabelsInfo + || (nodeLabelsInfo.size() == 1 && nodeLabelsInfo.get(0) + .getLabelName().isEmpty())) { + CSQueue root = cs.getRootQueue(); + CapacitySchedulerInfo sinfo = new CapacitySchedulerInfo(root, cs); + csqinfo.csinfo = sinfo; + csqinfo.qinfo = null; + + used = sinfo.getUsedCapacity() / 100; + //label is not enabled in the cluster or there's only "default" label, + ul.li(). a(_Q).$style(width(Q_MAX_WIDTH)). span().$style(join(width(used), ";left:0%;", used > 1 ? Q_OVER : Q_UNDER))._(".")._(). @@ -272,6 +304,42 @@ public void render(Block html) { span().$class("qstats").$style(left(Q_STATS_POS)). _(join(percent(used), " used"))._(). _(QueueBlock.class)._(); + } else { + for( RMNodeLabel label : nodeLabelsInfo) { + CSQueue root = cs.getRootQueue(); + if (!((AbstractCSQueue) root).accessibleToPartition(label + .getLabelName())) { + // Skip displaying the hierarchy for the labels which are not + // accessible by any queue + continue; + } + CapacitySchedulerInfo sinfo = + new CapacitySchedulerInfo(root, cs, label.getLabelName()); + csqinfo.csinfo = sinfo; + csqinfo.qinfo = null; + + QueueCapacities queueCapacities = root.getQueueCapacities(); + used = queueCapacities.getUsedCapacity(label.getLabelName()); + ul.li(). + a(_Q).$style(width(Q_MAX_WIDTH)). + span().$style(join(width(used), ";left:0%;", + used > 1 ? Q_OVER : Q_UNDER))._(".")._(). + span(".q", "Partition=" + label.getLabelName())._(). + span().$class("qstats").$style(left(Q_STATS_POS)). + _(join(percent(used), " used"))._(); + + //for the queue hierarchy under label + UL underLabel = html.ul("#pq"); + underLabel.li(). + a(_Q).$style(width(Q_MAX_WIDTH)). + span().$style(join(width(used), ";left:0%;", + used > 1 ? Q_OVER : Q_UNDER))._(".")._(). + span(".q", "root")._(). + span().$class("qstats").$style(left(Q_STATS_POS)). + _(join(percent(used), " used"))._(). + _(QueueBlock.class)._()._(); + } + } } ul._()._(). script().$type("text/javascript"). diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 9901878..29f3784 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -24,9 +24,11 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @XmlRootElement(name = "capacityScheduler") @XmlType(name = "capacityScheduler") @@ -55,7 +57,22 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { max = 1f; this.maxCapacity = max * 100; - queues = getQueues(parent); + queues = getQueues(parent, null); + health = new CapacitySchedulerHealthInfo(cs); + } + + public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs, + String label) { + QueueCapacities parentQueueCapacities = parent.getQueueCapacities(); + this.queueName = parent.getQueueName(); + this.usedCapacity = parentQueueCapacities.getUsedCapacity(label) * 100; + this.capacity = parentQueueCapacities.getCapacity(label) * 100; + float max = parentQueueCapacities.getMaximumCapacity(label); + if (max < EPSILON || max > 1f) + max = 1f; + this.maxCapacity = max * 100; + + queues = getQueues(parent, label); health = new CapacitySchedulerHealthInfo(cs); } @@ -79,18 +96,38 @@ public CapacitySchedulerQueueInfoList getQueues() { return this.queues; } - protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) { + protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent, + String nodeLabel) { CSQueue parentQueue = parent; CapacitySchedulerQueueInfoList queuesInfo = new CapacitySchedulerQueueInfoList(); - for (CSQueue queue : parentQueue.getChildQueues()) { - CapacitySchedulerQueueInfo info; - if (queue instanceof LeafQueue) { - info = new CapacitySchedulerLeafQueueInfo((LeafQueue)queue); - } else { - info = new CapacitySchedulerQueueInfo(queue); - info.queues = getQueues(queue); + if (null == nodeLabel) { + for (CSQueue queue : parentQueue.getChildQueues()) { + CapacitySchedulerQueueInfo info; + if (queue instanceof LeafQueue) { + info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue); + } else { + info = new CapacitySchedulerQueueInfo(queue); + info.queues = getQueues(queue, nodeLabel); + } + queuesInfo.addToQueueInfoList(info); + } + } else { + for (CSQueue queue : parentQueue.getChildQueues()) { + if (!((AbstractCSQueue)queue).accessibleToPartition(nodeLabel)) { + // Skip displaying the hierarchy for the queues for which the labels + // are not accessible + continue; + } + CapacitySchedulerQueueInfo info; + if (queue instanceof LeafQueue) { + info = + new CapacitySchedulerLeafQueueInfo((LeafQueue) queue, nodeLabel); + } else { + info = new CapacitySchedulerQueueInfo(queue, nodeLabel); + info.queues = getQueues(queue, nodeLabel); + } + queuesInfo.addToQueueInfoList(info); } - queuesInfo.addToQueueInfoList(info); } return queuesInfo; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index dc61078..ad4a7d8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -63,6 +63,24 @@ preemptionDisabled = q.getPreemptionDisabled(); orderingPolicyInfo = q.getOrderingPolicy().getInfo(); } + + + CapacitySchedulerLeafQueueInfo(LeafQueue q, String nodeLabel) { + super(q,nodeLabel); + numActiveApplications = q.getNumActiveApplications(); + numPendingApplications = q.getNumPendingApplications(); + numContainers = q.getNumContainers(); + maxApplications = q.getMaxApplications(); + maxApplicationsPerUser = q.getMaxApplicationsPerUser(); + userLimit = q.getUserLimit(); + users = new UsersInfo(q.getUsers()); + userLimitFactor = q.getUserLimitFactor(); + AMResourceLimit = new ResourceInfo(q.getAMResourceLimit()); + usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed()); + userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit()); + preemptionDisabled = q.getPreemptionDisabled(); + orderingPolicyInfo = q.getOrderingPolicy().getInfo(); + } public int getNumActiveApplications() { return numActiveApplications; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 40dddea..7b5d47a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -28,8 +28,10 @@ import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) @@ -88,6 +90,41 @@ Collections.sort(nodeLabels); } } + + CapacitySchedulerQueueInfo(CSQueue q, String nodeLabel) { + QueueCapacities qCapacities = q.getQueueCapacities(); + ResourceUsage queueResourceUsage = q.getQueueResourceUsage(); + + queuePath = q.getQueuePath(); + capacity = qCapacities.getCapacity(nodeLabel) * 100; + usedCapacity = qCapacities.getUsedCapacity(nodeLabel) * 100; + + maxCapacity = qCapacities.getMaximumCapacity(nodeLabel); + if (maxCapacity < EPSILON || maxCapacity > 1f) + maxCapacity = 1f; + maxCapacity *= 100; + + absoluteCapacity = + cap(qCapacities.getAbsoluteCapacity(nodeLabel), 0f, 1f) * 100; + absoluteMaxCapacity = + cap(qCapacities.getAbsoluteMaximumCapacity(nodeLabel), 0f, 1f) * 100; + absoluteUsedCapacity = + cap(qCapacities.getAbsoluteUsedCapacity(nodeLabel), 0f, 1f) * 100; + numApplications = q.getNumApplications(); + queueName = q.getQueueName(); + state = q.getState(); + resourcesUsed = new ResourceInfo(queueResourceUsage.getUsed(nodeLabel)); + if (q instanceof PlanQueue && !((PlanQueue) q).showReservationsAsQueues()) { + hideReservationQueues = true; + } + + // add labels + Set labelSet = q.getAccessibleNodeLabels(); + if (labelSet != null) { + nodeLabels.addAll(labelSet); + Collections.sort(nodeLabels); + } + } public float getCapacity() { return this.capacity;