diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 47cea19..929d44c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -560,7 +560,7 @@ boolean hasPendingResourceRequest(String nodePartition, queueUsage, nodePartition, cluster, schedulingMode); } - boolean accessibleToPartition(String nodePartition) { + public boolean accessibleToPartition(String nodePartition) { // if queue's label is *, it can access any node if (accessibleLabels != null && accessibleLabels.contains(RMNodeLabelsManager.ANY)) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index fa22a0d..9133e58 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -22,14 +22,19 @@ import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.nodelabels.RMNodeLabel; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerHealth; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerLeafQueueInfo; @@ -63,26 +68,33 @@ static class CSQInfo { CapacitySchedulerInfo csinfo; CapacitySchedulerQueueInfo qinfo; + String label; } static class LeafQueueInfoBlock extends HtmlBlock { final CapacitySchedulerLeafQueueInfo lqinfo; + private String nodeLabel; @Inject LeafQueueInfoBlock(ViewContext ctx, CSQInfo info) { super(ctx); lqinfo = (CapacitySchedulerLeafQueueInfo) info.qinfo; + nodeLabel = info.label; } @Override protected void render(Block html) { - + String labelTag = ""; + if (nodeLabel != null) { + nodeLabel = nodeLabel.length() == 0 ? "" : nodeLabel; + labelTag = " (Node-Label=" + nodeLabel + ")"; + } ResponseInfo ri = info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status"). _("Queue State:", lqinfo.getQueueState()). - _("Used Capacity:", percent(lqinfo.getUsedCapacity() / 100)). - _("Absolute Used Capacity:", percent(lqinfo.getAbsoluteUsedCapacity() / 100)). - _("Absolute Capacity:", percent(lqinfo.getAbsoluteCapacity() / 100)). - _("Absolute Max Capacity:", percent(lqinfo.getAbsoluteMaxCapacity() / 100)). - _("Used Resources:", lqinfo.getResourcesUsed().toString()). + _("Used Capacity"+labelTag+":", percent(lqinfo.getUsedCapacity() / 100)). + _("Absolute Used Capacity"+labelTag+":", percent(lqinfo.getAbsoluteUsedCapacity() / 100)). + _("Absolute Capacity"+labelTag+":", percent(lqinfo.getAbsoluteCapacity() / 100)). + _("Absolute Max Capacity"+labelTag+":", percent(lqinfo.getAbsoluteMaxCapacity() / 100)). + _("Used Resources"+labelTag+":", lqinfo.getResourcesUsed().toString()). _("Num Schedulable Applications:", Integer.toString(lqinfo.getNumActiveApplications())). _("Num Non-Schedulable Applications:", Integer.toString(lqinfo.getNumPendingApplications())). _("Num Containers:", Integer.toString(lqinfo.getNumContainers())). @@ -193,11 +205,15 @@ public void render(Block html) { final CapacityScheduler cs; final CSQInfo csqinfo; private final ResourceManager rm; + private List nodeLabelsInfo; @Inject QueuesBlock(ResourceManager rm, CSQInfo info) { cs = (CapacityScheduler) rm.getResourceScheduler(); csqinfo = info; this.rm = rm; + RMNodeLabelsManager nodeLabelManager = + rm.getRMContext().getNodeLabelManager(); + nodeLabelsInfo = nodeLabelManager.pullRMNodeLabelsInfo(); } @Override @@ -267,12 +283,6 @@ public void render(Block html) { span().$style(Q_END)._("100% ")._(). span(".q", "default")._()._(); } else { - CSQueue root = cs.getRootQueue(); - CapacitySchedulerInfo sinfo = new CapacitySchedulerInfo(root, cs); - csqinfo.csinfo = sinfo; - csqinfo.qinfo = null; - - float used = sinfo.getUsedCapacity() / 100; ul. li().$style("margin-bottom: 1em"). span().$style("font-weight: bold")._("Legend:")._(). @@ -284,8 +294,20 @@ public void render(Block html) { _("Used (over capacity)")._(). span().$class("qlegend ui-corner-all ui-state-default"). _("Max Capacity")._(). - _(). - li(). + _(); + + float used = 0; + if (null == nodeLabelsInfo + || (nodeLabelsInfo.size() == 1 && nodeLabelsInfo.get(0) + .getLabelName().isEmpty())) { + CSQueue root = cs.getRootQueue(); + CapacitySchedulerInfo sinfo = new CapacitySchedulerInfo(root, cs); + csqinfo.csinfo = sinfo; + csqinfo.qinfo = null; + + used = sinfo.getUsedCapacity() / 100; + //label is not enabled in the cluster or there's only "default" label, + ul.li(). a(_Q).$style(width(Q_MAX_WIDTH)). span().$style(join(width(used), ";left:0%;", used > 1 ? Q_OVER : Q_UNDER))._(".")._(). @@ -293,6 +315,44 @@ public void render(Block html) { span().$class("qstats").$style(left(Q_STATS_POS)). _(join(percent(used), " used"))._(). _(QueueBlock.class)._(); + } else { + for( RMNodeLabel label : nodeLabelsInfo) { + CSQueue root = cs.getRootQueue(); + if (!((AbstractCSQueue) root).accessibleToPartition(label + .getLabelName())) { + // Skip displaying the hierarchy for the labels which are not + // accessible by any queue + continue; + } + CapacitySchedulerInfo sinfo = + new CapacitySchedulerInfo(root, cs, label.getLabelName()); + csqinfo.csinfo = sinfo; + csqinfo.qinfo = null; + csqinfo.label = label.getLabelName(); + String nodeLabel = + csqinfo.label.length() == 0 ? "" : csqinfo.label; + QueueCapacities queueCapacities = root.getQueueCapacities(); + used = queueCapacities.getUsedCapacity(label.getLabelName()); + ul.li(). + a(_Q).$style(width(Q_MAX_WIDTH)). + span().$style(join(width(used), ";left:0%;", + used > 1 ? Q_OVER : Q_UNDER))._(".")._(). + span(".q", "Node-Label=" + nodeLabel)._(). + span().$class("qstats").$style(left(Q_STATS_POS)). + _(join(percent(used), " used"))._(); + + //for the queue hierarchy under label + UL underLabel = html.ul("#pq"); + underLabel.li(). + a(_Q).$style(width(Q_MAX_WIDTH)). + span().$style(join(width(used), ";left:0%;", + used > 1 ? Q_OVER : Q_UNDER))._(".")._(). + span(".q", "root")._(). + span().$class("qstats").$style(left(Q_STATS_POS)). + _(join(percent(used), " used"))._(). + _(QueueBlock.class)._()._(); + } + } } ul._()._(). script().$type("text/javascript"). diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 9901878..29f3784 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -24,9 +24,11 @@ import javax.xml.bind.annotation.XmlTransient; import javax.xml.bind.annotation.XmlType; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @XmlRootElement(name = "capacityScheduler") @XmlType(name = "capacityScheduler") @@ -55,7 +57,22 @@ public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs) { max = 1f; this.maxCapacity = max * 100; - queues = getQueues(parent); + queues = getQueues(parent, null); + health = new CapacitySchedulerHealthInfo(cs); + } + + public CapacitySchedulerInfo(CSQueue parent, CapacityScheduler cs, + String label) { + QueueCapacities parentQueueCapacities = parent.getQueueCapacities(); + this.queueName = parent.getQueueName(); + this.usedCapacity = parentQueueCapacities.getUsedCapacity(label) * 100; + this.capacity = parentQueueCapacities.getCapacity(label) * 100; + float max = parentQueueCapacities.getMaximumCapacity(label); + if (max < EPSILON || max > 1f) + max = 1f; + this.maxCapacity = max * 100; + + queues = getQueues(parent, label); health = new CapacitySchedulerHealthInfo(cs); } @@ -79,18 +96,38 @@ public CapacitySchedulerQueueInfoList getQueues() { return this.queues; } - protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent) { + protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent, + String nodeLabel) { CSQueue parentQueue = parent; CapacitySchedulerQueueInfoList queuesInfo = new CapacitySchedulerQueueInfoList(); - for (CSQueue queue : parentQueue.getChildQueues()) { - CapacitySchedulerQueueInfo info; - if (queue instanceof LeafQueue) { - info = new CapacitySchedulerLeafQueueInfo((LeafQueue)queue); - } else { - info = new CapacitySchedulerQueueInfo(queue); - info.queues = getQueues(queue); + if (null == nodeLabel) { + for (CSQueue queue : parentQueue.getChildQueues()) { + CapacitySchedulerQueueInfo info; + if (queue instanceof LeafQueue) { + info = new CapacitySchedulerLeafQueueInfo((LeafQueue) queue); + } else { + info = new CapacitySchedulerQueueInfo(queue); + info.queues = getQueues(queue, nodeLabel); + } + queuesInfo.addToQueueInfoList(info); + } + } else { + for (CSQueue queue : parentQueue.getChildQueues()) { + if (!((AbstractCSQueue)queue).accessibleToPartition(nodeLabel)) { + // Skip displaying the hierarchy for the queues for which the labels + // are not accessible + continue; + } + CapacitySchedulerQueueInfo info; + if (queue instanceof LeafQueue) { + info = + new CapacitySchedulerLeafQueueInfo((LeafQueue) queue, nodeLabel); + } else { + info = new CapacitySchedulerQueueInfo(queue, nodeLabel); + info.queues = getQueues(queue, nodeLabel); + } + queuesInfo.addToQueueInfoList(info); } - queuesInfo.addToQueueInfoList(info); } return queuesInfo; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index dc61078..ad4a7d8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -63,6 +63,24 @@ preemptionDisabled = q.getPreemptionDisabled(); orderingPolicyInfo = q.getOrderingPolicy().getInfo(); } + + + CapacitySchedulerLeafQueueInfo(LeafQueue q, String nodeLabel) { + super(q,nodeLabel); + numActiveApplications = q.getNumActiveApplications(); + numPendingApplications = q.getNumPendingApplications(); + numContainers = q.getNumContainers(); + maxApplications = q.getMaxApplications(); + maxApplicationsPerUser = q.getMaxApplicationsPerUser(); + userLimit = q.getUserLimit(); + users = new UsersInfo(q.getUsers()); + userLimitFactor = q.getUserLimitFactor(); + AMResourceLimit = new ResourceInfo(q.getAMResourceLimit()); + usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed()); + userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit()); + preemptionDisabled = q.getPreemptionDisabled(); + orderingPolicyInfo = q.getOrderingPolicy().getInfo(); + } public int getNumActiveApplications() { return numActiveApplications; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 40dddea..7b5d47a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -28,8 +28,10 @@ import javax.xml.bind.annotation.XmlTransient; import org.apache.hadoop.yarn.api.records.QueueState; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.PlanQueue; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities; @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) @@ -88,6 +90,41 @@ Collections.sort(nodeLabels); } } + + CapacitySchedulerQueueInfo(CSQueue q, String nodeLabel) { + QueueCapacities qCapacities = q.getQueueCapacities(); + ResourceUsage queueResourceUsage = q.getQueueResourceUsage(); + + queuePath = q.getQueuePath(); + capacity = qCapacities.getCapacity(nodeLabel) * 100; + usedCapacity = qCapacities.getUsedCapacity(nodeLabel) * 100; + + maxCapacity = qCapacities.getMaximumCapacity(nodeLabel); + if (maxCapacity < EPSILON || maxCapacity > 1f) + maxCapacity = 1f; + maxCapacity *= 100; + + absoluteCapacity = + cap(qCapacities.getAbsoluteCapacity(nodeLabel), 0f, 1f) * 100; + absoluteMaxCapacity = + cap(qCapacities.getAbsoluteMaximumCapacity(nodeLabel), 0f, 1f) * 100; + absoluteUsedCapacity = + cap(qCapacities.getAbsoluteUsedCapacity(nodeLabel), 0f, 1f) * 100; + numApplications = q.getNumApplications(); + queueName = q.getQueueName(); + state = q.getState(); + resourcesUsed = new ResourceInfo(queueResourceUsage.getUsed(nodeLabel)); + if (q instanceof PlanQueue && !((PlanQueue) q).showReservationsAsQueues()) { + hideReservationQueues = true; + } + + // add labels + Set labelSet = q.getAccessibleNodeLabels(); + if (labelSet != null) { + nodeLabels.addAll(labelSet); + Collections.sort(nodeLabels); + } + } public float getCapacity() { return this.capacity;