diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java index 65d32e8..56b972d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java @@ -87,6 +87,8 @@ protected CapacitySchedulerContext csContext; protected YarnAuthorizationProvider authorizer = null; + private Resource cr; + public AbstractCSQueue(CapacitySchedulerContext cs, String queueName, CSQueue parent, CSQueue old) throws IOException { this.labelManager = cs.getRMContext().getNodeLabelManager(); @@ -291,6 +293,7 @@ synchronized void setupQueueConfigs(Resource clusterResource) .getReservationContinueLook(); this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this); + this.cr = clusterResource; } protected QueueInfo getQueueInfo() { @@ -322,7 +325,9 @@ synchronized void allocateResource(Resource clusterResource, if (nodeLabels == null || nodeLabels.isEmpty()) { queueUsage.incUsed(resource); } else { - for (String label : Sets.intersection(accessibleLabels, nodeLabels)) { + Set anls = (accessibleLabels.contains(RMNodeLabelsManager.ANY)) + ? labelManager.getClusterNodeLabels() : accessibleLabels; + for (String label : Sets.intersection(anls, nodeLabels)) { queueUsage.incUsed(label, resource); } } @@ -338,7 +343,9 @@ protected synchronized void releaseResource(Resource clusterResource, if (null == nodeLabels || nodeLabels.isEmpty()) { queueUsage.decUsed(resource); } else { - for (String label : Sets.intersection(accessibleLabels, nodeLabels)) { + Set anls = (accessibleLabels.contains(RMNodeLabelsManager.ANY)) + ? labelManager.getClusterNodeLabels() : accessibleLabels; + for (String label : Sets.intersection(anls, nodeLabels)) { queueUsage.decUsed(label, resource); } } @@ -523,4 +530,33 @@ public boolean accessibleToPartition(String nodePartition) { // sorry, you cannot access return false; } + + /** + * Used capacity of label = + * (queue's used resources labeled by nodeLabel) + * / ( (all resources labeled by nodLabel) + * X (percent of labeled resources allocated to this queue) ) + */ + public synchronized float getUsedCapacity(String nodeLabel) { + Resource availableToQueue = + Resources.multiply(labelManager.getResourceByLabel(nodeLabel, cr), + queueCapacities.getAbsoluteCapacity(nodeLabel)); + if (!Resources.greaterThan(resourceCalculator, cr, + availableToQueue, Resources.none())) { + return 0.0f; + } + return + Resources.divide(resourceCalculator, cr, + queueUsage.getUsed(nodeLabel), availableToQueue); + } + + public float getAbsoluteUsedCapacity(String nodeLabel) { + Resource labeledResources = labelManager.getResourceByLabel(nodeLabel, cr); + if (!Resources.greaterThan(resourceCalculator, cr, + labeledResources, Resources.none())) { + return 0.0f; + } + return Resources.divide(resourceCalculator, cr, + queueUsage.getUsed(nodeLabel), labeledResources); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java index 1a9448a..0147910 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java @@ -284,4 +284,18 @@ public void attachContainer(Resource clusterResource, * @return resourceUsage */ public ResourceUsage getQueueResourceUsage(); + + /** + * Get used capacity for the specified label + * @param nodeLabel + * @return used capacity + */ + public float getUsedCapacity(String nodeLabel); + + /** + * Get absolute used capacity for the specified label + * @param nodeLabel + * @return absolute used capacity + */ + public float getAbsoluteUsedCapacity(String nodeLabel); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index e8daa88..02a55ef 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -424,8 +424,12 @@ public synchronized User getUser(String userName) { ArrayList usersToReturn = new ArrayList(); for (Map.Entry entry : users.entrySet()) { User user = entry.getValue(); - usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(user - .getUsed()), user.getActiveApplications(), user + Resource usedRes = Resource.newInstance(0, 0); + for (String nl : getAccessibleLabelSet()) { + Resources.addTo(usedRes, user.getUsed(nl)); + } + usersToReturn.add(new UserInfo(entry.getKey(), usedRes, + user.getActiveApplications(), user .getPendingApplications(), Resources.clone(user .getConsumedAMResources()), Resources.clone(user .getUserResourceLimit()))); @@ -433,6 +437,22 @@ public synchronized User getUser(String userName) { return usersToReturn; } + /** + * Gets the labels which are accessible by this queue. If ANY label can be + * accessed, put all labels in the set. + * @return accessiglbe node labels + */ + protected final Set getAccessibleLabelSet() { + Set nodeLabels = new HashSet(); + if (this.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) { + nodeLabels.addAll(labelManager.getClusterNodeLabels()); + } else { + nodeLabels.addAll(this.getAccessibleNodeLabels()); + } + nodeLabels.add(RMNodeLabelsManager.NO_LABEL); + return nodeLabels; + } + @Override public synchronized void reinitialize( CSQueue newlyParsedQueue, Resource clusterResource) diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 9f2af70..7b11845 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -307,8 +307,7 @@ public void render(Block html) { String nodeLabel = csqinfo.label.length() == 0 ? "" : csqinfo.label; - QueueCapacities queueCapacities = root.getQueueCapacities(); - used = queueCapacities.getUsedCapacity(label.getLabelName()); + used = root.getUsedCapacity(label.getLabelName()); String partitionUiTag = "Partition: " + nodeLabel + " " + label.getResource(); ul.li(). diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java index 8bf7ebf..5ee4973 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerInfo.java @@ -54,7 +54,7 @@ public CapacitySchedulerInfo(CSQueue parent, NodeLabel nodeLabel) { String label = nodeLabel.getLabelName(); QueueCapacities parentQueueCapacities = parent.getQueueCapacities(); this.queueName = parent.getQueueName(); - this.usedCapacity = parentQueueCapacities.getUsedCapacity(label) * 100; + this.usedCapacity = parent.getUsedCapacity(label) * 100; this.capacity = parentQueueCapacities.getCapacity(label) * 100; float max = parentQueueCapacities.getMaximumCapacity(label); if (max < EPSILON || max > 1f) @@ -98,8 +98,8 @@ protected CapacitySchedulerQueueInfoList getQueues(CSQueue parent, List childNonLeafQueues = new ArrayList<>(); for (CSQueue queue : parent.getChildQueues()) { if (!((AbstractCSQueue) queue).accessibleToPartition(nodeLabel - .getLabelName())) { - // Skip displaying the hierarchy for the queues for which the exclusive + .getLabelName())) { + // Skip displaying the hierarchy for the queues for which the // labels are not accessible continue; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java index 81b28fd..cdf16bf 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerQueueInfo.java @@ -67,7 +67,7 @@ queuePath = q.getQueuePath(); capacity = qCapacities.getCapacity(nodeLabel) * 100; - usedCapacity = qCapacities.getUsedCapacity(nodeLabel) * 100; + usedCapacity = q.getUsedCapacity(nodeLabel) * 100; maxCapacity = qCapacities.getMaximumCapacity(nodeLabel); if (maxCapacity < EPSILON || maxCapacity > 1f) @@ -78,8 +78,7 @@ cap(qCapacities.getAbsoluteCapacity(nodeLabel), 0f, 1f) * 100; absoluteMaxCapacity = cap(qCapacities.getAbsoluteMaximumCapacity(nodeLabel), 0f, 1f) * 100; - absoluteUsedCapacity = - cap(qCapacities.getAbsoluteUsedCapacity(nodeLabel), 0f, 1f) * 100; + absoluteUsedCapacity = q.getAbsoluteUsedCapacity(nodeLabel) * 100; numApplications = q.getNumApplications(); queueName = q.getQueueName(); state = q.getState();