diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java index bc22073..d9edda4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java @@ -41,6 +41,8 @@ private ApplicationAttemptId attemptId = null; // preemption info private Resource resourcePreempted = Resource.newInstance(0, 0); + // application headroom + private Resource headroom = Resource.newInstance(0, 0); private AtomicInteger numNonAMContainersPreempted = new AtomicInteger(0); private AtomicBoolean isPreempted = new AtomicBoolean(false); @@ -145,4 +147,12 @@ public void incNumAllocatedContainers(NodeType containerType, public int getTotalAllocatedContainers() { return this.totalAllocatedContainers; } + + public Resource getApplicationAttemptHeadroom() { + return headroom; + } + + public void setApplicationAttemptHeadRoom(Resource headRoom) { + this.headroom = headRoom; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index ed78097..9816699 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -632,4 +632,14 @@ public void incNumAllocatedContainers(NodeType containerType, requestType); } } + + public void setApplicationHeadroomForMetrics(Resource headroom) { + RMAppAttempt attempt = + rmContext.getRMApps().get(attemptId.getApplicationId()) + .getCurrentAppAttempt(); + if (attempt != null) { + attempt.getRMAppAttemptMetrics().setApplicationAttemptHeadRoom( + Resources.clone(headroom)); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index a607a62..0e47d09 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -423,7 +423,8 @@ public synchronized User getUser(String userName) { for (Map.Entry entry: users.entrySet()) { usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone( entry.getValue().getUsed()), entry.getValue().getActiveApplications(), - entry.getValue().getPendingApplications())); + entry.getValue().getPendingApplications(), Resources.clone(entry + .getValue().getConsumedAMResources()))); } return usersToReturn; } @@ -580,6 +581,12 @@ public synchronized Resource getUserAMResourceLimit() { userLimitFactor, minimumAllocation); } + public synchronized Resource getUserResourceLimit() { + return Resources.multiplyAndNormalizeUp(resourceCalculator, + absoluteCapacityResource, (userLimit / 100.0f) * userLimitFactor, + minimumAllocation); + } + private synchronized void activateApplications() { //limit of allowed resource usage for application masters Resource amLimit = getAMResourceLimit(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java index 65c911b..8193982 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java @@ -30,16 +30,19 @@ public class UserInfo { protected String username; protected ResourceInfo resourcesUsed; + protected ResourceInfo amResourcesUsed; protected int numPendingApplications; protected int numActiveApplications; UserInfo() {} - UserInfo(String username, Resource resUsed, int activeApps, int pendingApps) { + UserInfo(String username, Resource resUsed, int activeApps, int pendingApps, + Resource amResUsed) { this.username = username; this.resourcesUsed = new ResourceInfo(resUsed); this.numActiveApplications = activeApps; this.numPendingApplications = pendingApps; + this.amResourcesUsed = new ResourceInfo(amResUsed); } public String getUsername() { @@ -57,4 +60,8 @@ public int getNumPendingApplications() { public int getNumActiveApplications() { return numActiveApplications; } + + public ResourceInfo getAMResourcesUsed() { + return amResourcesUsed; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java index 9f97b13..e1f200c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java @@ -268,7 +268,10 @@ public synchronized Allocation getAllocation(ResourceCalculator rc, minimumAllocation, numCont); ContainersAndNMTokensAllocation allocation = pullNewlyAllocatedContainersAndNMTokens(); - return new Allocation(allocation.getContainerList(), getHeadroom(), null, + + Resource headroom = getHeadroom(); + setApplicationHeadroomForMetrics(headroom); + return new Allocation(allocation.getContainerList(), headroom, null, currentContPreemption, Collections.singletonList(rr), allocation.getNMTokenList()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java index e8a9555..cb60dda 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java @@ -938,9 +938,11 @@ public Allocation allocate(ApplicationAttemptId appAttemptId, application.updateBlacklist(blacklistAdditions, blacklistRemovals); ContainersAndNMTokensAllocation allocation = application.pullNewlyAllocatedContainersAndNMTokens(); - return new Allocation(allocation.getContainerList(), - application.getHeadroom(), preemptionContainerIds, null, null, - allocation.getNMTokenList()); + + Resource headroom = application.getHeadroom(); + application.setApplicationHeadroomForMetrics(headroom); + return new Allocation(allocation.getContainerList(), headroom, + preemptionContainerIds, null, null, allocation.getNMTokenList()); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index 00508b8..3333d41 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -165,6 +165,13 @@ protected void render(Block html) { int attemptNumNonAMContainerPreempted = attemptMetrics == null ? 0 : attemptMetrics .getNumNonAMContainersPreempted(); + Resource applicationHeadRoom = + attemptMetrics == null ? Resources.none() : attemptMetrics + .getApplicationAttemptHeadroom(); + // if application is in final state, reset headroom to 0 + if (rmApp.isAppFinalStateStored()) { + applicationHeadRoom = Resource.newInstance(0, 0); + } info("Application Overview") ._("User:", app.getUser()) @@ -200,7 +207,9 @@ protected void render(Block html) { attemptNumNonAMContainerPreempted) ._("Aggregate Resource Allocation:", String.format("%d MB-seconds, %d vcore-seconds", - appMerics.getMemorySeconds(), appMerics.getVcoreSeconds())); + appMerics.getMemorySeconds(), appMerics.getVcoreSeconds())) + ._("Application Headroom For Current Attempt:", applicationHeadRoom); + pdiv._(); Collection attempts = rmApp.getAppAttempts().values(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 83df72b..97d3a53 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -92,6 +92,7 @@ private String getPercentage(ResourceInfo numerator, ResourceInfo denominator) { @Override protected void render(Block html) { StringBuilder activeUserList = new StringBuilder(""); + StringBuilder userAMResourceList = new StringBuilder(""); ResourceInfo usedResources = lqinfo.getResourcesUsed(); ArrayList users = lqinfo.getUsers().getUsersList(); for (UserInfo entry: users) { @@ -100,6 +101,8 @@ protected void render(Block html) { .append(", Schedulable Apps: " + entry.getNumActiveApplications()) .append(", Non-Schedulable Apps: " + entry.getNumPendingApplications()) .append(">
"); //Force line break + userAMResourceList.append(entry.getUsername()).append( + entry.getAMResourcesUsed().toString()); } ResponseInfo ri = info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status"). @@ -115,11 +118,14 @@ protected void render(Block html) { _("Max Applications:", Integer.toString(lqinfo.getMaxApplications())). _("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())). _("Max Application Master Resources:", lqinfo.getAMResourceLimit().toString()). + _("Used Application Master Resources:", lqinfo.getUsedAMResource().toString()). _("Max Application Master Resources Per User:", lqinfo.getUserAMResourceLimit().toString()). + _("Used Application Master Resources Per User:", userAMResourceList.toString()). _("Configured Capacity:", percent(lqinfo.getCapacity() / 100)). _("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)). _("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%"). _("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())). + _("Max Resources Per User:", lqinfo.getUserResourceLimit().toString()). _r("Active Users: ", activeUserList.toString()). _("Accessible Node Labels:", StringUtils.join(",", lqinfo.getNodeLabels())). _("Preemption:", lqinfo.getPreemptionDisabled() ? "disabled" : "enabled"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index a8b0d32..cfc4c46 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -36,7 +36,9 @@ protected UsersInfo users; // To add another level in the XML protected float userLimitFactor; protected ResourceInfo aMResourceLimit; + protected ResourceInfo usedAMResource; protected ResourceInfo userAMResourceLimit; + protected ResourceInfo userResourceLimit; protected boolean preemptionDisabled; CapacitySchedulerLeafQueueInfo() { @@ -53,7 +55,9 @@ users = new UsersInfo(q.getUsers()); userLimitFactor = q.getUserLimitFactor(); aMResourceLimit = new ResourceInfo(q.getAMResourceLimit()); + usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed()); userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit()); + userResourceLimit = new ResourceInfo(q.getUserResourceLimit()); preemptionDisabled = q.getPreemptionDisabled(); } @@ -93,11 +97,19 @@ public float getUserLimitFactor() { public ResourceInfo getAMResourceLimit() { return aMResourceLimit; } + + public ResourceInfo getUsedAMResource() { + return usedAMResource; + } public ResourceInfo getUserAMResourceLimit() { return userAMResourceLimit; } + public ResourceInfo getUserResourceLimit() { + return userResourceLimit; + } + public boolean getPreemptionDisabled() { return preemptionDisabled; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java index 8cad057..3a46610 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java @@ -272,10 +272,9 @@ public void testLimitsComputation() throws Exception { LeafQueue queue = (LeafQueue)queues.get(A); - LOG.info("Queue 'A' -" + - " aMResourceLimit=" + queue.getAMResourceLimit() + - " UserAMResourceLimit=" + - queue.getUserAMResourceLimit()); + LOG.info("Queue 'A' -" + " aMResourceLimit=" + queue.getAMResourceLimit() + + " UserAMResourceLimit=" + queue.getUserAMResourceLimit() + + " UserResourceLimit" + queue.getUserResourceLimit()); assertEquals(queue.getAMResourceLimit(), Resource.newInstance(160*GB, 1)); assertEquals(queue.getUserAMResourceLimit(), diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index ba5c73b..9d5b5d6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -347,7 +347,7 @@ private void verifySubQueue(JSONObject info, String q, int numExpectedElements = 13; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 24; + numExpectedElements = 26; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); -- 1.9.2.msysgit.0