diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java index 4a82c93..74765f1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java @@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo; import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo; import org.apache.hadoop.yarn.util.ConverterUtils; +import org.apache.hadoop.yarn.util.resource.Resources; import org.apache.hadoop.yarn.webapp.YarnWebParams; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; @@ -172,7 +173,7 @@ public ApplicationAttemptReport run() throws Exception { ._("Diagnostics Info:", appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt.getDiagnosticsInfo()); - html._(InfoBlock.class); + if (exceptionWhenGetContainerReports) { html @@ -183,6 +184,14 @@ public ApplicationAttemptReport run() throws Exception { return; } + // TODO Need to get HeadRoom from scheduler and render it web ui + DIV pdiv = html._(InfoBlock.class).div(_INFO_WRAP); + info("Application Attempt Overview").clear(); + info("Application Attempt Metrics")._("Application Attempt Headroom : ", 0); + pdiv._(); + + html._(InfoBlock.class); + // Container Table TBODY> tbody = html.table("#containers").thead().tr().th(".id", "Container ID") diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java index a607a62..0bc0f99 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java @@ -421,9 +421,11 @@ public synchronized User getUser(String userName) { public synchronized ArrayList getUsers() { ArrayList usersToReturn = new ArrayList(); for (Map.Entry entry: users.entrySet()) { - usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone( - entry.getValue().getUsed()), entry.getValue().getActiveApplications(), - entry.getValue().getPendingApplications())); + usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(entry + .getValue().getUsed()), entry.getValue().getActiveApplications(), + entry.getValue().getPendingApplications(), Resources.clone(entry + .getValue().getConsumedAMResources()), Resources.clone(entry + .getValue().getUserResourceLimit()))); } return usersToReturn; } @@ -1158,7 +1160,7 @@ private Resource computeUserLimit(FiCaSchedulerApp application, " clusterCapacity: " + clusterResource ); } - + user.setUserResourceLimit(limit); return limit; } @@ -1818,6 +1820,7 @@ public synchronized void updateClusterResource(Resource clusterResource, @VisibleForTesting public static class User { ResourceUsage userResourceUsage = new ResourceUsage(); + Resource userResourceLimit = Resource.newInstance(0, 0); int pendingApplications = 0; int activeApplications = 0; @@ -1887,6 +1890,14 @@ public void releaseContainer(Resource resource, Set nodeLabels) { } } } + + public Resource getUserResourceLimit() { + return userResourceLimit; + } + + public synchronized void setUserResourceLimit(Resource userResourceLimit) { + this.userResourceLimit = userResourceLimit; + } } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java index 65c911b..3725e21 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/UserInfo.java @@ -32,14 +32,19 @@ protected ResourceInfo resourcesUsed; protected int numPendingApplications; protected int numActiveApplications; + protected ResourceInfo AMResourceUsed; + protected ResourceInfo userResourceLimit; UserInfo() {} - UserInfo(String username, Resource resUsed, int activeApps, int pendingApps) { + UserInfo(String username, Resource resUsed, int activeApps, int pendingApps, + Resource amResUsed, Resource resourceLimit) { this.username = username; this.resourcesUsed = new ResourceInfo(resUsed); this.numActiveApplications = activeApps; this.numPendingApplications = pendingApps; + this.AMResourceUsed = new ResourceInfo(amResUsed); + this.userResourceLimit = new ResourceInfo(resourceLimit); } public String getUsername() { @@ -57,4 +62,12 @@ public int getNumPendingApplications() { public int getNumActiveApplications() { return numActiveApplications; } + + public ResourceInfo getAMResourcesUsed() { + return AMResourceUsed; + } + + public ResourceInfo getUserResourceLimit() { + return userResourceLimit; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java index 028bb31..c6c410c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java @@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.LI; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.UL; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; @@ -67,41 +69,8 @@ lqinfo = (CapacitySchedulerLeafQueueInfo) info.qinfo; } - //Return a string describing one resource as a percentage of another - private String getPercentage(ResourceInfo numerator, ResourceInfo denominator) { - StringBuilder percentString = new StringBuilder("Memory: "); - if (numerator != null) { - percentString.append(numerator.getMemory()); - } - if (denominator.getMemory() != 0) { - percentString.append(" (") - .append(StringUtils.format("%.2f", numerator.getMemory() * 100.0 / - denominator.getMemory()) + "%)"); - } - percentString.append(", vCores: "); - if (numerator != null) { - percentString.append(numerator.getvCores()); - } - if (denominator.getvCores() != 0) { - percentString.append(" (") - .append(StringUtils.format("%.2f", numerator.getvCores() * 100.0 / - denominator.getvCores()) + "%)"); - } - return percentString.toString(); - } - @Override protected void render(Block html) { - StringBuilder activeUserList = new StringBuilder(""); - ResourceInfo usedResources = lqinfo.getResourcesUsed(); - ArrayList users = lqinfo.getUsers().getUsersList(); - for (UserInfo entry: users) { - activeUserList.append(entry.getUsername()).append(" <") - .append(getPercentage(entry.getResourcesUsed(), usedResources)) - .append(", Schedulable Apps: " + entry.getNumActiveApplications()) - .append(", Non-Schedulable Apps: " + entry.getNumPendingApplications()) - .append(">
"); //Force line break - } ResponseInfo ri = info("\'" + lqinfo.getQueuePath().substring(5) + "\' Queue Status"). _("Queue State:", lqinfo.getQueueState()). @@ -116,12 +85,12 @@ protected void render(Block html) { _("Max Applications:", Integer.toString(lqinfo.getMaxApplications())). _("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())). _("Max Application Master Resources:", lqinfo.getAMResourceLimit().toString()). + _("Used Application Master Resources:", lqinfo.getUsedAMResource().toString()). _("Max Application Master Resources Per User:", lqinfo.getUserAMResourceLimit().toString()). _("Configured Capacity:", percent(lqinfo.getCapacity() / 100)). _("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)). _("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%"). _("Configured User Limit Factor:", String.format("%.1f", lqinfo.getUserLimitFactor())). - _r("Active Users: ", activeUserList.toString()). _("Accessible Node Labels:", StringUtils.join(",", lqinfo.getNodeLabels())). _("Preemption:", lqinfo.getPreemptionDisabled() ? "disabled" : "enabled"); @@ -132,6 +101,44 @@ protected void render(Block html) { } } + static class QueueUsersInfoBlock extends HtmlBlock { + final CapacitySchedulerLeafQueueInfo lqinfo; + + @Inject + QueueUsersInfoBlock(ViewContext ctx, CSQInfo info) { + super(ctx); + lqinfo = (CapacitySchedulerLeafQueueInfo) info.qinfo; + } + + @Override + protected void render(Block html) { + TBODY> tbody = + html.table("#userinfo").thead().$class("ui-widget-header").tr().th() + .$class("ui-state-default")._("User Name")._().th() + .$class("ui-state-default")._("Max Resource")._().th() + .$class("ui-state-default")._("Used Resource")._().th() + .$class("ui-state-default")._("Max AM Resource")._().th() + .$class("ui-state-default")._("Used AM Resource")._().th() + .$class("ui-state-default")._("Schedulable Apps")._().th() + .$class("ui-state-default")._("Non-Schedulable Apps")._()._()._() + .tbody(); + + ArrayList users = lqinfo.getUsers().getUsersList(); + for (UserInfo userInfo : users) { + tbody.tr().td(userInfo.getUsername()) + .td(userInfo.getUserResourceLimit().toString()) + .td(userInfo.getResourcesUsed().toString()) + .td(lqinfo.getUserAMResourceLimit().toString()) + .td(userInfo.getAMResourcesUsed().toString()) + .td(Integer.toString(userInfo.getNumActiveApplications())) + .td(Integer.toString(userInfo.getNumPendingApplications()))._(); + } + + html.div().$class("usersinfo").h5("Active Users Info")._(); + tbody._()._(); + } + } + public static class QueueBlock extends HtmlBlock { final CSQInfo csqinfo; @@ -166,6 +173,7 @@ public void render(Block html) { csqinfo.qinfo = info; if (info.getQueues() == null) { li.ul("#lq").li()._(LeafQueueInfoBlock.class)._()._(); + li.ul("#lq").li()._(QueueUsersInfoBlock.class)._()._(); } else { li._(QueueBlock.class); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java index 88efe47..6930975 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/MetricsOverviewTable.java @@ -21,6 +21,7 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerCommonInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; @@ -153,6 +154,31 @@ protected void render(Block html) { } } + + SchedulerCommonInfo schedulerInfo=new SchedulerCommonInfo(this.rm); + + div.h3("Scheduler Metrics"). + table("#schedulermetricsoverview"). + thead().$class("ui-widget-header"). + tr(). + th().$class("ui-state-default")._("Scheduler Type")._(). + th().$class("ui-state-default")._("Scheduling Resource Type")._(). + th().$class("ui-state-default")._("Minimum Allocation Memory")._(). + th().$class("ui-state-default")._("Maximum Allocation Memory")._(). + th().$class("ui-state-default")._("Minimum Allocation VCores")._(). + th().$class("ui-state-default")._("Maximum Allocation VCores")._(). + _(). + _(). + tbody().$class("ui-widget-content"). + tr(). + td(String.valueOf(schedulerInfo.getSchedulerType())). + td(String.valueOf(schedulerInfo.getSchedulerResourceTypes())). + td(StringUtils.byteDesc(schedulerInfo.getMinAllocationMemory()* BYTES_IN_MB)). + td(StringUtils.byteDesc(schedulerInfo.getMaxAllocationMemory()* BYTES_IN_MB)). + td(String.valueOf(schedulerInfo.getMinAllocationVirtualCores())). + td(String.valueOf(schedulerInfo.getMaxAllocationVirtualCores())). + _(). + _()._(); div._(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java index a8b0d32..5258b3d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java @@ -35,7 +35,8 @@ protected int userLimit; protected UsersInfo users; // To add another level in the XML protected float userLimitFactor; - protected ResourceInfo aMResourceLimit; + protected ResourceInfo AMResourceLimit; + protected ResourceInfo usedAMResource; protected ResourceInfo userAMResourceLimit; protected boolean preemptionDisabled; @@ -52,7 +53,8 @@ userLimit = q.getUserLimit(); users = new UsersInfo(q.getUsers()); userLimitFactor = q.getUserLimitFactor(); - aMResourceLimit = new ResourceInfo(q.getAMResourceLimit()); + AMResourceLimit = new ResourceInfo(q.getAMResourceLimit()); + usedAMResource = new ResourceInfo(q.getQueueResourceUsage().getAMUsed()); userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit()); preemptionDisabled = q.getPreemptionDisabled(); } @@ -91,9 +93,13 @@ public float getUserLimitFactor() { } public ResourceInfo getAMResourceLimit() { - return aMResourceLimit; + return AMResourceLimit; } + public ResourceInfo getUsedAMResource() { + return usedAMResource; + } + public ResourceInfo getUserAMResourceLimit() { return userAMResourceLimit; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerCommonInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerCommonInfo.java new file mode 100644 index 0000000..cc3f456 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerCommonInfo.java @@ -0,0 +1,90 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.EnumSet; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; +import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; + +@XmlRootElement(name = "SchedulerMetrics") +@XmlAccessorType(XmlAccessType.FIELD) +public class SchedulerCommonInfo { + + protected long minAllocMemory; + protected long maxAllocMemory; + protected long minAllocVirtualCores; + protected long maxAllocVirtualCores; + protected String schedulerName; + protected EnumSet schedulingResourceTypes; + + public SchedulerCommonInfo() { + } // JAXB needs this + + public SchedulerCommonInfo(final ResourceManager rm) { + ResourceScheduler rs = rm.getResourceScheduler(); + + if (rs instanceof CapacityScheduler) { + this.schedulerName = "Capacity Scheduler"; + } else if (rs instanceof FairScheduler) { + this.schedulerName = "Fair Scheduler"; + } else if (rs instanceof FifoScheduler) { + this.schedulerName = "Fifo Scheduler"; + } + this.minAllocMemory = rs.getMinimumResourceCapability().getMemory(); + this.maxAllocMemory = rs.getMaximumResourceCapability().getMemory(); + this.minAllocVirtualCores = + rs.getMinimumResourceCapability().getVirtualCores(); + this.maxAllocVirtualCores = + rs.getMaximumResourceCapability().getVirtualCores(); + this.schedulingResourceTypes = rs.getSchedulingResourceTypes(); + } + + public String getSchedulerType() { + return this.schedulerName; + } + + public long getMinAllocationMemory() { + return this.minAllocMemory; + } + + public long getMaxAllocationMemory() { + return this.maxAllocMemory; + } + + public long getMinAllocationVirtualCores() { + return this.minAllocVirtualCores; + } + + public long getMaxAllocationVirtualCores() { + return this.maxAllocVirtualCores; + } + + public String getSchedulerResourceTypes() { + return this.schedulingResourceTypes.toString(); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java index ba5c73b..eb42679 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java @@ -347,7 +347,7 @@ private void verifySubQueue(JSONObject info, String q, int numExpectedElements = 13; boolean isParentQueue = true; if (!info.has("queues")) { - numExpectedElements = 24; + numExpectedElements = 25; isParentQueue = false; } assertEquals("incorrect number of elements", numExpectedElements, info.length()); -- 1.9.2.msysgit.0