diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java index 0c8491f..27fb5ae 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceRequestPBImpl.java @@ -140,13 +140,13 @@ public void setCapability(Resource capability) { this.capability = capability; } @Override - public int getNumContainers() { + public synchronized int getNumContainers() { ResourceRequestProtoOrBuilder p = viaProto ? proto : builder; return (p.getNumContainers()); } @Override - public void setNumContainers(int numContainers) { + public synchronized void setNumContainers(int numContainers) { maybeInitBuilder(); builder.setNumContainers((numContainers)); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index a9a459f..97dc231 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -20,12 +20,14 @@ import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; import java.util.TreeSet; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; @@ -64,7 +66,7 @@ final Set priorities = new TreeSet( new org.apache.hadoop.yarn.server.resourcemanager.resource.Priority.Comparator()); final Map> requests = - new HashMap>(); + new ConcurrentHashMap>(); private Set blacklist = new HashSet(); //private final ApplicationStore store; @@ -159,7 +161,7 @@ synchronized public void updateResourceRequests( Map asks = this.requests.get(priority); if (asks == null) { - asks = new HashMap(); + asks = new ConcurrentHashMap(); this.requests.put(priority, asks); this.priorities.add(priority); } @@ -221,7 +223,7 @@ synchronized public void updateBlacklist( return requests.get(priority); } - synchronized public List getAllResourceRequests() { + public List getAllResourceRequests() { List ret = new ArrayList(); for (Map r : requests.values()) { ret.addAll(r.values()); @@ -300,17 +302,11 @@ synchronized private void allocateNodeLocal(SchedulerNode node, Priority priority, ResourceRequest nodeLocalRequest, Container container, List resourceRequests) { // Update future requirements - nodeLocalRequest.setNumContainers(nodeLocalRequest.getNumContainers() - 1); - if (nodeLocalRequest.getNumContainers() == 0) { - this.requests.get(priority).remove(node.getNodeName()); - } + decResourceRequest(node.getNodeName(), priority, nodeLocalRequest); ResourceRequest rackLocalRequest = requests.get(priority).get( node.getRackName()); - rackLocalRequest.setNumContainers(rackLocalRequest.getNumContainers() - 1); - if (rackLocalRequest.getNumContainers() == 0) { - this.requests.get(priority).remove(node.getRackName()); - } + decResourceRequest(node.getRackName(), priority, rackLocalRequest); ResourceRequest offRackRequest = requests.get(priority).get( ResourceRequest.ANY); @@ -322,6 +318,14 @@ synchronized private void allocateNodeLocal(SchedulerNode node, resourceRequests.add(cloneResourceRequest(offRackRequest)); } + private void decResourceRequest(String resourceName, Priority priority, + ResourceRequest request) { + request.setNumContainers(request.getNumContainers() - 1); + if (request.getNumContainers() == 0) { + requests.get(priority).remove(resourceName); + } + } + /** * The {@link ResourceScheduler} is allocating data-local resources to the * application. @@ -333,11 +337,8 @@ synchronized private void allocateRackLocal(SchedulerNode node, Priority priority, ResourceRequest rackLocalRequest, Container container, List resourceRequests) { // Update future requirements - rackLocalRequest.setNumContainers(rackLocalRequest.getNumContainers() - 1); - if (rackLocalRequest.getNumContainers() == 0) { - this.requests.get(priority).remove(node.getRackName()); - } - + decResourceRequest(node.getRackName(), priority, rackLocalRequest); + ResourceRequest offRackRequest = requests.get(priority).get( ResourceRequest.ANY); decrementOutstanding(offRackRequest); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java index d5b6ce6..532df05 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java @@ -153,7 +153,11 @@ public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId, public synchronized Collection getLiveContainers() { return new ArrayList(liveContainers.values()); } - + + public AppSchedulingInfo getAppSchedulingInfo() { + return this.appSchedulingInfo; + } + /** * Is this application pending? * @return true if it is else false. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java index c2b376e..a19b26b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppBlock.java @@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus; import org.apache.hadoop.yarn.api.records.QueueACL; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceRequest; import org.apache.hadoop.yarn.api.records.YarnApplicationState; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; @@ -50,6 +51,7 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV; import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; import org.apache.hadoop.yarn.webapp.util.WebAppUtils; import org.apache.hadoop.yarn.webapp.view.HtmlBlock; import org.apache.hadoop.yarn.webapp.view.InfoBlock; @@ -90,7 +92,8 @@ protected void render(Block html) { puts("Application not found: "+ aid); return; } - AppInfo app = new AppInfo(rmApp, true, WebAppUtils.getHttpSchemePrefix(conf)); + AppInfo app = + new AppInfo(rm, rmApp, true, WebAppUtils.getHttpSchemePrefix(conf)); // Check for the authorization. String remoteUser = request().getRemoteUser(); @@ -134,7 +137,7 @@ protected void render(Block html) { ._("Application Type:", app.getApplicationType()) ._("Application Tags:", app.getApplicationTags()) ._("YarnApplicationState:", clarifyAppState(app.getState())) - ._("FinalStatus reported by AM:", + ._("FinalStatus Reported by AM:", clairfyAppFinalStatus(app.getFinalStatus())) ._("Started:", Times.format(app.getStartTime())) ._("Elapsed:", @@ -200,6 +203,34 @@ protected void render(Block html) { table._(); div._(); + + createResourceRequestsTable(html, app); + } + + private void createResourceRequestsTable(Block html, AppInfo app) { + html.div().$class("ResourceRequests").h3("Outstanding Resource Requests")._(); + TBODY> tbody = + html.table("#ResourceRequests").thead().tr() + .th(".priority", "Priority") + .th(".resourceName", "ResourceName") + .th(".totalResource", "Capability") + .th(".numContainers", "NumContainers") + .th(".relaxLocality", "RelaxLocality") + .th(".labelExpression", "LabelExpression")._()._().tbody(); + + if (app.getResourceRequests() != null) { + for (ResourceRequest request : app.getResourceRequests()) { + tbody.tr() + .td(String.valueOf(request.getPriority())) + .td(request.getResourceName()) + .td(String.valueOf(request.getCapability())) + .td(String.valueOf(request.getNumContainers())) + .td(String.valueOf(request.getRelaxLocality())) + .td(request.getNodeLabelExpression() == null ? "N/A" : request + .getNodeLabelExpression())._(); + } + } + tbody._()._(); } private String clarifyAppState(YarnApplicationState state) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java index a55c62f..8993324 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java @@ -18,12 +18,16 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp; +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; + import org.apache.hadoop.yarn.webapp.SubView; public class AppPage extends RmView { @Override protected void preHead(Page.HTML<_> html) { commonPreHead(html); + set(DATATABLES_ID, "ResourceRequests"); + setTableStyles(html, "ResourceRequests"); } @Override protected Class content() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java index 054a1a7..935be61 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlock.java @@ -46,12 +46,13 @@ class AppsBlock extends HtmlBlock { final ConcurrentMap apps; private final Configuration conf; - + final ResourceManager rm; @Inject AppsBlock(ResourceManager rm, ViewContext ctx, Configuration conf) { super(ctx); apps = rm.getRMContext().getRMApps(); this.conf = conf; + this.rm = rm; } @Override public void render(Block html) { @@ -85,7 +86,7 @@ if (reqAppStates != null && !reqAppStates.contains(app.createApplicationState())) { continue; } - AppInfo appInfo = new AppInfo(app, true, WebAppUtils.getHttpSchemePrefix(conf)); + AppInfo appInfo = new AppInfo(rm, app, true, WebAppUtils.getHttpSchemePrefix(conf)); String percent = String.format("%.1f", appInfo.getProgress()); //AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js appsTableData.append("[\"