diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java index 070aa1f..e888cc5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java @@ -72,8 +72,8 @@ protected Dispatcher dispatcher; - protected ConcurrentMap labelCollections = - new ConcurrentHashMap(); + protected ConcurrentMap labelCollections = + new ConcurrentHashMap(); protected ConcurrentMap nodeCollections = new ConcurrentHashMap(); @@ -82,19 +82,6 @@ protected NodeLabelsStore store; - protected static class Label { - private Resource resource; - - protected Label() { - this.resource = Resource.newInstance(0, 0); - } - - public Resource getResource() { - return this.resource; - } - - } - /** * A Host can have multiple Nodes */ @@ -201,7 +188,7 @@ protected void initDispatcher(Configuration conf) { protected void serviceInit(Configuration conf) throws Exception { initNodeLabelStore(conf); - labelCollections.put(NO_LABEL, new Label()); + labelCollections.put(NO_LABEL, new NodeLabel(NO_LABEL)); } protected void initNodeLabelStore(Configuration conf) throws Exception { @@ -271,7 +258,7 @@ public void addToCluserNodeLabels(Set labels) throws IOException { for (String label : labels) { // shouldn't overwrite it to avoid changing the Label.resource if (this.labelCollections.get(label) == null) { - this.labelCollections.put(label, new Label()); + this.labelCollections.put(label, new NodeLabel(label)); newLabels.add(label); } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java new file mode 100644 index 0000000..2d06f8a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabel.java @@ -0,0 +1,59 @@ +package org.apache.hadoop.yarn.nodelabels; + +import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.util.resource.Resources; + +public class NodeLabel implements Comparable { + private Resource resource; + private int numActiveNMs; + private String labelName; + + public NodeLabel(String labelName) { + this(labelName, Resource.newInstance(0, 0), 0); + } + + protected NodeLabel(String labelName, Resource res, int activeNMs) { + this.labelName = labelName; + this.resource = res; + this.numActiveNMs = activeNMs; + } + + public void addNode(Resource nodeRes) { + Resources.addTo(resource, nodeRes); + numActiveNMs++; + } + + public void removeNode(Resource nodeRes) { + Resources.subtractFrom(resource, nodeRes); + numActiveNMs--; + } + + public Resource getResource() { + return this.resource; + } + + public int getNumActiveNMs() { + return numActiveNMs; + } + + public String getLabelName() { + return labelName; + } + + public NodeLabel getCopy() { + return new NodeLabel(labelName, resource, numActiveNMs); + } + + @Override + public int compareTo(NodeLabel o) { + // We should always put empty label entry first after sorting + if (labelName.isEmpty() != o.getLabelName().isEmpty()) { + if (labelName.isEmpty()) { + return -1; + } + return 1; + } + + return labelName.compareTo(o.getLabelName()); + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java index 91d2a20..62c3c7a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java @@ -32,4 +32,5 @@ String APP_STATE = "app.state"; String QUEUE_NAME = "queue.name"; String NODE_STATE = "node.state"; + String NODE_LABEL = "node.label"; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java index 646441a..828d1bc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java @@ -19,10 +19,12 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels; import java.io.IOException; +import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -37,6 +39,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.apache.hadoop.yarn.nodelabels.NodeLabel; import org.apache.hadoop.yarn.server.resourcemanager.RMContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent; import org.apache.hadoop.yarn.util.resource.Resources; @@ -360,8 +363,8 @@ private void updateResourceMappings(Map before, // no label in the past if (oldLabels.isEmpty()) { // update labels - Label label = labelCollections.get(NO_LABEL); - Resources.subtractFrom(label.getResource(), oldNM.resource); + NodeLabel label = labelCollections.get(NO_LABEL); + label.removeNode(oldNM.resource); // update queues, all queue can access this node for (Queue q : queueCollections.values()) { @@ -370,11 +373,11 @@ private void updateResourceMappings(Map before, } else { // update labels for (String labelName : oldLabels) { - Label label = labelCollections.get(labelName); + NodeLabel label = labelCollections.get(labelName); if (null == label) { continue; } - Resources.subtractFrom(label.getResource(), oldNM.resource); + label.removeNode(oldNM.resource); } // update queues, only queue can access this node will be subtract @@ -395,8 +398,8 @@ private void updateResourceMappings(Map before, // no label in the past if (newLabels.isEmpty()) { // update labels - Label label = labelCollections.get(NO_LABEL); - Resources.addTo(label.getResource(), newNM.resource); + NodeLabel label = labelCollections.get(NO_LABEL); + label.addNode(newNM.resource); // update queues, all queue can access this node for (Queue q : queueCollections.values()) { @@ -405,8 +408,8 @@ private void updateResourceMappings(Map before, } else { // update labels for (String labelName : newLabels) { - Label label = labelCollections.get(labelName); - Resources.addTo(label.getResource(), newNM.resource); + NodeLabel label = labelCollections.get(labelName); + label.addNode(newNM.resource); } // update queues, only queue can access this node will be subtract @@ -475,4 +478,21 @@ public boolean checkAccess(UserGroupInformation user) { public void setRMContext(RMContext rmContext) { this.rmContext = rmContext; } + + public List pullRMNodeLabelsInfo() { + try { + readLock.lock(); + List infos = new ArrayList(); + + for (Entry entry : labelCollections.entrySet()) { + NodeLabel label = entry.getValue(); + infos.add(label.getCopy()); + } + + Collections.sort(infos); + return infos; + } finally { + readLock.unlock(); + } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java index ce8fd9e..db00bb0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NavBlock.java @@ -33,7 +33,8 @@ h3("Cluster"). ul(). li().a(url("cluster"), "About")._(). - li().a(url("nodes"), "Nodes")._(); + li().a(url("nodes"), "Nodes")._(). + li().a(url("nodelabels"), "Node Labels")._(); UL>>> subAppsList = mainList. li().a(url("apps"), "Applications"). ul(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java new file mode 100644 index 0000000..5e8c1ed --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodeLabelsPage.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp; + +import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; + +import org.apache.hadoop.yarn.nodelabels.NodeLabel; +import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; +import org.apache.hadoop.yarn.webapp.SubView; +import org.apache.hadoop.yarn.webapp.YarnWebParams; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY; +import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TR; +import org.apache.hadoop.yarn.webapp.view.HtmlBlock; + +import com.google.inject.Inject; + +public class NodeLabelsPage extends RmView { + static class NodeLabelsBlock extends HtmlBlock { + final ResourceManager rm; + + @Inject + NodeLabelsBlock(ResourceManager rm, ViewContext ctx) { + super(ctx); + this.rm = rm; + } + + @Override + protected void render(Block html) { + TBODY> tbody = html.table("#nodelabels"). + thead(). + tr(). + th(".name", "Label Name"). + th(".numOfActiveNMs", "Num Of Active NMs"). + th(".totalResource", "Total Resource"). + _()._(). + tbody(); + + RMNodeLabelsManager nlm = rm.getRMContext().getNodeLabelManager(); + for (NodeLabel info : nlm.pullRMNodeLabelsInfo()) { + TR>> row = + tbody.tr().td( + info.getLabelName().isEmpty() ? "" : info + .getLabelName()); + int nActiveNMs = info.getNumActiveNMs(); + if (nActiveNMs > 0) { + row = row.td() + .a(url("nodes", + "?" + YarnWebParams.NODE_LABEL + "=" + info.getLabelName()), + String.valueOf(nActiveNMs)) + ._(); + } else { + row = row.td(String.valueOf(nActiveNMs)); + } + row.td(info.getResource().toString())._(); + } + tbody._()._(); + } + } + + @Override protected void preHead(Page.HTML<_> html) { + commonPreHead(html); + String title = "Node labels of the cluster"; + setTitle(title); + set(DATATABLES_ID, "nodelabels"); + setTableStyles(html, "nodelabels", ".healthStatus {width:10em}", + ".healthReport {width:10em}"); + } + + @Override protected Class content() { + return NodeLabelsBlock.class; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java index d3849ae..f28a9a8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/NodesPage.java @@ -1,24 +1,25 @@ /** -* Licensed to the Apache Software Foundation (ASF) under one -* or more contributor license agreements. See the NOTICE file -* distributed with this work for additional information -* regarding copyright ownership. The ASF licenses this file -* to you under the Apache License, Version 2.0 (the -* "License"); you may not use this file except in compliance -* with the License. You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hadoop.yarn.server.resourcemanager.webapp; -import static org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp.NODE_STATE; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_STATE; +import static org.apache.hadoop.yarn.webapp.YarnWebParams.NODE_LABEL; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID; import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID; @@ -28,7 +29,9 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.NodeState; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager; +import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; @@ -60,26 +63,20 @@ protected void render(Block html) { ResourceScheduler sched = rm.getResourceScheduler(); String type = $(NODE_STATE); - TBODY> tbody = html.table("#nodes"). - thead(). - tr(). - th(".nodelabels", "Node Labels"). - th(".rack", "Rack"). - th(".state", "Node State"). - th(".nodeaddress", "Node Address"). - th(".nodehttpaddress", "Node HTTP Address"). - th(".lastHealthUpdate", "Last health-update"). - th(".healthReport", "Health-report"). - th(".containers", "Containers"). - th(".mem", "Mem Used"). - th(".mem", "Mem Avail"). - th(".vcores", "VCores Used"). - th(".vcores", "VCores Avail"). - th(".nodeManagerVersion", "Version"). - _()._(). - tbody(); + String labelFilter = $(NODE_LABEL, CommonNodeLabelsManager.ANY).trim(); + TBODY> tbody = + html.table("#nodes").thead().tr().th(".nodelabels", "Node Labels") + .th(".rack", "Rack").th(".state", "Node State") + .th(".nodeaddress", "Node Address") + .th(".nodehttpaddress", "Node HTTP Address") + .th(".lastHealthUpdate", "Last health-update") + .th(".healthReport", "Health-report") + .th(".containers", "Containers").th(".mem", "Mem Used") + .th(".mem", "Mem Avail").th(".vcores", "VCores Used") + .th(".vcores", "VCores Avail") + .th(".nodeManagerVersion", "Version")._()._().tbody(); NodeState stateFilter = null; - if(type != null && !type.isEmpty()) { + if (type != null && !type.isEmpty()) { stateFilter = NodeState.valueOf(type.toUpperCase()); } Collection rmNodes = this.rm.getRMContext().getRMNodes().values(); @@ -97,9 +94,9 @@ protected void render(Block html) { } } for (RMNode ni : rmNodes) { - if(stateFilter != null) { + if (stateFilter != null) { NodeState state = ni.getState(); - if(!stateFilter.equals(state)) { + if (!stateFilter.equals(state)) { continue; } } else { @@ -109,61 +106,71 @@ protected void render(Block html) { continue; } } + // Besides state, we need to filter label as well. + if (!labelFilter.equals(RMNodeLabelsManager.ANY)) { + if (labelFilter.isEmpty()) { + // Empty label filter means only shows nodes without label + if (!ni.getNodeLabels().isEmpty()) { + continue; + } + } else if (!ni.getNodeLabels().contains(labelFilter)) { + // Only nodes have given label can show on web page. + continue; + } + } NodeInfo info = new NodeInfo(ni, sched); - int usedMemory = (int)info.getUsedMemory(); - int availableMemory = (int)info.getAvailableMemory(); - TR>> row = tbody.tr(). - td(StringUtils.join(",", info.getNodeLabels())). - td(info.getRack()). - td(info.getState()). - td(info.getNodeId()); + int usedMemory = (int) info.getUsedMemory(); + int availableMemory = (int) info.getAvailableMemory(); + TR>> row = + tbody.tr().td(StringUtils.join(",", info.getNodeLabels())) + .td(info.getRack()).td(info.getState()).td(info.getNodeId()); if (isInactive) { row.td()._("N/A")._(); } else { String httpAddress = info.getNodeHTTPAddress(); - row.td().a("//" + httpAddress, - httpAddress)._(); + row.td().a("//" + httpAddress, httpAddress)._(); } - row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._(). - _(Times.format(info.getLastHealthUpdate()))._(). - td(info.getHealthReport()). - td(String.valueOf(info.getNumContainers())). - td().br().$title(String.valueOf(usedMemory))._(). - _(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._(). - td().br().$title(String.valueOf(availableMemory))._(). - _(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._(). - td(String.valueOf(info.getUsedVirtualCores())). - td(String.valueOf(info.getAvailableVirtualCores())). - td(ni.getNodeManagerVersion()). - _(); + row.td().br().$title(String.valueOf(info.getLastHealthUpdate()))._() + ._(Times.format(info.getLastHealthUpdate()))._() + .td(info.getHealthReport()) + .td(String.valueOf(info.getNumContainers())).td().br() + .$title(String.valueOf(usedMemory))._() + ._(StringUtils.byteDesc(usedMemory * BYTES_IN_MB))._().td().br() + .$title(String.valueOf(availableMemory))._() + ._(StringUtils.byteDesc(availableMemory * BYTES_IN_MB))._() + .td(String.valueOf(info.getUsedVirtualCores())) + .td(String.valueOf(info.getAvailableVirtualCores())) + .td(ni.getNodeManagerVersion())._(); } tbody._()._(); } } - @Override protected void preHead(Page.HTML<_> html) { + @Override + protected void preHead(Page.HTML<_> html) { commonPreHead(html); String type = $(NODE_STATE); String title = "Nodes of the cluster"; - if(type != null && !type.isEmpty()) { - title = title+" ("+type+")"; + if (type != null && !type.isEmpty()) { + title = title + " (" + type + ")"; } setTitle(title); set(DATATABLES_ID, "nodes"); set(initID(DATATABLES, "nodes"), nodesTableInit()); setTableStyles(html, "nodes", ".healthStatus {width:10em}", - ".healthReport {width:10em}"); + ".healthReport {width:10em}"); } - @Override protected Class content() { + @Override + protected Class content() { return NodesBlock.class; } private String nodesTableInit() { StringBuilder b = tableInit().append(", aoColumnDefs: ["); b.append("{'bSearchable': false, 'aTargets': [ 6 ]}"); - b.append(", {'sType': 'title-numeric', 'bSearchable': false, " + - "'aTargets': [ 7, 8 ] }"); + b.append(", {'sType': 'title-numeric', 'bSearchable': false, " + + "'aTargets': [ 7, 8 ] }"); b.append(", {'sType': 'title-numeric', 'aTargets': [ 4 ]}"); b.append("]}"); return b.toString(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java index 67c73b8..c0e6834 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java @@ -61,6 +61,7 @@ public void setup() { route(pajoin("/app", APPLICATION_ID), RmController.class, "app"); route("/scheduler", RmController.class, "scheduler"); route(pajoin("/queue", QUEUE_NAME), RmController.class, "queue"); + route("/nodelabels", RmController.class, "nodelabels"); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java index f186bf4..972432b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RmController.java @@ -28,7 +28,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler; import org.apache.hadoop.yarn.util.StringHelper; import org.apache.hadoop.yarn.webapp.Controller; -import org.apache.hadoop.yarn.webapp.WebAppException; import org.apache.hadoop.yarn.webapp.YarnWebParams; import com.google.inject.Inject; @@ -93,4 +92,9 @@ public void queue() { public void submit() { setTitle("Application Submission Not Allowed"); } + + public void nodelabels() { + setTitle("Node Labels"); + render(NodeLabelsPage.class); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java index 278c151..2d863d1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java @@ -30,11 +30,13 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; +import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse; import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode; import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; /** @@ -53,7 +55,12 @@ // One unhealthy node per rack. list.add(nodeInfo(i, perNode, NodeState.UNHEALTHY)); } - list.add(newNodeInfo(i, perNode)); + if (j == 0) { + // One node with label + list.add(nodeInfo(i, perNode, NodeState.RUNNING, ImmutableSet.of("x"))); + } else { + list.add(newNodeInfo(i, perNode)); + } } } return list; @@ -100,10 +107,12 @@ public static Resource newAvailResource(Resource total, Resource used) { private String healthReport; private long lastHealthReportTime; private NodeState state; + private Set labels; public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, Resource perNode, String rackName, String healthReport, - long lastHealthReportTime, int cmdPort, String hostName, NodeState state) { + long lastHealthReportTime, int cmdPort, String hostName, NodeState state, + Set labels) { this.nodeId = nodeId; this.nodeAddr = nodeAddr; this.httpAddress = httpAddress; @@ -114,6 +123,7 @@ public MockRMNodeImpl(NodeId nodeId, String nodeAddr, String httpAddress, this.cmdPort = cmdPort; this.hostName = hostName; this.state = state; + this.labels = labels; } @Override @@ -207,16 +217,33 @@ public long getLastHealthReportTime() { @Override public Set getNodeLabels() { - return RMNodeLabelsManager.EMPTY_STRING_SET; + if (labels != null) { + return labels; + } + return CommonNodeLabelsManager.EMPTY_STRING_SET; } }; - private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr) { - return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123); + private static RMNode buildRMNode(int rack, final Resource perNode, + NodeState state, String httpAddr) { + return buildRMNode(rack, perNode, state, httpAddr, null); } - + + private static RMNode buildRMNode(int rack, final Resource perNode, + NodeState state, String httpAddr, Set labels) { + return buildRMNode(rack, perNode, state, httpAddr, NODE_ID++, null, 123, + labels); + } + private static RMNode buildRMNode(int rack, final Resource perNode, NodeState state, String httpAddr, int hostnum, String hostName, int port) { + return buildRMNode(rack, perNode, state, httpAddr, hostnum, hostName, port, + null); + } + + private static RMNode buildRMNode(int rack, final Resource perNode, + NodeState state, String httpAddr, int hostnum, String hostName, int port, + Set labels) { final String rackName = "rack"+ rack; final int nid = hostnum; final String nodeAddr = hostName + ":" + nid; @@ -228,13 +255,18 @@ private static RMNode buildRMNode(int rack, final Resource perNode, final String httpAddress = httpAddr; String healthReport = (state == NodeState.UNHEALTHY) ? null : "HealthyMe"; return new MockRMNodeImpl(nodeID, nodeAddr, httpAddress, perNode, - rackName, healthReport, 0, nid, hostName, state); + rackName, healthReport, 0, nid, hostName, state, labels); } public static RMNode nodeInfo(int rack, final Resource perNode, NodeState state) { return buildRMNode(rack, perNode, state, "N/A"); } + + public static RMNode nodeInfo(int rack, final Resource perNode, + NodeState state, Set labels) { + return buildRMNode(rack, perNode, state, "N/A", labels); + } public static RMNode newNodeInfo(int rack, final Resource perNode) { return buildRMNode(rack, perNode, NodeState.RUNNING, "localhost:0"); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java index ed675f3..e4cdc71 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Set; @@ -27,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.NodeId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager; +import org.apache.hadoop.yarn.nodelabels.NodeLabel; import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase; import org.apache.hadoop.yarn.util.resource.Resources; import org.junit.After; @@ -428,4 +430,35 @@ public void testRemoveLabelsFromNode() throws Exception { Assert.fail("IOException from removeLabelsFromNode " + e); } } + + private void checkNodeLabelInfo(List infos, String labelName, int activeNMs, int memory) { + for (NodeLabel info : infos) { + if (info.getLabelName().equals(labelName)) { + Assert.assertEquals(activeNMs, info.getNumActiveNMs()); + Assert.assertEquals(memory, info.getResource().getMemory()); + return; + } + } + Assert.fail("Failed to find info has label=" + labelName); + } + + @Test(timeout = 5000) + public void testPullRMNodeLabelsInfo() throws IOException { + mgr.addToCluserNodeLabels(toSet("x", "y", "z")); + mgr.activateNode(NodeId.newInstance("n1", 1), Resource.newInstance(10, 0)); + mgr.activateNode(NodeId.newInstance("n2", 1), Resource.newInstance(10, 0)); + mgr.activateNode(NodeId.newInstance("n3", 1), Resource.newInstance(10, 0)); + mgr.activateNode(NodeId.newInstance("n4", 1), Resource.newInstance(10, 0)); + mgr.activateNode(NodeId.newInstance("n5", 1), Resource.newInstance(10, 0)); + mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("x"), + toNodeId("n2"), toSet("x"), toNodeId("n3"), toSet("y"))); + + // x, y, z and "" + List infos = mgr.pullRMNodeLabelsInfo(); + Assert.assertEquals(4, infos.size()); + checkNodeLabelInfo(infos, RMNodeLabelsManager.NO_LABEL, 2, 20); + checkNodeLabelInfo(infos, "x", 2, 20); + checkNodeLabelInfo(infos, "y", 1, 10); + checkNodeLabelInfo(infos, "z", 0, 0); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java index bb38079..62713cf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestNodesPage.java @@ -106,4 +106,49 @@ public void testNodesBlockRenderForLostNodes() { * numberOfActualTableHeaders + numberOfThInMetricsTable)).print( "