From 5b29bd9363a2679421b71b75df7c545873136d30 Mon Sep 17 00:00:00 2001 From: Sunil Date: Wed, 5 Oct 2016 23:03:28 +0530 Subject: [PATCH] YARN-5256 --- .../webapp/JAXBContextResolver.java | 2 +- .../resourcemanager/webapp/RMWebServices.java | 40 +++++++++++++ .../webapp/dao/PartitionStatisticsInfo.java | 67 ++++++++++++++++++++++ .../webapp/dao/PartitionsStatisticsInfo.java | 54 +++++++++++++++++ .../webapp/TestRMWebServicesNodeLabels.java | 58 +++++++++++++++++++ 5 files changed, 220 insertions(+), 1 deletion(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionStatisticsInfo.java create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionsStatisticsInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index b61072d..4fbfdc9 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -54,7 +54,7 @@ public JAXBContextResolver() throws Exception { CapacitySchedulerQueueInfoList.class, ResourceInfo.class, UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, - FairSchedulerQueueInfoList.class}; + FairSchedulerQueueInfoList.class, PartitionStatisticsInfo.class}; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java index 99440a8..206a760 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java @@ -1198,6 +1198,46 @@ public KillApplicationResponse run() throws IOException, } @GET + @Path("/partition-stats-info") + @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) + public PartitionsStatisticsInfo getPartitionStats( + @QueryParam("partionLabels") Set labels, + @Context HttpServletRequest hsr) throws IOException, YarnException { + init(); + + PartitionsStatisticsInfo partitionStats = new PartitionsStatisticsInfo(); + Map> labelsToNodeId = null; + + if (labels == null || labels.size() == 0) { + // we can return partition based stats for all cluster labels. + labels = rm.getRMContext().getNodeLabelManager() + .getClusterNodeLabelNames(); + labelsToNodeId = rm.getRMContext().getNodeLabelManager() + .getLabelsInfoToNodes(); + } else { + labelsToNodeId = rm.getRMContext().getNodeLabelManager() + .getLabelsInfoToNodes(labels); + } + + for (String label : labels) { + Set nodes = null; + ArrayList nodeList = new ArrayList(); + if ((nodes = labelsToNodeId.get(NodeLabel.newInstance(label))) != null) { + for (NodeId nodeId : nodes) { + nodeList.add(nodeId.toString()); + } + } + + Resource labelResource = rm.getRMContext().getNodeLabelManager() + .getResourceByLabel(label, null); + partitionStats.addToPartitionStatsInfo( + new PartitionStatisticsInfo(label, nodeList, labelResource)); + } + + return partitionStats; + } + + @GET @Path("/apps/{appid}/priority") @Produces({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML }) public AppPriority getAppPriority(@Context HttpServletRequest hsr, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionStatisticsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionStatisticsInfo.java new file mode 100644 index 0000000..dcaf23a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionStatisticsInfo.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.Collections; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.Resource; + +@XmlRootElement(name = "partitionStats") +@XmlAccessorType(XmlAccessType.FIELD) +public class PartitionStatisticsInfo { + + protected String labelName; + protected ArrayList nodeList = new ArrayList(); + protected long totalResourceMem; + protected long totalResourceCores; + + public PartitionStatisticsInfo() { + // JAXB needs this + } + + public PartitionStatisticsInfo(String labelName, ArrayList nodesList, + Resource partitionResource) { + this.labelName = labelName; + this.nodeList.addAll(nodesList); + Collections.sort(this.nodeList); + this.totalResourceMem = partitionResource.getMemorySize(); + this.totalResourceCores = partitionResource.getVirtualCores(); + } + + public ArrayList getNodeList() { + return nodeList; + } + + public long getTotalResourceMem() { + return totalResourceMem; + } + + public long getTotalResourceCores() { + return totalResourceCores; + } + + public String getLabelName() { + return labelName; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionsStatisticsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionsStatisticsInfo.java new file mode 100644 index 0000000..ea11882 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/PartitionsStatisticsInfo.java @@ -0,0 +1,54 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + + +@XmlRootElement(name = "partitionsStatsInfo") +@XmlAccessorType(XmlAccessType.FIELD) +public class PartitionsStatisticsInfo { + + @XmlElement(name = "partitionStatsInfo") + private ArrayList partitionStatsInfo = + new ArrayList(); + + public PartitionsStatisticsInfo() { + // JAXB needs this + } + + public PartitionsStatisticsInfo(ArrayList partitionStatsInfo) { + this.partitionStatsInfo = partitionStatsInfo; + } + + public ArrayList getPartitionStatsInfo() { + return partitionStatsInfo; + } + + public void addToPartitionStatsInfo( + PartitionStatisticsInfo partitionStatsInfo) { + this.partitionStatsInfo.add(partitionStatsInfo); + } + +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java index 0cfaac1..f4ae2e6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java @@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntry; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo; +import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.PartitionsStatisticsInfo; import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList; import org.apache.hadoop.yarn.webapp.GenericExceptionHandler; import org.apache.hadoop.yarn.webapp.GuiceServletConfig; @@ -671,6 +672,63 @@ public void testLabelInvalidRemove() validateJsonExceptionContent(response, expectedmessage); } + @Test + public void testSingleNodeLabelStatsDefault() throws JSONException, Exception { + testSingleNodeLabelStats("a", MediaType.APPLICATION_JSON); + } + + public void testSingleNodeLabelStats(String label, String media) + throws JSONException, Exception { + WebResource r = resource(); + ClientResponse response; + // Add a label + NodeLabelsInfo nlsifo = new NodeLabelsInfo(); + nlsifo.getNodeLabelsInfo().add(new NodeLabelInfo("a")); + response = r.path("ws").path("v1").path("cluster").path("add-node-labels") + .queryParam("user.name", userName).accept(MediaType.APPLICATION_JSON) + .entity(toJson(nlsifo, NodeLabelsInfo.class), + MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + + // Verify + response = r.path("ws").path("v1").path("cluster").path("get-node-labels") + .queryParam("user.name", userName).accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + nlsifo = response.getEntity(NodeLabelsInfo.class); + assertEquals(1, nlsifo.getNodeLabels().size()); + for (NodeLabelInfo nl : nlsifo.getNodeLabelsInfo()) { + assertEquals("a", nl.getName()); + assertTrue(nl.getExclusivity()); + } + + // Add labels to a node + MultivaluedMapImpl params = new MultivaluedMapImpl(); + params.add("labels", "a"); + response = r.path("ws").path("v1").path("cluster").path("nodes") + .path("nid:0").path("replace-labels").queryParam("user.name", userName) + .queryParams(params).accept(MediaType.APPLICATION_JSON) + .post(ClientResponse.class); + LOG.info("posted node nodelabel"); + + MultivaluedMapImpl labels = new MultivaluedMapImpl(); + labels.add("partionLabels", label); + response = r.path("ws").path("v1").path("cluster") + .path("partition-stats-info").queryParam("user.name", userName) + .queryParams(labels).accept(media).get(ClientResponse.class); + + assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType()); + PartitionsStatisticsInfo partitionStats = response + .getEntity(PartitionsStatisticsInfo.class); + + assertEquals("a", + partitionStats.getPartitionStatsInfo().get(0).getLabelName()); + assertEquals(1, + partitionStats.getPartitionStatsInfo().get(0).getNodeList().size()); + assertEquals("nid:0", + partitionStats.getPartitionStatsInfo().get(0).getNodeList().get(0)); + } + @SuppressWarnings("rawtypes") private String toJson(Object nsli, Class klass) throws Exception { StringWriter sw = new StringWriter(); -- 2.7.4 (Apple Git-66)