From e518a3b283df4044385243d874adc539bc32e61f Mon Sep 17 00:00:00 2001 From: Sunil G Date: Thu, 25 Jan 2018 18:30:24 +0530 Subject: [PATCH] YARN-7817 --- .../webapp/JAXBContextResolver.java | 2 +- .../webapp/dao/ClusterMetricsInfo.java | 24 +++++++++++ .../resourcemanager/webapp/dao/NodeInfo.java | 20 +++++++++ .../resourcemanager/webapp/dao/ResourceInfo.java | 18 ++++++-- .../webapp/dao/ResourceInformationsInfo.java | 48 ++++++++++++++++++++++ 5 files changed, 108 insertions(+), 4 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index 2f50a24eac3..2e4204e08af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -55,7 +55,7 @@ public JAXBContextResolver() throws Exception { UsersInfo.class, UserInfo.class, ApplicationStatisticsInfo.class, StatisticsItemInfo.class, CapacitySchedulerHealthInfo.class, FairSchedulerQueueInfoList.class, AppTimeoutsInfo.class, - AppTimeoutInfo.class }; + AppTimeoutInfo.class, ResourceInformationsInfo.class }; // these dao classes need root unwrapping final Class[] rootUnwrappedTypes = { NewApplication.class, ApplicationSubmissionContextInfo.class, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index 3214cb9f85e..84f70d9f2b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -61,6 +61,12 @@ private int activeNodes; private int shutdownNodes; + // Total used resource of the cluster, including all partitions + private ResourceInfo totalUsedResourcesAcrossPartition; + + // Total registered resources of the cluster, including all partitions + private ResourceInfo totalClusterResourcesAcrossPartition; + public ClusterMetricsInfo() { } // JAXB needs this @@ -92,9 +98,20 @@ public ClusterMetricsInfo(final ResourceScheduler rs) { this.containersReserved = metrics.getReservedContainers(); if (rs instanceof CapacityScheduler) { + CapacityScheduler cs = (CapacityScheduler) rs; this.totalMB = availableMB + allocatedMB + reservedMB; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores + containersReserved; + // TODO, add support of other schedulers to get total used resources + // across partition. + if (cs.getRootQueue() != null + && cs.getRootQueue().getQueueResourceUsage() != null + && cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) { + totalUsedResourcesAcrossPartition = new ResourceInfo( + cs.getRootQueue().getQueueResourceUsage().getAllUsed()); + totalClusterResourcesAcrossPartition = new ResourceInfo( + cs.getClusterResource()); + } } else { this.totalMB = availableMB + allocatedMB; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores; @@ -310,4 +327,11 @@ public void setShutdownNodes(int shutdownNodes) { this.shutdownNodes = shutdownNodes; } + public ResourceInfo getTotalUsedResourcesAcrossPartition() { + return totalUsedResourcesAcrossPartition; + } + + public ResourceInfo getTotalClusterResourcesAcrossPartition() { + return totalClusterResourcesAcrossPartition; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index 2530c8ea117..3cec215f0e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -58,6 +58,8 @@ private int numQueuedContainers; protected ArrayList nodeLabels = new ArrayList(); protected ResourceUtilizationInfo resourceUtilization; + protected ResourceInfo usedResource; + protected ResourceInfo availableResource; public NodeInfo() { } // JAXB needs this @@ -75,6 +77,8 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) { this.usedVirtualCores = report.getUsedResource().getVirtualCores(); this.availableVirtualCores = report.getAvailableResource().getVirtualCores(); + this.usedResource = new ResourceInfo(report.getUsedResource()); + this.availableResource = new ResourceInfo(report.getAvailableResource()); } this.id = id.toString(); this.rack = ni.getRackName(); @@ -183,6 +187,22 @@ public int getNumQueuedContainers() { return this.nodeLabels; } + public ResourceInfo getUsedResource() { + return usedResource; + } + + public void setUsedResource(ResourceInfo used) { + this.usedResource = used; + } + + public ResourceInfo getAvailableResource() { + return availableResource; + } + + public void setAvailableResource(ResourceInfo avail) { + this.availableResource = avail; + } + public ResourceUtilizationInfo getResourceUtilization() { return this.resourceUtilization; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index e13980afc39..5bed936d402 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.util.resource.Resources; + @XmlRootElement @XmlAccessorType(XmlAccessType.NONE) public class ResourceInfo { @@ -34,6 +35,9 @@ long memory; @XmlElement int vCores; + @XmlElement + ResourceInformationsInfo resourceInformations = + new ResourceInformationsInfo(); private Resource resources; @@ -41,9 +45,13 @@ public ResourceInfo() { } public ResourceInfo(Resource res) { - memory = res.getMemorySize(); - vCores = res.getVirtualCores(); - resources = Resources.clone(res); + // Make sure no NPE. + if (res != null) { + memory = res.getMemorySize(); + vCores = res.getVirtualCores(); + resources = Resources.clone(res); + resourceInformations.addAll(res.getAllResourcesListCopy()); + } } public long getMemorySize() { @@ -84,4 +92,8 @@ public void setvCores(int vCores) { public Resource getResource() { return Resource.newInstance(resources); } + + public ResourceInformationsInfo getResourcesInformations() { + return resourceInformations; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java new file mode 100644 index 00000000000..651d7dfef88 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ResourceInformation; + +@XmlRootElement(name = "resourceInformations") +@XmlAccessorType(XmlAccessType.FIELD) +public class ResourceInformationsInfo { + + @XmlElement(name = "resourceInformation") + protected ArrayList resourceInformation = + new ArrayList(); + + public ResourceInformationsInfo() { + } // JAXB needs this + + public ArrayList getApps() { + return resourceInformation; + } + + public void addAll(List resourcesInformationsInfo) { + resourceInformation.addAll(resourcesInformationsInfo); + } +} -- 2.14.3 (Apple Git-98)