From 9547ab553ad8ec537e62bfe432583ecdd43f1916 Mon Sep 17 00:00:00 2001 From: Sunil G Date: Thu, 25 Jan 2018 18:13:08 +0530 Subject: [PATCH] YARN-7817 --- .../webapp/dao/ClusterMetricsInfo.java | 24 +++++++++++ .../resourcemanager/webapp/dao/NodeInfo.java | 20 ++++++++++ .../resourcemanager/webapp/dao/ResourceInfo.java | 18 +++++++-- .../webapp/dao/ResourceInformationsInfo.java | 46 ++++++++++++++++++++++ 4 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java index 3214cb9f85e..84f70d9f2b6 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterMetricsInfo.java @@ -61,6 +61,12 @@ private int activeNodes; private int shutdownNodes; + // Total used resource of the cluster, including all partitions + private ResourceInfo totalUsedResourcesAcrossPartition; + + // Total registered resources of the cluster, including all partitions + private ResourceInfo totalClusterResourcesAcrossPartition; + public ClusterMetricsInfo() { } // JAXB needs this @@ -92,9 +98,20 @@ public ClusterMetricsInfo(final ResourceScheduler rs) { this.containersReserved = metrics.getReservedContainers(); if (rs instanceof CapacityScheduler) { + CapacityScheduler cs = (CapacityScheduler) rs; this.totalMB = availableMB + allocatedMB + reservedMB; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores + containersReserved; + // TODO, add support of other schedulers to get total used resources + // across partition. + if (cs.getRootQueue() != null + && cs.getRootQueue().getQueueResourceUsage() != null + && cs.getRootQueue().getQueueResourceUsage().getAllUsed() != null) { + totalUsedResourcesAcrossPartition = new ResourceInfo( + cs.getRootQueue().getQueueResourceUsage().getAllUsed()); + totalClusterResourcesAcrossPartition = new ResourceInfo( + cs.getClusterResource()); + } } else { this.totalMB = availableMB + allocatedMB; this.totalVirtualCores = availableVirtualCores + allocatedVirtualCores; @@ -310,4 +327,11 @@ public void setShutdownNodes(int shutdownNodes) { this.shutdownNodes = shutdownNodes; } + public ResourceInfo getTotalUsedResourcesAcrossPartition() { + return totalUsedResourcesAcrossPartition; + } + + public ResourceInfo getTotalClusterResourcesAcrossPartition() { + return totalClusterResourcesAcrossPartition; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java index 2530c8ea117..3cec215f0e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java @@ -58,6 +58,8 @@ private int numQueuedContainers; protected ArrayList nodeLabels = new ArrayList(); protected ResourceUtilizationInfo resourceUtilization; + protected ResourceInfo usedResource; + protected ResourceInfo availableResource; public NodeInfo() { } // JAXB needs this @@ -75,6 +77,8 @@ public NodeInfo(RMNode ni, ResourceScheduler sched) { this.usedVirtualCores = report.getUsedResource().getVirtualCores(); this.availableVirtualCores = report.getAvailableResource().getVirtualCores(); + this.usedResource = new ResourceInfo(report.getUsedResource()); + this.availableResource = new ResourceInfo(report.getAvailableResource()); } this.id = id.toString(); this.rack = ni.getRackName(); @@ -183,6 +187,22 @@ public int getNumQueuedContainers() { return this.nodeLabels; } + public ResourceInfo getUsedResource() { + return usedResource; + } + + public void setUsedResource(ResourceInfo used) { + this.usedResource = used; + } + + public ResourceInfo getAvailableResource() { + return availableResource; + } + + public void setAvailableResource(ResourceInfo avail) { + this.availableResource = avail; + } + public ResourceUtilizationInfo getResourceUtilization() { return this.resourceUtilization; } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java index e13980afc39..670832533fe 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInfo.java @@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.util.resource.Resources; + @XmlRootElement @XmlAccessorType(XmlAccessType.NONE) public class ResourceInfo { @@ -35,15 +36,22 @@ @XmlElement int vCores; + @XmlElement + ResourceInformationsInfo resourcesInformations = + new ResourceInformationsInfo(); private Resource resources; public ResourceInfo() { } public ResourceInfo(Resource res) { - memory = res.getMemorySize(); - vCores = res.getVirtualCores(); - resources = Resources.clone(res); + // Make sure no NPE. + if (res != null) { + memory = res.getMemorySize(); + vCores = res.getVirtualCores(); + resources = Resources.clone(res); + resourcesInformations.addAll(res.getAllResourcesListCopy()); + } } public long getMemorySize() { @@ -84,4 +92,8 @@ public void setvCores(int vCores) { public Resource getResource() { return Resource.newInstance(resources); } + + public ResourceInformationsInfo getResourcesInformations() { + return resourcesInformations; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java new file mode 100644 index 00000000000..5d373468238 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceInformationsInfo.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao; + +import java.util.ArrayList; +import java.util.List; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; + +import org.apache.hadoop.yarn.api.records.ResourceInformation; + +@XmlRootElement(name = "resourceInformation") +@XmlAccessorType(XmlAccessType.FIELD) +public class ResourceInformationsInfo { + + protected List resourceInformation = + new ArrayList(); + + public ResourceInformationsInfo() { + } // JAXB needs this + + public List getApps() { + return resourceInformation; + } + + public void addAll(List resourcesInformationsInfo) { + resourceInformation.addAll(resourcesInformationsInfo); + } +} -- 2.14.3 (Apple Git-98)