diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
new file mode 100644
index 0000000..b8ad66e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+15 * See the License for the specific language governing permissions and
+16 * limitations under the License.
+17 */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ *
+ * ResourceUtilization models the utilization of a set of computer
+ * resources in the cluster.
+ *
+ *
+ * @see Resource
+ */
+@Public
+@Evolving
+public abstract class ResourceUtilization implements
+ Comparable {
+ @Public
+ @Evolving
+ public static ResourceUtilization newInstance(int pmem, int vmem, float cpu) {
+ ResourceUtilization utilization =
+ Records.newRecord(ResourceUtilization.class);
+ utilization.setPhysicalMemory(pmem);
+ utilization.setVirtualMemory(vmem);
+ utilization.setCPUUsage(cpu);
+ return utilization;
+ }
+
+ /**
+ * Get used virtual memory.
+ *
+ * @return virtual memory in MB
+ */
+ @Public
+ @Evolving
+ public abstract int getVirtualMemory();
+
+ /**
+ * Set used virtual memory.
+ *
+ * @param vmem virtual memory in MB
+ */
+ @Public
+ @Evolving
+ public abstract void setVirtualMemory(int vmem);
+
+ /**
+ * Get physical memory.
+ *
+ * @return physical memory in MB
+ */
+ @Public
+ @Evolving
+ public abstract int getPhysicalMemory();
+
+ /**
+ * Set physical memory.
+ *
+ * @param pmem physical memory in MB
+ */
+ @Public
+ @Evolving
+ public abstract void setPhysicalMemory(int pmem);
+
+ /**
+ * Get CPU utilization.
+ *
+ * @return CPU utilization normalized to 1 CPU
+ */
+ @Public
+ @Evolving
+ public abstract float getCPUUsage();
+
+ /**
+ * Set CPU utilization.
+ *
+ * @param cpu CPU utilization normalized to 1 CPU
+ */
+ @Public
+ @Evolving
+ public abstract void setCPUUsage(float cpu);
+
+ @Override
+ public int hashCode() {
+ final int prime = 263167;
+ int result = 3571;
+ result = 939769357 + getVirtualMemory(); // prime * result = 939769357
+ // initially
+ result = prime * result + getPhysicalMemory();
+ result = 31 * result + Float.valueOf(getCPUUsage()).hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof ResourceUtilization)) {
+ return false;
+ }
+ ResourceUtilization other = (ResourceUtilization) obj;
+ if (getVirtualMemory() != other.getVirtualMemory()
+ || getPhysicalMemory() != other.getPhysicalMemory()
+ || getCPUUsage() != other.getCPUUsage()) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "";
+ }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index f801409..47b731b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -58,6 +58,12 @@ message ResourceProto {
optional int32 virtual_cores = 2;
}
+message ResourceUtilizationProto {
+ optional int32 pmem = 1;
+ optional int32 vmem = 2;
+ optional float cpu = 3;
+}
+
message ResourceOptionProto {
optional ResourceProto resource = 1;
optional int32 over_commit_timeout = 2;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java
new file mode 100644
index 0000000..a24df77
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java
@@ -0,0 +1,105 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProtoOrBuilder;
+
+@Private
+@Unstable
+public class ResourceUtilizationPBImpl extends ResourceUtilization {
+ ResourceUtilizationProto proto = ResourceUtilizationProto
+ .getDefaultInstance();
+ ResourceUtilizationProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public ResourceUtilizationPBImpl() {
+ builder = ResourceUtilizationProto.newBuilder();
+ }
+
+ public ResourceUtilizationPBImpl(ResourceUtilizationProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ResourceUtilizationProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ResourceUtilizationProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int getPhysicalMemory() {
+ ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getPmem());
+ }
+
+ @Override
+ public void setPhysicalMemory(int pmem) {
+ maybeInitBuilder();
+ builder.setPmem(pmem);
+ }
+
+ @Override
+ public int getVirtualMemory() {
+ ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getVmem());
+ }
+
+ @Override
+ public void setVirtualMemory(int vmem) {
+ maybeInitBuilder();
+ builder.setPmem(vmem);
+ }
+
+ @Override
+ public float getCPUUsage() {
+ ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getCpu();
+ }
+
+ @Override
+ public void setCPUUsage(float cpu) {
+ maybeInitBuilder();
+ builder.setCpu(cpu);
+ }
+
+ @Override
+ public int compareTo(ResourceUtilization other) {
+ int diff = this.getPhysicalMemory() - other.getPhysicalMemory();
+ if (diff == 0) {
+ diff = this.getVirtualMemory() - other.getVirtualMemory();
+ if (diff == 0) {
+ diff = Float.compare(this.getCPUUsage(), other.getCPUUsage());
+ }
+ }
+ return diff;
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtilizations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtilizations.java
new file mode 100644
index 0000000..ef65be6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtilizations.java
@@ -0,0 +1,55 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.util.resource;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.util.Records;
+
+@InterfaceAudience.LimitedPrivate({"YARN", "MapReduce"})
+@Unstable
+public class ResourceUtilizations {
+ public static ResourceUtilization createResource(int pmem, int vmem, float cpu) {
+ ResourceUtilization utilization =
+ Records.newRecord(ResourceUtilization.class);
+ utilization.setPhysicalMemory(pmem);
+ utilization.setVirtualMemory(vmem);
+ utilization.setCPUUsage(cpu);
+ return utilization;
+ }
+
+ public static ResourceUtilization clone(ResourceUtilization res) {
+ return createResource(res.getPhysicalMemory(), res.getVirtualMemory(),
+ res.getCPUUsage());
+ }
+
+ public static ResourceUtilization addTo(ResourceUtilization lhs,
+ ResourceUtilization rhs) {
+ lhs.setPhysicalMemory(lhs.getPhysicalMemory() + rhs.getPhysicalMemory());
+ lhs.setVirtualMemory(lhs.getVirtualMemory() + rhs.getVirtualMemory());
+ lhs.setCPUUsage(lhs.getCPUUsage() + rhs.getCPUUsage());
+ return lhs;
+ }
+
+ public static ResourceUtilization add(ResourceUtilization lhs,
+ ResourceUtilization rhs) {
+ return addTo(clone(lhs), rhs);
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 8dacd3b..9c91041 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -140,6 +140,7 @@
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
import org.apache.hadoop.yarn.api.records.Token;
@@ -175,6 +176,7 @@
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceOptionPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourceRequestPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourceUtilizationPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.StrictPreemptionContractPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl;
@@ -209,6 +211,7 @@
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceOptionProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceRequestProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProto;
import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto;
import org.apache.hadoop.yarn.proto.YarnProtos.StrictPreemptionContractProto;
import org.apache.hadoop.yarn.proto.YarnProtos.URLProto;
@@ -453,6 +456,7 @@ public static void setup() throws Exception {
generateByNewInstance(Resource.class);
generateByNewInstance(ResourceBlacklistRequest.class);
generateByNewInstance(ResourceOption.class);
+ generateByNewInstance(ResourceUtilization.class);
generateByNewInstance(LocalResource.class);
generateByNewInstance(Priority.class);
generateByNewInstance(NodeId.class);
@@ -1058,6 +1062,12 @@ public void testResourcePBImpl() throws Exception {
}
@Test
+ public void testResourceUtilizationPBImpl() throws Exception {
+ validatePBImplRecord(ResourceUtilizationPBImpl.class,
+ ResourceUtilizationProto.class);
+ }
+
+ @Test
public void testResourceRequestPBImpl() throws Exception {
validatePBImplRecord(ResourceRequestPBImpl.class, ResourceRequestProto.class);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeHealthStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeHealthStatus.java
index b21b880..6c608c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeHealthStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeHealthStatus.java
@@ -23,6 +23,7 @@
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.util.Records;
/**
@@ -90,4 +91,17 @@ public static NodeHealthStatus newInstance(boolean isNodeHealthy,
@Private
@Unstable
public abstract void setLastHealthReportTime(long lastHealthReport);
+
+ /**
+ * Get the resource utilization of the containers.
+ * @return resource utilization of the containers
+ */
+ @Public
+ @Stable
+ public abstract ResourceUtilization getContainersUtilization();
+
+ @Private
+ @Unstable
+ public abstract void setContainersUtilization(
+ ResourceUtilization containersUtilization);
}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java
index 75aa3d1..7687efc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeHealthStatusPBImpl.java
@@ -18,6 +18,11 @@
package org.apache.hadoop.yarn.server.api.records.impl.pb;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourceUtilizationPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProtoOrBuilder;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
@@ -128,4 +133,33 @@ public void setLastHealthReportTime(long lastHealthReport) {
this.builder.setLastHealthReportTime((lastHealthReport));
}
+ @Override
+ public ResourceUtilization getContainersUtilization() {
+ NodeHealthStatusProtoOrBuilder p =
+ this.viaProto ? this.proto : this.builder;
+ if (!p.hasContainersUtilization()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getContainersUtilization());
+ }
+
+ @Override
+ public void setContainersUtilization(ResourceUtilization containersUtilization) {
+ maybeInitBuilder();
+ if (containersUtilization == null) {
+ this.builder.clearContainersUtilization();
+ return;
+ }
+ this.builder
+ .setContainersUtilization(convertToProtoFormat(containersUtilization));
+ }
+
+ private ResourceUtilizationProto convertToProtoFormat(ResourceUtilization r) {
+ return ((ResourceUtilizationPBImpl) r).getProto();
+ }
+
+ private ResourceUtilizationPBImpl convertFromProtoFormat(
+ ResourceUtilizationProto p) {
+ return new ResourceUtilizationPBImpl(p);
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
index 99149ac..af36fc7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
@@ -47,6 +47,7 @@ message NodeHealthStatusProto {
optional bool is_node_healthy = 1;
optional string health_report = 2;
optional int64 last_health_report_time = 3;
+ optional ResourceUtilizationProto containers_utilization = 4;
}
message VersionProto {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 3721b0e..df25419 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -52,6 +52,7 @@
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -77,6 +78,7 @@
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider;
import org.apache.hadoop.yarn.util.Records;
@@ -424,11 +426,15 @@ private NodeStatus getNodeStatus(int responseId) throws IOException {
nodeHealthStatus.setIsNodeHealthy(healthChecker.isHealthy());
nodeHealthStatus.setLastHealthReportTime(healthChecker
.getLastHealthReportTime());
+
+ ResourceUtilization containersUtilization = getContainersUtilization();
+ nodeHealthStatus.setContainersUtilization(containersUtilization);
if (LOG.isDebugEnabled()) {
LOG.debug("Node's health-status : " + nodeHealthStatus.getIsNodeHealthy()
+ ", " + nodeHealthStatus.getHealthReport());
}
List containersStatuses = getContainerStatuses();
+
NodeStatus nodeStatus =
NodeStatus.newInstance(nodeId, responseId, containersStatuses,
createKeepAliveApplicationList(), nodeHealthStatus);
@@ -436,6 +442,18 @@ private NodeStatus getNodeStatus(int responseId) throws IOException {
return nodeStatus;
}
+ /**
+ * Get the aggregated utilization of the containers in this node.
+ * @return Resource utilization of all the containers.
+ */
+ private ResourceUtilization getContainersUtilization() {
+ ContainerManagerImpl containerManager =
+ (ContainerManagerImpl) this.context.getContainerManager();
+ ContainersMonitor containersMonitor =
+ containerManager.getContainersMonitor();
+ return containersMonitor.getContainersUtilization();
+ }
+
// Iterate through the NMContext and clone and get all the containers'
// statuses. If it's a completed container, add into the
// recentlyStoppedContainers collections.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 289d6d0..bdc9a89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -45,6 +45,7 @@
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
index d3e2bf2..4d69dbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
@@ -19,10 +19,11 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
public interface ContainersMonitor extends Service,
EventHandler, ResourceView {
-
+ public ResourceUtilization getContainersUtilization();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index b5f154d..7713eeb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -41,6 +42,7 @@
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.resource.ResourceUtilizations;
import com.google.common.base.Preconditions;
@@ -78,6 +80,8 @@
private static final long UNKNOWN_MEMORY_LIMIT = -1L;
private int nodeCpuPercentageForYARN;
+ private ResourceUtilization containersUtilization;
+
public ContainersMonitorImpl(ContainerExecutor exec,
AsyncDispatcher dispatcher, Context context) {
super("containers-monitor");
@@ -89,6 +93,8 @@ public ContainersMonitorImpl(ContainerExecutor exec,
this.containersToBeAdded = new HashMap();
this.containersToBeRemoved = new ArrayList();
this.monitoringThread = new MonitoringThread();
+
+ this.containersUtilization = ResourceUtilization.newInstance(0, 0, 0.0f);
}
@Override
@@ -384,6 +390,11 @@ public void run() {
containersToBeRemoved.clear();
}
+ // Temporary structure to calculate the total resource utilization of
+ // the containers
+ ResourceUtilization auxContainersUtilization =
+ ResourceUtilization.newInstance(0, 0, 0.0f);
+
// Now do the monitoring for the trackingContainers
// Check memory usage and kill any overflowing containers
long vmemUsageByAllContainers = 0;
@@ -463,6 +474,18 @@ public void run() {
currentPmemUsage, pmemLimit));
}
+ // Save utilization for this container
+ ResourceUtilization containerUtilization =
+ ResourceUtilization.newInstance((int) (currentPmemUsage >> 20),
+ (int) (currentVmemUsage >> 20), milliVcoresUsed / 1000.0f);
+ ResourceUtilizations.addTo(auxContainersUtilization,
+ containerUtilization);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Container " + containerId + " utilization is "
+ + containerUtilization);
+ }
+
// Add usage to container metrics
if (containerMetricsEnabled) {
ContainerMetrics.forContainer(
@@ -542,6 +565,9 @@ public void run() {
+ cpuUsagePercentPerCoreByAllContainers);
}
+ // Save the aggregated utilization of the containers
+ setContainersUtilization(auxContainersUtilization);
+
try {
Thread.sleep(monitoringInterval);
} catch (InterruptedException e) {
@@ -614,6 +640,15 @@ public boolean isVmemCheckEnabled() {
}
@Override
+ public ResourceUtilization getContainersUtilization() {
+ return this.containersUtilization;
+ }
+
+ public void setContainersUtilization(ResourceUtilization utilization) {
+ this.containersUtilization = utilization;
+ }
+
+ @Override
public void handle(ContainersMonitorEvent monitoringEvent) {
if (!isEnabled()) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index b2ccb61..637f7c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;