diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
index edc62fc486..c2fabf19e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerStatus.java
@@ -223,4 +223,24 @@ public ContainerSubState getContainerSubState() {
throw new UnsupportedOperationException(
"subclass must implement this method");
}
+
+ /**
+ * Get the ResourceUtilization of the container.
+ *
+ * @return ResourceUtilization of the container
+ */
+ @Public
+ @Unstable
+ public ResourceUtilization getUtilization() {
+ throw new UnsupportedOperationException(
+ "subclass must implement this method");
+ }
+
+ @Private
+ @Unstable
+ public void setUtilization(
+ ResourceUtilization utilization) {
+ throw new UnsupportedOperationException(
+ "subclass must implement this method");
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
index a6668dad7b..2de5aad64a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerStatusPBImpl.java
@@ -28,6 +28,7 @@
import org.apache.hadoop.yarn.api.records.ContainerSubState;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.proto.YarnProtos;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
@@ -52,6 +53,7 @@
private ContainerId containerId = null;
private static final String HOST = "HOST";
private static final String IPS = "IPS";
+ private static final String RESOURCE_UTILIZATION = "RESOURCE_UTILIZATION";
private Map containerAttributes = new HashMap<>();
@@ -98,6 +100,7 @@ public String toString() {
sb.append("ExitStatus: ").append(getExitStatus()).append(", ");
sb.append("IP: ").append(getIPs()).append(", ");
sb.append("Host: ").append(getHost()).append(", ");
+ sb.append("Utilization: ").append(getUtilization()).append(", ");
sb.append("ContainerSubState: ").append(getContainerSubState());
sb.append("]");
return sb.toString();
@@ -336,6 +339,39 @@ public synchronized void setHost(String host) {
containerAttributes.put(HOST, host);
}
+ @Override
+ public synchronized ResourceUtilization getUtilization() {
+ if (containerAttributes.get(RESOURCE_UTILIZATION) == null) {
+ initContainerAttributes();
+ }
+ String resourceUtilizationStr =
+ containerAttributes.get(RESOURCE_UTILIZATION);
+ if (resourceUtilizationStr != null
+ && resourceUtilizationStr.split(",").length == 3) {
+ String[] splits = resourceUtilizationStr.split(",");
+ int pmem = Integer.valueOf(splits[0]);
+ int vmem = Integer.valueOf(splits[1]);
+ float cpu = Float.valueOf(splits[2]);
+ return ResourceUtilization.newInstance(pmem, vmem, cpu);
+ }
+ return null;
+ }
+
+ @Override
+ public synchronized void setUtilization(
+ ResourceUtilization utilization) {
+ maybeInitBuilder();
+ if (utilization == null) {
+ containerAttributes.remove(RESOURCE_UTILIZATION);
+ return;
+ }
+ String utilizationStr = String
+ .format("%s,%s,%s", utilization.getPhysicalMemory(),
+ utilization.getVirtualMemory(), utilization.getCPU());
+ containerAttributes.put(RESOURCE_UTILIZATION, utilizationStr);
+ }
+
+
private ContainerStateProto convertToProtoFormat(ContainerState e) {
return ProtoUtils.convertToProtoFormat(e);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
index 74f19e5a4b..805e652b70 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
@@ -39,6 +39,7 @@
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
@@ -191,10 +192,14 @@ public void testContainerStatus() {
List ips = Arrays.asList("127.0.0.1", "139.5.25.2");
status.setIPs(ips);
status.setHost("locahost123");
+ ResourceUtilization utilization =
+ ResourceUtilization.newInstance(4096, 8192, 0.8f);
+ status.setUtilization(utilization);
ContainerStatusPBImpl pb =
new ContainerStatusPBImpl(((ContainerStatusPBImpl) status).getProto());
Assert.assertEquals(ips, pb.getIPs());
Assert.assertEquals("locahost123", pb.getHost());
+ Assert.assertEquals(utilization, pb.getUtilization());
status.setIPs(null);
Assert.assertNull(status.getIPs());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index c09c7f1ac2..bdd92b5e89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -36,6 +36,7 @@
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerSubState;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler.UpdateContainerSchedulerEvent;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -846,6 +847,14 @@ public ContainerStatus cloneAndGetContainerStatus() {
Arrays.asList(ips.split(",")));
status.setHost(host);
status.setContainerSubState(getContainerSubState());
+ if (containerMetrics != null) {
+ ResourceUtilization utilization = ResourceUtilization.newInstance(
+ (int) containerMetrics.pMemMBsStat.lastStat().mean(),
+ (int) containerMetrics.vMemMBsStat.lastStat().mean(),
+ (float) containerMetrics.milliVcoresUsed.lastStat().mean()
+ / 1000.0f);
+ status.setUtilization(utilization);
+ }
return status;
} finally {
this.readLock.unlock();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
index 2a958494e8..c490a67f81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerMetrics.java
@@ -55,6 +55,8 @@
public static final String VCORE_LIMIT_METRIC_NAME = "vCoreLimit";
public static final String PMEM_USAGE_METRIC_NAME = "pMemUsageMBs";
public static final String PMEM_USAGE_QUANTILES_NAME = "pMemUsageMBHistogram";
+ public static final String VMEM_USAGE_METRIC_NAME = "vMemUsageMBs";
+ public static final String VMEM_USAGE_QUANTILES_NAME = "vMemUsageMBHistogram";
public static final String LAUNCH_DURATION_METRIC_NAME = "launchDurationMs";
public static final String LOCALIZATION_DURATION_METRIC_NAME =
"localizationDurationMs";
@@ -72,6 +74,12 @@
@Metric
public MutableQuantiles pMemMBQuantiles;
+ @Metric
+ public MutableStat vMemMBsStat;
+
+ @Metric
+ public MutableQuantiles vMemMBQuantiles;
+
// This tracks overall CPU percentage of the machine in terms of percentage
// of 1 core similar to top
// Thus if you use 2 cores completely out of 4 available cores this value
@@ -154,6 +162,13 @@
this.pMemMBQuantiles = registry
.newQuantiles(PMEM_USAGE_QUANTILES_NAME, "Physical memory quantiles",
"Usage", "MBs", 1);
+
+ this.vMemMBsStat = registry.newStat(
+ VMEM_USAGE_METRIC_NAME, "Virtual memory stats", "Usage", "MBs", true);
+ this.vMemMBQuantiles = registry
+ .newQuantiles(VMEM_USAGE_QUANTILES_NAME, "Virtual memory quantiles",
+ "Usage", "MBs", 1);
+
ContainerMetricsQuantiles memEstimator =
new ContainerMetricsQuantiles(MutableQuantiles.quantiles);
pMemMBQuantiles.setEstimator(memEstimator);
@@ -262,6 +277,13 @@ public void recordMemoryUsage(int memoryMBs) {
}
}
+ public void recordVMemoryUsage(int vMemoryMBs) {
+ if (vMemoryMBs >= 0) {
+ this.vMemMBsStat.add(vMemoryMBs);
+ this.vMemMBQuantiles.add(vMemoryMBs);
+ }
+ }
+
public void recordCpuUsage(
int totalPhysicalCpuPercent, int milliVcoresUsed) {
if (totalPhysicalCpuPercent >=0) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 35015c29ed..6daf47a392 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -629,6 +629,10 @@ private void recordUsage(ContainerId containerId, String pId,
containerId, containerMetricsPeriodMs,
containerMetricsUnregisterDelayMs).recordMemoryUsage(
(int) (currentPmemUsage >> 20));
+ ContainerMetrics.forContainer(
+ containerId, containerMetricsPeriodMs,
+ containerMetricsUnregisterDelayMs).recordVMemoryUsage(
+ (int) (currentVmemUsage >> 20));
ContainerMetrics.forContainer(
containerId, containerMetricsPeriodMs,
containerMetricsUnregisterDelayMs).recordCpuUsage((int)