diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
new file mode 100644
index 0000000..ee0de70
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ *
+ * ResourceUtilization models the utilization of a set of computer
+ * resources in the cluster.
+ *
+ *
+ * @see Resource
+ */
+@Public
+@Evolving
+public abstract class ResourceUtilization implements Comparable {
+
+ @Public
+ @Evolving
+ public static ResourceUtilization newInstance(int pmem, int vmem, float cpu) {
+ ResourceUtilization utilization =
+ Records.newRecord(ResourceUtilization.class);
+ utilization.setPhysicalMemory(pmem);
+ utilization.setVirtualMemory(vmem);
+ utilization.setCPUUsage(cpu);
+ return utilization;
+ }
+
+ /**
+ * Get used virtual memory.
+ *
+ * @return virtual memory in MB
+ */
+ @Public
+ @Evolving
+ public abstract int getVirtualMemory();
+
+ /**
+ * Set used virtual memory.
+ *
+ * @param virtual memory in MB
+ */
+ @Public
+ @Evolving
+ public abstract void setVirtualMemory(int memory);
+
+ /**
+ * Get physical memory.
+ *
+ * @return physical memory in MB
+ */
+ @Public
+ @Evolving
+ public abstract int getPhysicalMemory();
+
+ /**
+ * Set physical memory.
+ *
+ * @param physical memory in MB
+ */
+ @Public
+ @Evolving
+ public abstract void setPhysicalMemory(int memory);
+
+ /**
+ * Get CPU utilization
+ *
+ * @return CPU utilization normalized to 1 CPU
+ */
+ @Public
+ @Evolving
+ public abstract float getCPUUsage();
+
+ /**
+ * Set CPU utilization
+ *
+ * @param CPU utilization normalized to 1 CPU
+ */
+ @Public
+ @Evolving
+ public abstract void setCPUUsage(float cpu);
+
+ @Override
+ public int hashCode() {
+ final int prime = 263167;
+ int result = 3571;
+ result = 939769357 + getVirtualMemory(); // prime * result = 939769357
+ // initially
+ result = prime * result + getPhysicalMemory();
+ result = 31 * result + Float.valueOf(getCPUUsage()).hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (!(obj instanceof ResourceUtilization))
+ return false;
+ ResourceUtilization other = (ResourceUtilization) obj;
+ if (getVirtualMemory() != other.getVirtualMemory()
+ || getPhysicalMemory() != other.getPhysicalMemory()
+ || getCPUUsage() != other.getCPUUsage()) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "";
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ea0d71d..3d6c1cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -944,11 +944,21 @@ private static void addDeprecatedKeys() {
+ "webapp.https.address";
public static final int DEFAULT_NM_WEBAPP_HTTPS_PORT = 8044;
public static final String DEFAULT_NM_WEBAPP_HTTPS_ADDRESS = "0.0.0.0:"
- + DEFAULT_NM_WEBAPP_HTTPS_PORT;
+ + DEFAULT_NM_WEBAPP_HTTPS_PORT;
+ /** How often to monitor utilization. */
+ public static final String NM_USAGE_MON_INTERVAL_MS = NM_PREFIX
+ + "usage-monitor.interval-ms";
+ public final static int DEFAULT_NM_USAGE_MON_INTERVAL_MS = 3000;
+
+ /** Class that calculates nodes current resource utilization.*/
+ public static final String NM_NODE_MON_RESOURCE_CALCULATOR =
+ NM_PREFIX + "node-monitor.resource-calculator.class";
+
/** How often to monitor containers.*/
public final static String NM_CONTAINER_MON_INTERVAL_MS =
NM_PREFIX + "container-monitor.interval-ms";
+ @Deprecated
public final static int DEFAULT_NM_CONTAINER_MON_INTERVAL_MS = 3000;
/** Class that calculates containers current resource utilization.*/
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index b9969b0..f8a3fa4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -58,6 +58,12 @@ message ResourceProto {
optional int32 virtual_cores = 2;
}
+message ResourceUtilizationProto {
+ optional int32 pmem = 1;
+ optional int32 vmem = 2;
+ optional float cpu = 3;
+}
+
message ResourceOptionProto {
optional ResourceProto resource = 1;
optional int32 over_commit_timeout = 2;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtilizations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtilizations.java
new file mode 100644
index 0000000..f6ec6e9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtilizations.java
@@ -0,0 +1,16 @@
+package org.apache.hadoop.yarn.util.resource;
+
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.util.Records;
+
+public class ResourceUtilizations {
+ public static ResourceUtilization createResourceUtilization(int pmem,
+ int vmem, float cpu) {
+ ResourceUtilization utilization =
+ Records.newRecord(ResourceUtilization.class);
+ utilization.setPhysicalMemory(pmem);
+ utilization.setVirtualMemory(vmem);
+ utilization.setCPUUsage(cpu);
+ return utilization;
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java
index be13d22..9d8b80d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java
@@ -19,7 +19,15 @@
package org.apache.hadoop.yarn.server.nodemanager;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+/**
+ * Interface for monitoring the resources of a node.
+ */
public interface NodeResourceMonitor extends Service {
-
+ /**
+ * Get the resource utilization of the node.
+ * @return resource utilization of the node.
+ */
+ public ResourceUtilization getUtilization();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
index ea82546..5c26dd2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
@@ -18,13 +18,154 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.resource.ResourceUtilizations;
+/**
+ * Implementation of the node resource monitor. It periodically tracks the
+ * resource utilization of the node and reports it to the NM.
+ */
public class NodeResourceMonitorImpl extends AbstractService implements
NodeResourceMonitor {
+ /** Logging infrastructure. */
+ final static Log LOG = LogFactory
+ .getLog(NodeResourceMonitorImpl.class);
+
+ /** Interval to monitor the node resource utilization. */
+ private long monitoringInterval;
+ /** Thread to monitor the node resource utilization. */
+ private MonitoringThread monitoringThread;
+
+ /** Resource calculator. */
+ private ResourceCalculatorPlugin resourceCalculatorPlugin;
+
+ /** Current resource utilization of the node. */
+ private ResourceUtilization nodeUtilization;
+
+ /**
+ * Initialize the node resource monitor.
+ */
public NodeResourceMonitorImpl() {
super(NodeResourceMonitorImpl.class.getName());
+
+ this.monitoringThread = new MonitoringThread();
+ }
+
+ /**
+ * Initialize the service with the proper parameters.
+ */
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ this.monitoringInterval =
+ conf.getLong(YarnConfiguration.NM_USAGE_MON_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_NM_USAGE_MON_INTERVAL_MS);
+
+ Class extends ResourceCalculatorPlugin> clazz =
+ conf.getClass(YarnConfiguration.NM_NODE_MON_RESOURCE_CALCULATOR, null,
+ ResourceCalculatorPlugin.class);
+
+ this.resourceCalculatorPlugin =
+ ResourceCalculatorPlugin.getResourceCalculatorPlugin(clazz, conf);
+
+ LOG.info(" Using ResourceCalculatorPlugin : "
+ + this.resourceCalculatorPlugin);
}
+ /**
+ * Check if we should be monitoring.
+ * @return true if we can monitor the node resource utilization.
+ */
+ private boolean isEnabled() {
+ if (resourceCalculatorPlugin == null) {
+ LOG.info("ResourceCalculatorPlugin is unavailable on this system. "
+ + this.getClass().getName() + " is disabled.");
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Start the thread that does the node resource utilization monitoring.
+ */
+ @Override
+ protected void serviceStart() throws Exception {
+ if (this.isEnabled()) {
+ this.monitoringThread.start();
+ }
+ super.serviceStart();
+ }
+
+ /**
+ * Stop the thread that does the node resource utilization monitoring.
+ */
+ @Override
+ protected void serviceStop() throws Exception {
+ if (this.isEnabled()) {
+ this.monitoringThread.interrupt();
+ try {
+ this.monitoringThread.join(10 * 1000);
+ } catch (InterruptedException e) {
+ LOG.info("Could not wait for the node resource monitoring thread");
+ }
+ }
+ super.serviceStop();
+ }
+
+ /**
+ * Thread that monitors the resource utilization of this node.
+ */
+ private class MonitoringThread extends Thread {
+ /**
+ * Initialize the node resource monitoring thread.
+ */
+ public MonitoringThread() {
+ super("Node Resource Monitor");
+ this.setDaemon(true);
+ }
+
+ /**
+ * Periodically monitor the resource utilization of the node.
+ */
+ @Override
+ public void run() {
+ while (true) {
+ // Get node utilization and save it into the health status
+ long pmem = resourceCalculatorPlugin.getPhysicalMemorySize() -
+ resourceCalculatorPlugin.getAvailablePhysicalMemorySize();
+ long vmem =
+ resourceCalculatorPlugin.getVirtualMemorySize()
+ - resourceCalculatorPlugin.getAvailableVirtualMemorySize();
+ float cpu = resourceCalculatorPlugin.getCpuUsage();
+ nodeUtilization =
+ ResourceUtilizations.createResourceUtilization(
+ (int) (pmem >> 20), // B -> MB
+ (int) (vmem >> 20), // B -> MB
+ cpu); // 1 CPU at 100% is 1
+
+ try {
+ Thread.sleep(monitoringInterval);
+ } catch (InterruptedException e) {
+ LOG.warn(NodeResourceMonitorImpl.class.getName()
+ + " is interrupted. Exiting.");
+ break;
+ }
+ }
+ }
+ }
+
+ /**
+ * Get the resource utilization of the node.
+ * @return resource utilization of the node.
+ */
+ @Override
+ public ResourceUtilization getUtilization() {
+ return this.nodeUtilization;
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeResourceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeResourceMonitor.java
new file mode 100644
index 0000000..7804f4f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeResourceMonitor.java
@@ -0,0 +1,12 @@
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+
+public class TestNodeResourceMonitor extends BaseContainerManagerTest {
+
+ public TestNodeResourceMonitor() throws UnsupportedFileSystemException {
+ super();
+ }
+
+}