diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java index a02eeef28b7..0776447bd80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClusterMetrics.java @@ -21,8 +21,12 @@ import static org.apache.hadoop.metrics2.lib.Interns.info; import java.util.Map; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.metrics2.MetricsInfo; import org.apache.hadoop.metrics2.MetricsSystem; @@ -66,6 +70,8 @@ rmEventProcCPUAvg; @Metric("RM Event Processor CPU Usage 60 second Max") MutableGaugeLong rmEventProcCPUMax; + @Metric("# of Containers assigned in last second") MutableGaugeInt + numContainerAssignedPerSecond; private boolean rmEventProcMonitorEnable = false; @@ -85,6 +91,29 @@ private static volatile ClusterMetrics INSTANCE = null; private static MetricsRegistry registry; + private volatile AtomicLong numContainersAssigned = new AtomicLong(0); + private long numContainersAssignedLast = 0; + + /** + * The executor service that count containers assigned in last second. + * + */ + private static final ScheduledThreadPoolExecutor containersAssignedCounter + = new ScheduledThreadPoolExecutor(1, new ThreadFactoryBuilder(). + setDaemon(true).setNameFormat("ContainersAssigned_Counter"). + build()); + + ClusterMetrics() { + containersAssignedCounter.scheduleAtFixedRate(new Runnable() { + @Override + public void run() { + long n = numContainersAssigned.get() - numContainersAssignedLast; + numContainersAssignedLast = numContainersAssigned.get(); + numContainerAssignedPerSecond.set((int)n); + } + }, 0, 1, TimeUnit.SECONDS); + } + public static ClusterMetrics getMetrics() { if(!isInitialized.get()){ synchronized (ClusterMetrics.class) { @@ -122,6 +151,9 @@ private static void registerMetrics() { public synchronized static void destroy() { isInitialized.set(false); INSTANCE = null; + if(containersAssignedCounter != null){ + containersAssignedCounter.shutdown(); + } } // Indicate whether RM Event Thread CPU Monitor is enabled @@ -319,4 +351,13 @@ public long getUtilizedVirtualCores() { public void incrUtilizedVirtualCores(long delta) { utilizedVirtualCores.incr(delta); } + + public int getnumContainerAssignedPerSecond() { + return numContainerAssignedPerSecond.value(); + } + + public void incrNumContainerAssigned() { + numContainersAssigned.incrementAndGet(); + } + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java index a3b235bc3c9..2b2f8324655 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java @@ -33,6 +33,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; +import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.classification.InterfaceAudience.Private; @@ -773,6 +774,7 @@ public static void updateMetrics(ApplicationId applicationId, NodeType type, containerAllocated.getContainer().getResource()); } queue.getMetrics().incrNodeTypeAggregations(user, type); + ClusterMetrics.getMetrics().incrNumContainerAssigned(); } // Get AppPlacementAllocator by specified schedulerKey diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClusterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClusterMetrics.java index d81e27860ec..14ab7fce92a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClusterMetrics.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClusterMetrics.java @@ -63,4 +63,13 @@ public void tearDown() { DefaultMetricsSystem.shutdown(); } } + + @Test + public void testClusterMetrics() throws Exception { + assert(metrics != null); + Assert.assertTrue(!metrics.numContainerAssignedPerSecond.changed()); + metrics.incrNumContainerAssigned(); + Thread.sleep(2000); + Assert.assertEquals(metrics.getnumContainerAssignedPerSecond(),1L); + } }