diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java index bb9c183..80d49c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ProcfsBasedProcessTree.java @@ -467,6 +467,14 @@ private BigInteger getTotalProcessJiffies() { return totalStime.add(BigInteger.valueOf(totalUtime)); } + /** + * Get the CPU usage by all the processes in the process-tree in Unix. + * Note: UNAVAILABLE will be returned in case when CPU usage is not + * available. It is NOT advised to return any other error code. + * + * @return percentage CPU usage since the process-tree was created, + * {@link #UNAVAILABLE} if CPU usage cannot be calculated or not available. + */ @Override public float getCpuUsagePercent() { BigInteger processTotalJiffies = getTotalProcessJiffies(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java index 7214c75..771ec86 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ResourceCalculatorProcessTree.java @@ -187,9 +187,11 @@ public long getCumulativeCpuTime() { * Get the CPU usage by all the processes in the process-tree based on * average between samples as a ratio of overall CPU cycles similar to top. * Thus, if 2 out of 4 cores are used this should return 200.0. + * Note: UNAVAILABLE will be returned in case when CPU usage is not + * available. It is NOT advised to return any other error code. * * @return percentage CPU usage since the process-tree was created, - * {@link #UNAVAILABLE} if it cannot be calculated. + * {@link #UNAVAILABLE} if CPU usage cannot be calculated or not available. */ public float getCpuUsagePercent() { return UNAVAILABLE; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java index 7858292..1c7eaf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/WindowsBasedProcessTree.java @@ -268,6 +268,14 @@ private BigInteger getTotalProcessMs() { return BigInteger.valueOf(totalMs); } + /** + * Get the CPU usage by all the processes in the process-tree in Windows. + * Note: UNAVAILABLE will be returned in case when CPU usage is not + * available. It is NOT advised to return any other error code. + * + * @return percentage CPU usage since the process-tree was created, + * {@link #UNAVAILABLE} if CPU usage cannot be calculated or not available. + */ @Override public float getCpuUsagePercent() { BigInteger processTotalMs = getTotalProcessMs(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index 3fbae10..0c6c21c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -455,6 +455,16 @@ public void run() { // cpuUsagePercentPerCore should be 300% and // cpuUsageTotalCoresPercentage should be 50% float cpuUsagePercentPerCore = pTree.getCpuUsagePercent(); + LOG.info("For me Sunil: " + cpuUsagePercentPerCore); + if (cpuUsagePercentPerCore < 0) { + // CPU usage is not available likely because the container just + // started. Let us skip this turn and consider this container + // in the next iteration. + LOG.info("Skipping monitoring container " + containerId + + " since CPU usage is not yet available."); + continue; + } + float cpuUsageTotalCoresPercentage = cpuUsagePercentPerCore / resourceCalculatorPlugin.getNumProcessors(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java new file mode 100644 index 0000000..18e3d90 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockCPUResourceCalculatorProcessTree.java @@ -0,0 +1,70 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor; + +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; + +public class MockCPUResourceCalculatorProcessTree extends ResourceCalculatorProcessTree { + + private long rssMemorySize = 0; + private long cpuPercentage = ResourceCalculatorProcessTree.UNAVAILABLE; + + public MockCPUResourceCalculatorProcessTree(String root) { + super(root); + } + + @Override + public void updateProcessTree() { + } + + @Override + public String getProcessTreeDump() { + return ""; + } + + @Override + public long getCumulativeCpuTime() { + return 0; + } + + @Override + public boolean checkPidPgrpidForMatch() { + return true; + } + + public void setRssMemorySize(long rssMemorySize) { + this.rssMemorySize = rssMemorySize; + } + + public long getRssMemorySize() { + return this.rssMemorySize; + } + + @Override + public float getCpuUsagePercent() { + long cpu = this.cpuPercentage; + // First getter call will be returned with -1, and other calls will + // return non-zero value as defined below. + if (cpu == ResourceCalculatorProcessTree.UNAVAILABLE) { + // Set a default value other than 0 for test. + this.cpuPercentage = 50; + } + return cpu; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorProcessTree.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorProcessTree.java index c5aaa77..d47f40a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorProcessTree.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/MockResourceCalculatorProcessTree.java @@ -23,6 +23,7 @@ public class MockResourceCalculatorProcessTree extends ResourceCalculatorProcessTree { private long rssMemorySize = 0; + private long cpuPercentage = 0; public MockResourceCalculatorProcessTree(String root) { super(root); @@ -54,4 +55,9 @@ public void setRssMemorySize(long rssMemorySize) { public long getRssMemorySize() { return this.rssMemorySize; } + + @Override + public float getCpuUsagePercent() { + return this.cpuPercentage; + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java index 1a0c690..f30570e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java @@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.ExecutionType; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.api.records.ResourceUtilization; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.EventHandler; @@ -48,6 +49,8 @@ import org.junit.Test; import org.mockito.Mockito; +import junit.framework.Assert; + import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertEquals; @@ -232,6 +235,40 @@ public void testContainersResourceChangeIsTriggeredImmediately() containersMonitor.stop(); } + @Test + public void testContainersCPUResourceForDefaultValue() throws Exception { + Configuration newConf = new Configuration(conf); + // set container monitor interval to be 20s + newConf.setLong(YarnConfiguration.NM_CONTAINER_MON_INTERVAL_MS, 20L); + containersMonitor = createContainersMonitor(executor, dispatcher, context); + newConf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, + MockCPUResourceCalculatorProcessTree.class.getCanonicalName()); + // set container monitor interval to be 20ms + containersMonitor.init(newConf); + containersMonitor.start(); + + // create container 1 + containersMonitor.handle(new ContainerStartMonitoringEvent( + getContainerId(1), 2100L, 1000L, 1, 0, 0)); + + // Verify the container utilization value. + // Since MockCPUResourceCalculatorProcessTree will return a -1 as CPU + // utilization, containersUtilization will not be calculated and hence it + // will be 0. + assertEquals(0, containersMonitor.getContainersUtilization() + .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f))); + // sleep 200ms to make sure the container monitor thread is + // now waiting for the next monitor cycle + Thread.sleep(200); + + // Verify the container utilization value. Since one round is done, we can + // expect a non-zero value for container utilization as + // MockCPUResourceCalculatorProcessTree#getCpuUsagePercent will return 50. + assertTrue(0 != containersMonitor.getContainersUtilization() + .compareTo(ResourceUtilization.newInstance(0, 0, 0.0f))); + containersMonitor.stop(); + } + private ContainersMonitorImpl createContainersMonitor( ContainerExecutor containerExecutor, AsyncDispatcher dispatcher, Context context) { -- 2.7.4 (Apple Git-66)