diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 6a90079..a95397a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -437,6 +437,16 @@ public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb"; public static final int DEFAULT_NM_PMEM_MB = 8 * 1024; + /** Specifies whether physical memory check is enabled. */ + public static final String NM_PMEM_CHECK_ENABLED = NM_PREFIX + + "pmem-check-enabled"; + public static final boolean DEFAULT_NM_PMEM_CHECK_ENABLED = true; + + /** Specifies whether physical memory check is enabled. */ + public static final String NM_VMEM_CHECK_ENABLED = NM_PREFIX + + "vmem-check-enabled"; + public static final boolean DEFAULT_NM_VMEM_CHECK_ENABLED = true; + /** Conversion ratio for physical memory to virtual memory. */ public static final String NM_VMEM_PMEM_RATIO = NM_PREFIX + "vmem-pmem-ratio"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 08936a3..cfe97ca 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -448,6 +448,20 @@ + Whether physical memory limits will be enforced for + containers. + yarn.nodemanager.pmem-check-enabled + true + + + + Whether virtual memory limits will be enforced for + containers. + yarn.nodemanager.vmem-check-enabled + true + + + Ratio between virtual memory to physical memory when setting memory limits for containers. Container allocations are expressed in terms of physical memory, and virtual memory usage diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java index ab47fdd..9e9fdbe 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ResourceView.java @@ -21,6 +21,10 @@ public interface ResourceView { long getVmemAllocatedForContainers(); + + boolean isVmemCheckEnabled(); long getPmemAllocatedForContainers(); + + boolean isPmemCheckEnabled(); } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index d73f52c..052b0ce 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -63,14 +63,13 @@ private Configuration conf; private Class processTreeClass; - private long maxVmemAllottedForContainers = DISABLED_MEMORY_LIMIT; - private long maxPmemAllottedForContainers = DISABLED_MEMORY_LIMIT; + private long maxVmemAllottedForContainers = UNKNOWN_MEMORY_LIMIT; + private long maxPmemAllottedForContainers = UNKNOWN_MEMORY_LIMIT; + + private boolean pmemCheckEnabled; + private boolean vmemCheckEnabled; - /** - * A value which if set for memory related configuration options, indicates - * that the options are turned off. - */ - public static final long DISABLED_MEMORY_LIMIT = -1L; + private static final long UNKNOWN_MEMORY_LIMIT = -1L; public ContainersMonitorImpl(ContainerExecutor exec, AsyncDispatcher dispatcher, Context context) { @@ -104,65 +103,58 @@ public synchronized void init(Configuration conf) { LOG.info(" Using ResourceCalculatorProcessTree : " + this.processTreeClass); - long totalPhysicalMemoryOnNM = DISABLED_MEMORY_LIMIT; - if (this.resourceCalculatorPlugin != null) { - totalPhysicalMemoryOnNM = - this.resourceCalculatorPlugin.getPhysicalMemorySize(); - if (totalPhysicalMemoryOnNM <= 0) { - LOG.warn("NodeManager's totalPmem could not be calculated. " - + "Setting it to " + DISABLED_MEMORY_LIMIT); - totalPhysicalMemoryOnNM = DISABLED_MEMORY_LIMIT; - } - } + + long configuredPMemForContainers = conf.getLong( + YarnConfiguration.NM_PMEM_MB, + YarnConfiguration.DEFAULT_NM_PMEM_MB) * 1024 * 1024l; + // Setting these irrespective of whether checks are enabled. Required in the + // UI. // ///////// Physical memory configuration ////// - this.maxPmemAllottedForContainers = - conf.getLong(YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB); - this.maxPmemAllottedForContainers = - this.maxPmemAllottedForContainers * 1024 * 1024L; //Normalize to bytes - - if (totalPhysicalMemoryOnNM != DISABLED_MEMORY_LIMIT && - this.maxPmemAllottedForContainers > - totalPhysicalMemoryOnNM * 0.80f) { - LOG.warn("NodeManager configured with " + - TraditionalBinaryPrefix.long2String(maxPmemAllottedForContainers, "", 1) + - " physical memory allocated to containers, which is more than " + - "80% of the total physical memory available (" + - TraditionalBinaryPrefix.long2String(totalPhysicalMemoryOnNM, "", 1) + - "). Thrashing might happen."); - } - + this.maxPmemAllottedForContainers = configuredPMemForContainers; + // ///////// Virtual memory configuration ////// - float vmemRatio = conf.getFloat( - YarnConfiguration.NM_VMEM_PMEM_RATIO, + float vmemRatio = conf.getFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO); Preconditions.checkArgument(vmemRatio > 0.99f, - YarnConfiguration.NM_VMEM_PMEM_RATIO + - " should be at least 1.0"); - this.maxVmemAllottedForContainers = - (long)(vmemRatio * maxPmemAllottedForContainers); + YarnConfiguration.NM_VMEM_PMEM_RATIO + " should be at least 1.0"); + this.maxVmemAllottedForContainers = + (long) (vmemRatio * configuredPMemForContainers); + + pmemCheckEnabled = conf.getBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, + YarnConfiguration.DEFAULT_NM_PMEM_CHECK_ENABLED); + vmemCheckEnabled = conf.getBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, + YarnConfiguration.DEFAULT_NM_VMEM_CHECK_ENABLED); + LOG.info("Physical memory check enabled: " + pmemCheckEnabled); + LOG.info("Virtual memory check enabled: " + vmemCheckEnabled); + + if (pmemCheckEnabled) { + // Logging if actual pmem cannot be determined. + long totalPhysicalMemoryOnNM = UNKNOWN_MEMORY_LIMIT; + if (this.resourceCalculatorPlugin != null) { + totalPhysicalMemoryOnNM = this.resourceCalculatorPlugin + .getPhysicalMemorySize(); + if (totalPhysicalMemoryOnNM <= 0) { + LOG.warn("NodeManager's totalPmem could not be calculated. " + + "Setting it to " + UNKNOWN_MEMORY_LIMIT); + totalPhysicalMemoryOnNM = UNKNOWN_MEMORY_LIMIT; + } + } + if (totalPhysicalMemoryOnNM != UNKNOWN_MEMORY_LIMIT + && this.maxPmemAllottedForContainers > totalPhysicalMemoryOnNM * 0.80f) { + LOG.warn("NodeManager configured with " + + TraditionalBinaryPrefix.long2String(maxPmemAllottedForContainers, + "", 1) + + " physical memory allocated to containers, which is more than " + + "80% of the total physical memory available (" + + TraditionalBinaryPrefix.long2String(totalPhysicalMemoryOnNM, "", + 1) + "). Thrashing might happen."); + } + } super.init(conf); } - /** - * Is the total physical memory check enabled? - * - * @return true if total physical memory check is enabled. - */ - boolean isPhysicalMemoryCheckEnabled() { - return !(this.maxPmemAllottedForContainers == DISABLED_MEMORY_LIMIT); - } - - /** - * Is the total virtual memory check enabled? - * - * @return true if total virtual memory check is enabled. - */ - boolean isVirtualMemoryCheckEnabled() { - return !(this.maxVmemAllottedForContainers == DISABLED_MEMORY_LIMIT); - } - private boolean isEnabled() { if (resourceCalculatorPlugin == null) { LOG.info("ResourceCalculatorPlugin is unavailable on this system. " @@ -174,7 +166,7 @@ private boolean isEnabled() { + this.getClass().getName() + " is disabled."); return false; } - if (!(isPhysicalMemoryCheckEnabled() || isVirtualMemoryCheckEnabled())) { + if (!(isPmemCheckEnabled() || isVmemCheckEnabled())) { LOG.info("Neither virutal-memory nor physical-memory monitoring is " + "needed. Not running the monitor-thread"); return false; @@ -412,7 +404,7 @@ public void run() { boolean isMemoryOverLimit = false; String msg = ""; - if (isVirtualMemoryCheckEnabled() + if (isVmemCheckEnabled() && isProcessTreeOverLimit(containerId.toString(), currentVmemUsage, curMemUsageOfAgedProcesses, vmemLimit)) { // Container (the root process) is still alive and overflowing @@ -423,7 +415,7 @@ public void run() { currentPmemUsage, pmemLimit, pId, containerId, pTree); isMemoryOverLimit = true; - } else if (isPhysicalMemoryCheckEnabled() + } else if (isPmemCheckEnabled() && isProcessTreeOverLimit(containerId.toString(), currentPmemUsage, curRssMemUsageOfAgedProcesses, pmemLimit)) { @@ -507,11 +499,31 @@ public long getVmemAllocatedForContainers() { return this.maxVmemAllottedForContainers; } + /** + * Is the total physical memory check enabled? + * + * @return true if total physical memory check is enabled. + */ + @Override + public boolean isPmemCheckEnabled() { + return this.pmemCheckEnabled; + } + @Override public long getPmemAllocatedForContainers() { return this.maxPmemAllottedForContainers; } + /** + * Is the total virtual memory check enabled? + * + * @return true if total virtual memory check is enabled. + */ + @Override + public boolean isVmemCheckEnabled() { + return this.vmemCheckEnabled; + } + @Override public void handle(ContainersMonitorEvent monitoringEvent) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java index 9decac6..2831a18 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java @@ -67,8 +67,12 @@ protected void render(Block html) { info("NodeManager information") ._("Total Vmem allocated for Containers", StringUtils.byteDesc(info.getTotalVmemAllocated() * BYTES_IN_MB)) + ._("Vmem enforcement enabled", + info.isVmemCheckEnabled()) ._("Total Pmem allocated for Container", StringUtils.byteDesc(info.getTotalPmemAllocated() * BYTES_IN_MB)) + ._("Pmem enforcement enabled", + info.isVmemCheckEnabled()) ._("NodeHealthyStatus", info.getHealthStatus()) ._("LastNodeHealthTime", new Date( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java index 0eb1bef..458979b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/dao/NodeInfo.java @@ -36,6 +36,8 @@ protected String healthReport; protected long totalVmemAllocatedContainersMB; protected long totalPmemAllocatedContainersMB; + protected boolean vmemCheckEnabled; + protected boolean pmemCheckEnabled; protected long lastNodeUpdateTime; protected boolean nodeHealthy; protected String nodeManagerVersion; @@ -56,8 +58,10 @@ public NodeInfo(final Context context, final ResourceView resourceView) { this.nodeHostName = context.getNodeId().getHost(); this.totalVmemAllocatedContainersMB = resourceView .getVmemAllocatedForContainers() / BYTES_IN_MB; + this.vmemCheckEnabled = resourceView.isVmemCheckEnabled(); this.totalPmemAllocatedContainersMB = resourceView .getPmemAllocatedForContainers() / BYTES_IN_MB; + this.pmemCheckEnabled = resourceView.isPmemCheckEnabled(); this.nodeHealthy = context.getNodeHealthStatus().getIsNodeHealthy(); this.lastNodeUpdateTime = context.getNodeHealthStatus() .getLastHealthReportTime(); @@ -119,9 +123,17 @@ public String getHealthReport() { public long getTotalVmemAllocated() { return this.totalVmemAllocatedContainersMB; } + + public boolean isVmemCheckEnabled() { + return this.vmemCheckEnabled; + } public long getTotalPmemAllocated() { return this.totalPmemAllocatedContainersMB; } + + public boolean isPmemCheckEnabled() { + return this.pmemCheckEnabled; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index 41456fd..55033cc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -18,8 +18,10 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; import java.io.BufferedReader; import java.io.File; @@ -52,8 +54,11 @@ import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.api.records.URL; import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.event.AsyncDispatcher; +import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; @@ -281,4 +286,53 @@ public void testContainerKillOnMemoryOverflow() throws IOException, exec.signalContainer(user, pid, Signal.NULL)); } + + @Test(timeout = 20000) + public void testContainerMonitorMemFlags() { + ContainersMonitor cm = null; + + long expPmem = 8192 * 1024 * 1024l; + long expVmem = (long) (expPmem * 2.1f); + + cm = new ContainersMonitorImpl(mock(ContainerExecutor.class), + mock(AsyncDispatcher.class), mock(Context.class)); + cm.init(getConfForCM(false, false, 8192, 2.1f)); + assertEquals(expPmem, cm.getPmemAllocatedForContainers()); + assertEquals(expVmem, cm.getVmemAllocatedForContainers()); + assertEquals(false, cm.isPmemCheckEnabled()); + assertEquals(false, cm.isVmemCheckEnabled()); + + cm = new ContainersMonitorImpl(mock(ContainerExecutor.class), + mock(AsyncDispatcher.class), mock(Context.class)); + cm.init(getConfForCM(true, false, 8192, 2.1f)); + assertEquals(expPmem, cm.getPmemAllocatedForContainers()); + assertEquals(expVmem, cm.getVmemAllocatedForContainers()); + assertEquals(true, cm.isPmemCheckEnabled()); + assertEquals(false, cm.isVmemCheckEnabled()); + + cm = new ContainersMonitorImpl(mock(ContainerExecutor.class), + mock(AsyncDispatcher.class), mock(Context.class)); + cm.init(getConfForCM(true, true, 8192, 2.1f)); + assertEquals(expPmem, cm.getPmemAllocatedForContainers()); + assertEquals(expVmem, cm.getVmemAllocatedForContainers()); + assertEquals(true, cm.isPmemCheckEnabled()); + assertEquals(true, cm.isVmemCheckEnabled()); + + cm = new ContainersMonitorImpl(mock(ContainerExecutor.class), + mock(AsyncDispatcher.class), mock(Context.class)); + cm.init(getConfForCM(false, true, 8192, 2.1f)); + assertEquals(expPmem, cm.getPmemAllocatedForContainers()); + assertEquals(expVmem, cm.getVmemAllocatedForContainers()); + assertEquals(false, cm.isPmemCheckEnabled()); + assertEquals(true, cm.isVmemCheckEnabled()); + } + + private YarnConfiguration getConfForCM(boolean pMemEnabled, boolean vMemEnabled, int nmPmem, float vMemToPMemRatio) { + YarnConfiguration conf = new YarnConfiguration(); + conf.setInt(YarnConfiguration.NM_PMEM_MB, nmPmem); + conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, pMemEnabled); + conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, vMemEnabled); + conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, vMemToPMemRatio); + return conf; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java index 76ace14..d29e73e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java @@ -86,6 +86,14 @@ public long getVmemAllocatedForContainers() { public long getPmemAllocatedForContainers() { return 0; } + @Override + public boolean isVmemCheckEnabled() { + return true; + } + @Override + public boolean isPmemCheckEnabled() { + return true; + } }; Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); @@ -126,6 +134,14 @@ public long getVmemAllocatedForContainers() { public long getPmemAllocatedForContainers() { return 0; } + @Override + public boolean isVmemCheckEnabled() { + return true; + } + @Override + public boolean isPmemCheckEnabled() { + return true; + } }; Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java index 1a48db1..89441d7 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java @@ -100,6 +100,14 @@ public long getPmemAllocatedForContainers() { // 16G in bytes return new Long("17179869184"); } + @Override + public boolean isVmemCheckEnabled() { + return true; + } + @Override + public boolean isPmemCheckEnabled() { + return true; + } }; Configuration conf = new Configuration(); conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); @@ -294,6 +302,8 @@ public void verifyNodesXML(NodeList nodes) throws JSONException, Exception { "totalVmemAllocatedContainersMB"), WebServicesTestUtils.getXmlLong(element, "totalPmemAllocatedContainersMB"), + WebServicesTestUtils.getXmlBoolean(element, "vmemCheckEnabled"), + WebServicesTestUtils.getXmlBoolean(element, "pmemCheckEnabled"), WebServicesTestUtils.getXmlLong(element, "lastNodeUpdateTime"), WebServicesTestUtils.getXmlBoolean(element, "nodeHealthy"), WebServicesTestUtils.getXmlString(element, "nodeHostName"), @@ -310,10 +320,12 @@ public void verifyNodesXML(NodeList nodes) throws JSONException, Exception { public void verifyNodeInfo(JSONObject json) throws JSONException, Exception { assertEquals("incorrect number of elements", 1, json.length()); JSONObject info = json.getJSONObject("nodeInfo"); - assertEquals("incorrect number of elements", 13, info.length()); + assertEquals("incorrect number of elements", 15, info.length()); verifyNodeInfoGeneric(info.getString("id"), info.getString("healthReport"), info.getLong("totalVmemAllocatedContainersMB"), info.getLong("totalPmemAllocatedContainersMB"), + info.getBoolean("vmemCheckEnabled"), + info.getBoolean("pmemCheckEnabled"), info.getLong("lastNodeUpdateTime"), info.getBoolean("nodeHealthy"), info.getString("nodeHostName"), info.getString("hadoopVersionBuiltOn"), info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"), @@ -325,6 +337,7 @@ public void verifyNodeInfo(JSONObject json) throws JSONException, Exception { public void verifyNodeInfoGeneric(String id, String healthReport, long totalVmemAllocatedContainersMB, long totalPmemAllocatedContainersMB, + boolean vmemCheckEnabled, boolean pmemCheckEnabled, long lastNodeUpdateTime, Boolean nodeHealthy, String nodeHostName, String hadoopVersionBuiltOn, String hadoopBuildVersion, String hadoopVersion, String resourceManagerVersionBuiltOn, @@ -337,6 +350,8 @@ public void verifyNodeInfoGeneric(String id, String healthReport, totalVmemAllocatedContainersMB); assertEquals("totalPmemAllocatedContainersMB incorrect", 16384, totalPmemAllocatedContainersMB); + assertEquals("vmemCheckEnabled incorrect", true, vmemCheckEnabled); + assertEquals("pmemCheckEnabled incorrect", true, pmemCheckEnabled); assertTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime == nmContext .getNodeHealthStatus().getLastHealthReportTime()); assertTrue("nodeHealthy isn't true", nodeHealthy); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java index f0c5272..39764e6 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java @@ -106,6 +106,16 @@ public long getPmemAllocatedForContainers() { // 16G in bytes return new Long("17179869184"); } + + @Override + public boolean isVmemCheckEnabled() { + return true; + } + + @Override + public boolean isPmemCheckEnabled() { + return true; + } }; conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java index 8abf160..1f678d4 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java @@ -106,6 +106,16 @@ public long getPmemAllocatedForContainers() { // 16G in bytes return new Long("17179869184"); } + + @Override + public boolean isVmemCheckEnabled() { + return true; + } + + @Override + public boolean isPmemCheckEnabled() { + return true; + } }; conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath()); conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());