diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java index 9dc16c3..434f204 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandler.java @@ -71,7 +71,7 @@ public String getName() { } } - String CGROUP_FILE_TASKS = "tasks"; + String CGROUP_FILE_TASKS = "cgroup.procs"; String CGROUP_PARAM_CLASSID = "classid"; String CGROUP_PARAM_BLKIO_WEIGHT = "weight"; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/DefaultOOMHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/DefaultOOMHandler.java index 86137b5..e2ed8d8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/DefaultOOMHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/DefaultOOMHandler.java @@ -117,7 +117,8 @@ private boolean isContainerOutOfLimit(Container container) { * * @param container Container to clean up */ - private void sigKill(Container container) { + private boolean sigKill(Container container) { + boolean containerKilled = false; boolean finished = false; try { while (!finished) { @@ -154,11 +155,17 @@ private void sigKill(Container container) { LOG.debug("Interrupted while waiting for processes to disappear"); } } + containerKilled = true; } catch (ResourceHandlerException ex) { + // the tasks file of the container may not be available because the + // container may not have been launched at this point when the root + // cgroup is under oom LOG.warn(String.format( "Cannot list more tasks in container %s to kill.", container.getContainerId())); } + + return containerKilled; } /** @@ -216,19 +223,33 @@ protected boolean killContainer() { ArrayList candidates = new ArrayList<>(0); for (Container container : context.getContainers().values()) { + if (!container.isRunning()) { + // skip containers that are not running yet because killing them + // won't release any memory to get us out of OOM. + continue; + // note even if it is indicated that the container is running from + // container.isRunning(), the container process might not have been + // running yet. From NM's perspective, a container is running as + // soon as the container launch is handed over the container executor + } candidates.add( new ContainerCandidate(container, isContainerOutOfLimit(container))); } Collections.sort(candidates); + if (candidates.isEmpty()) { + LOG.warn("Found no running containers to kill in order to release memory"); + } - if (candidates.size() > 0) { - ContainerCandidate candidate = candidates.get(0); - sigKill(candidate.container); - String message = String.format( - "container %s killed by elastic cgroups OOM handler.", - candidate.container.getContainerId()); - LOG.warn(message); - containerKilled = true; + // make sure one container is killed successfully to release memory + for(int i = 0; !containerKilled && i < candidates.size(); i++) { + ContainerCandidate candidate = candidates.get(i); + if (sigKill(candidate.container)) { + String message = String.format( + "container %s killed by elastic cgroups OOM handler.", + candidate.container.getContainerId()); + LOG.warn(message); + containerKilled = true; + } } return containerKilled; } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java index e239067..dae656d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestDefaultOOMHandler.java @@ -75,16 +75,48 @@ protected CGroupsHandler getCGroupsHandler() { } /** - * We have two guaranteed containers, both of which are out of limit. + * Test an OOM situation where there are no running containers that can be killed. + */ + @Test(expected = YarnRuntimeException.class) + public void testExceptionThrownWithNoRunningContainersToKill() + throws Exception { + ConcurrentHashMap containers = + new ConcurrentHashMap<>(); + Container c1 = createContainer(1, true, 1L, false); + containers.put(c1.getContainerId(), c1); + + Context context = mock(Context.class); + when(context.getContainers()).thenReturn(containers); + + CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class); + when(cGroupsHandler.getCGroupParam( + CGroupsHandler.CGroupController.MEMORY, + "", + CGROUP_PARAM_MEMORY_OOM_CONTROL)) + .thenReturn("under_oom 1").thenReturn("under_oom 0"); + + DefaultOOMHandler handler = new DefaultOOMHandler(context, false) { + @Override + protected CGroupsHandler getCGroupsHandler() { + return cGroupsHandler; + } + }; + + handler.run(); + } + + /** + * We have two running guaranteed containers, both of which are out of limit. * We should kill the later one. */ @Test - public void testBothGuaranteedContainersOverLimitUponOOM() throws Exception { + public void testBothRunningGuaranteedContainersOverLimitUponOOM() + throws Exception { ConcurrentHashMap containers = new ConcurrentHashMap<>(); - Container c1 = createContainer(1, true, 1L); + Container c1 = createContainer(1, true, 1L, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(2, true, 2L); + Container c2 = createContainer(2, true, 2L, true); containers.put(c2.getContainerId(), c2); ContainerExecutor ex = createContainerExecutor(containers); @@ -139,7 +171,7 @@ protected CGroupsHandler getCGroupsHandler() { } /** - * We have two GUARANTEED containers, one of which is out of limit. + * We have two running GUARANTEED containers, one of which is out of limit. * We should kill the one that's out of its limit. This should * happen even if it was launched earlier than the other one. */ @@ -147,9 +179,9 @@ protected CGroupsHandler getCGroupsHandler() { public void testOneGuaranteedContainerOverLimitUponOOM() throws Exception { ConcurrentHashMap containers = new ConcurrentHashMap<>(); - Container c1 = createContainer(1, true, 2L); + Container c1 = createContainer(1, true, 2L, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(2, true, 1L); + Container c2 = createContainer(2, true, 1L, true); containers.put(c2.getContainerId(), c2); ContainerExecutor ex = createContainerExecutor(containers); @@ -204,16 +236,16 @@ protected CGroupsHandler getCGroupsHandler() { } /** - * We have two GUARANTEE containers, neither of which is out of limit. + * We have two running GUARANTEE containers, neither of which is out of limit. * We should kill the later launched one. */ @Test public void testNoGuaranteedContainerOverLimitOOM() throws Exception { ConcurrentHashMap containers = new ConcurrentHashMap<>(); - Container c1 = createContainer(1, true, 1L); + Container c1 = createContainer(1, true, 1L, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(2, true, 2L); + Container c2 = createContainer(2, true, 2L, true); containers.put(c2.getContainerId(), c2); ContainerExecutor ex = createContainerExecutor(containers); @@ -266,17 +298,249 @@ protected CGroupsHandler getCGroupsHandler() { } /** - * We have two opportunistic containers, both of which are out of limit. - * We should kill the later one. + * We have two OPPORTUNISTIC containers, one running and the other not. + * We should kill the running one. + */ + @Test + public void testKillOnlyRunningContainersUponOOM() throws Exception { + ConcurrentHashMap containers = + new ConcurrentHashMap<>(); + Container c1 = createContainer(1, false, 1L, false); + containers.put(c1.getContainerId(), c1); + Container c2 = createContainer(2, false, 2L, true); + containers.put(c2.getContainerId(), c2); + + ContainerExecutor ex = createContainerExecutor(containers); + Context context = mock(Context.class); + when(context.getContainers()).thenReturn(containers); + when(context.getContainerExecutor()).thenReturn(ex); + + CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class); + when(cGroupsHandler.getCGroupParam( + CGroupsHandler.CGroupController.MEMORY, + "", + CGROUP_PARAM_MEMORY_OOM_CONTROL)) + .thenReturn("under_oom 1").thenReturn("under_oom 0"); + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c2.getContainerId().toString(), CGROUP_FILE_TASKS)) + .thenReturn("1234").thenReturn(""); + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES)) + .thenReturn(getMB(9)); + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c2.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES)) + .thenReturn(getMB(9)); + + DefaultOOMHandler handler = + new DefaultOOMHandler(context, false) { + @Override + protected CGroupsHandler getCGroupsHandler() { + return cGroupsHandler; + } + }; + handler.run(); + + verify(ex, times(1)).signalContainer( + new ContainerSignalContext.Builder() + .setPid("1235") + .setContainer(c2) + .setSignal(ContainerExecutor.Signal.KILL) + .build() + ); + verify(ex, times(1)).signalContainer(any()); + } + + + /** + * We have two 'running' OPPORTUNISTIC containers. Killing the most- + * recently launched one fails because its cgroup.procs file is not + * available. The other OPPORTUNISTIC containers should be killed in + * this case. + */ + @Test + public void testKillOpportunisticContainerWithKillFailuresUponOOM() + throws Exception { + ConcurrentHashMap containers = + new ConcurrentHashMap<>(); + Container c1 = createContainer(1, false, 1L, true); + containers.put(c1.getContainerId(), c1); + Container c2 = createContainer(2, false, 2L, true); + containers.put(c2.getContainerId(), c2); + + ContainerExecutor ex = createContainerExecutor(containers); + Context context = mock(Context.class); + when(context.getContainers()).thenReturn(containers); + when(context.getContainerExecutor()).thenReturn(ex); + + CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class); + when(cGroupsHandler.getCGroupParam( + CGroupsHandler.CGroupController.MEMORY, + "", + CGROUP_PARAM_MEMORY_OOM_CONTROL)) + .thenReturn("under_oom 1").thenReturn("under_oom 0"); + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c1.getContainerId().toString(), CGROUP_FILE_TASKS)) + .thenReturn("1234").thenReturn(""); + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES)) + .thenReturn(getMB(9)); + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c1.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES)) + .thenReturn(getMB(9)); + // c2 process has not started, hence no cgroup.procs file yet + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c2.getContainerId().toString(), CGROUP_FILE_TASKS)) + .thenThrow( + new ResourceHandlerException(CGROUP_FILE_TASKS + " not found")); + + DefaultOOMHandler handler = + new DefaultOOMHandler(context, false) { + @Override + protected CGroupsHandler getCGroupsHandler() { + return cGroupsHandler; + } + }; + handler.run(); + + verify(ex, times(1)).signalContainer( + new ContainerSignalContext.Builder() + .setPid("1235") + .setContainer(c1) + .setSignal(ContainerExecutor.Signal.KILL) + .build() + ); + verify(ex, times(1)).signalContainer(any()); + } + + /** + * We have two 'running' OPPORTUNISTIC containers and one GUARANTEED + * container. Killing two OPPORTUNISTIC containers fails because they + * have not really started running as processes since the root cgroup + * is under oom. We should try to kill one container successfully. In + * this case, the GUARANTEED container should be killed. + */ + @Test + public void testKillGuaranteedContainerWithKillFailuresUponOOM() throws Exception { + ConcurrentHashMap containers = + new ConcurrentHashMap<>(); + Container c1 = createContainer(1, false, 1L, true); + containers.put(c1.getContainerId(), c1); + Container c2 = createContainer(2, false, 2L, true); + containers.put(c2.getContainerId(), c2); + Container c3 = createContainer(3, true, 2L, true); + containers.put(c3.getContainerId(), c3); + + ContainerExecutor ex = createContainerExecutor(containers); + Context context = mock(Context.class); + when(context.getContainers()).thenReturn(containers); + when(context.getContainerExecutor()).thenReturn(ex); + + CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class); + when(cGroupsHandler.getCGroupParam( + CGroupsHandler.CGroupController.MEMORY, + "", + CGROUP_PARAM_MEMORY_OOM_CONTROL)) + .thenReturn("under_oom 1").thenReturn("under_oom 0"); + // c1 process has not started, hence no cgroup.procs file yet + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c1.getContainerId().toString(), CGROUP_FILE_TASKS)) + .thenThrow( + new ResourceHandlerException(CGROUP_FILE_TASKS + " not found")); + // c2 process has not started, hence no cgroup.procs file yet + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c2.getContainerId().toString(), CGROUP_FILE_TASKS)) + .thenThrow( + new ResourceHandlerException(CGROUP_FILE_TASKS + " not found")); + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c3.getContainerId().toString(), CGROUP_FILE_TASKS)) + .thenReturn("1234").thenReturn(""); + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_USAGE_BYTES)) + .thenReturn(getMB(9)); + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c3.getContainerId().toString(), CGROUP_PARAM_MEMORY_MEMSW_USAGE_BYTES)) + .thenReturn(getMB(9)); + + DefaultOOMHandler handler = + new DefaultOOMHandler(context, false) { + @Override + protected CGroupsHandler getCGroupsHandler() { + return cGroupsHandler; + } + }; + handler.run(); + + verify(ex, times(1)).signalContainer( + new ContainerSignalContext.Builder() + .setPid("1235") + .setContainer(c3) + .setSignal(ContainerExecutor.Signal.KILL) + .build() + ); + verify(ex, times(1)).signalContainer(any()); + } + + /** + * Test an OOM situation where no containers are killed successfully. + * + * We have two 'running' containers, none of which are actually + * running as processes. Their cgroup.procs file is not available, + * so kill them won't succeed. + */ + @Test(expected = YarnRuntimeException.class) + public void testExceptionThrownWhenNoContainersKilledSuccessfully() + throws Exception { + ConcurrentHashMap containers = + new ConcurrentHashMap<>(); + Container c1 = createContainer(1, false, 1L, true); + containers.put(c1.getContainerId(), c1); + Container c2 = createContainer(2, false, 2L, true); + containers.put(c2.getContainerId(), c2); + + ContainerExecutor ex = createContainerExecutor(containers); + Context context = mock(Context.class); + when(context.getContainers()).thenReturn(containers); + when(context.getContainerExecutor()).thenReturn(ex); + + CGroupsHandler cGroupsHandler = mock(CGroupsHandler.class); + when(cGroupsHandler.getCGroupParam( + CGroupsHandler.CGroupController.MEMORY, + "", + CGROUP_PARAM_MEMORY_OOM_CONTROL)) + .thenReturn("under_oom 1").thenReturn("under_oom 0"); + // c1 process has not started, hence no cgroup.procs file yet + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c1.getContainerId().toString(), CGROUP_FILE_TASKS)) + .thenThrow( + new ResourceHandlerException(CGROUP_FILE_TASKS + " not found")); + // c2 process has not started, hence no cgroup.procs file yet + when(cGroupsHandler.getCGroupParam(CGroupsHandler.CGroupController.MEMORY, + c2.getContainerId().toString(), CGROUP_FILE_TASKS)) + .thenThrow( + new ResourceHandlerException(CGROUP_FILE_TASKS + " not found")); + + DefaultOOMHandler handler = + new DefaultOOMHandler(context, false) { + @Override + protected CGroupsHandler getCGroupsHandler() { + return cGroupsHandler; + } + }; + handler.run(); + } + + /** + * We have two running opportunistic containers, both of which are out of + * limit. We should kill the later one. */ @Test public void testBothOpportunisticContainersOverLimitUponOOM() throws Exception { ConcurrentHashMap containers = new ConcurrentHashMap<>(); - Container c1 = createContainer(1, false, 1L); + Container c1 = createContainer(1, false, 1L, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(2, false, 2L); + Container c2 = createContainer(2, false, 2L, true); containers.put(c2.getContainerId(), c2); ContainerExecutor ex = createContainerExecutor(containers); @@ -331,17 +595,17 @@ protected CGroupsHandler getCGroupsHandler() { } /** - * We have two OPPORTUNISTIC containers, one of which is out of limit. - * We should kill the one that's out of its limit. This should + * We have two running OPPORTUNISTIC containers, one of which is out of + * limit. We should kill the one that's out of its limit. This should * happen even if it was launched earlier than the other one. */ @Test public void testOneOpportunisticContainerOverLimitUponOOM() throws Exception { ConcurrentHashMap containers = new ConcurrentHashMap<>(); - Container c1 = createContainer(1, false, 2L); + Container c1 = createContainer(1, false, 2L, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(2, false, 1L); + Container c2 = createContainer(2, false, 1L, true); containers.put(c2.getContainerId(), c2); ContainerExecutor ex = createContainerExecutor(containers); @@ -395,16 +659,16 @@ protected CGroupsHandler getCGroupsHandler() { } /** - * We have two OPPORTUNISTIC containers, neither of which is out of limit. - * We should kill the later one. + * We have two running OPPORTUNISTIC containers, neither of which is out of + * limit. We should kill the later one. */ @Test public void testNoOpportunisticContainerOverLimitOOM() throws Exception { ConcurrentHashMap containers = new ConcurrentHashMap<>(); - Container c1 = createContainer(1, false, 1L); + Container c1 = createContainer(1, false, 1L, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(2, false, 2L); + Container c2 = createContainer(2, false, 2L, true); containers.put(c2.getContainerId(), c2); ContainerExecutor ex = createContainerExecutor(containers); @@ -457,8 +721,8 @@ protected CGroupsHandler getCGroupsHandler() { } /** - * We have two OPPORTUNISTIC containers and one GUARANTEED container. - * One of the OPPORTUNISTIC container is out of limit. + * We have two running OPPORTUNISTIC containers and one running GUARANTEED + * container. One of the OPPORTUNISTIC container is out of limit. * OOM is resolved after killing the OPPORTUNISTIC container that * exceeded its limit even though it is launched earlier than the * other OPPORTUNISTIC container. @@ -469,11 +733,11 @@ public void testKillOneOverLimitOpportunisticContainerUponOOM() ConcurrentHashMap containers = new ConcurrentHashMap<>(); int currentContainerId = 0; - Container c1 = createContainer(currentContainerId++, false, 2); + Container c1 = createContainer(currentContainerId++, false, 2, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(currentContainerId++, false, 1); + Container c2 = createContainer(currentContainerId++, false, 1, true); containers.put(c2.getContainerId(), c2); - Container c3 = createContainer(currentContainerId++, true, 1); + Container c3 = createContainer(currentContainerId++, true, 1, true); containers.put(c3.getContainerId(), c3); ContainerExecutor ex = createContainerExecutor(containers); @@ -538,8 +802,8 @@ protected CGroupsHandler getCGroupsHandler() { verify(ex, times(1)).signalContainer(any()); } /** - * We have two OPPORTUNISTIC containers and one GUARANTEED container. - * None of the containers exceeded its memory limit. + * We have two running OPPORTUNISTIC containers and one running GUARANTEED + * container. None of the containers exceeded its memory limit. * OOM is resolved after killing the most recently launched OPPORTUNISTIC * container. */ @@ -548,11 +812,11 @@ public void testKillOneLaterOpportunisticContainerUponOOM() throws Exception { ConcurrentHashMap containers = new ConcurrentHashMap<>(); int currentContainerId = 0; - Container c1 = createContainer(currentContainerId++, false, 1); + Container c1 = createContainer(currentContainerId++, false, 1, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(currentContainerId++, false, 2); + Container c2 = createContainer(currentContainerId++, false, 2, true); containers.put(c2.getContainerId(), c2); - Container c3 = createContainer(currentContainerId++, true, 1); + Container c3 = createContainer(currentContainerId++, true, 1, true); containers.put(c3.getContainerId(), c3); ContainerExecutor ex = createContainerExecutor(containers); @@ -615,8 +879,8 @@ protected CGroupsHandler getCGroupsHandler() { } /** - * We have two OPPORTUNISTIC containers and one GUARANTEED container. - * One of the OPPORTUNISTIC container is out of limit. + * We have two running OPPORTUNISTIC containers and one running GUARANTEED + * container. One of the OPPORTUNISTIC container is out of limit. * OOM is resolved after killing both OPPORTUNISTIC containers. */ @Test @@ -625,11 +889,11 @@ public void testKillBothOpportunisticContainerUponOOM() throws Exception { ConcurrentHashMap containers = new ConcurrentHashMap<>(); - Container c1 = createContainer(currentContainerId++, false, 2); + Container c1 = createContainer(currentContainerId++, false, 2, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(currentContainerId++, false, 1); + Container c2 = createContainer(currentContainerId++, false, 1, true); containers.put(c2.getContainerId(), c2); - Container c3 = createContainer(currentContainerId++, true, 1); + Container c3 = createContainer(currentContainerId++, true, 1, true); containers.put(c3.getContainerId(), c3); ContainerExecutor ex = createContainerExecutor(containers); @@ -701,8 +965,8 @@ protected CGroupsHandler getCGroupsHandler() { } /** - * We have two OPPORTUNISTIC containers and one GUARANTEED container. - * the GUARANTEED container is out of limit. OOM is resolved + * We have two running OPPORTUNISTIC containers and one running GUARANTEED + * container. The GUARANTEED container is out of limit. OOM is resolved * after first killing the two OPPORTUNISTIC containers and then the * GUARANTEED container. */ @@ -712,11 +976,11 @@ public void testKillGuaranteedContainerUponOOM() throws Exception { ConcurrentHashMap containers = new ConcurrentHashMap<>(); - Container c1 = createContainer(currentContainerId++, false, 2); + Container c1 = createContainer(currentContainerId++, false, 2, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(currentContainerId++, false, 1); + Container c2 = createContainer(currentContainerId++, false, 1, true); containers.put(c2.getContainerId(), c2); - Container c3 = createContainer(currentContainerId++, true, 1); + Container c3 = createContainer(currentContainerId++, true, 1, true); containers.put(c3.getContainerId(), c3); ContainerExecutor ex = createContainerExecutor(containers); @@ -795,8 +1059,8 @@ protected CGroupsHandler getCGroupsHandler() { } /** - * We have two OPPORTUNISTIC containers and one GUARANTEED container. - * None of the containers exceeded its memory limit. + * We have two running OPPORTUNISTIC containers and one running GUARANTEED + * container. None of the containers exceeded its memory limit. * OOM is resolved after killing all running containers. */ @Test @@ -805,11 +1069,11 @@ public void testKillAllContainersUponOOM() throws Exception { ConcurrentHashMap containers = new ConcurrentHashMap<>(); - Container c1 = createContainer(currentContainerId++, false, 1); + Container c1 = createContainer(currentContainerId++, false, 1, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(currentContainerId++, false, 2); + Container c2 = createContainer(currentContainerId++, false, 2, true); containers.put(c2.getContainerId(), c2); - Container c3 = createContainer(currentContainerId++, true, 1); + Container c3 = createContainer(currentContainerId++, true, 1, true); containers.put(c3.getContainerId(), c3); ContainerExecutor ex = createContainerExecutor(containers); @@ -888,7 +1152,8 @@ protected CGroupsHandler getCGroupsHandler() { } /** - * We have two OPPORTUNISTIC containers and one GUARANTEED container. + * We have two running OPPORTUNISTIC containers and one running + * GUARANTEED container. * None of the containers exceeded its memory limit. * OOM is not resolved even after killing all running containers. * A YarnRuntimeException is excepted to be thrown. @@ -899,11 +1164,11 @@ public void testOOMUnresolvedAfterKillingAllContainers() throws Exception { ConcurrentHashMap containers = new ConcurrentHashMap<>(); - Container c1 = createContainer(currentContainerId++, false, 1); + Container c1 = createContainer(currentContainerId++, false, 1, true); containers.put(c1.getContainerId(), c1); - Container c2 = createContainer(currentContainerId++, false, 2); + Container c2 = createContainer(currentContainerId++, false, 2, true); containers.put(c2.getContainerId(), c2); - Container c3 = createContainer(currentContainerId++, true, 3); + Container c3 = createContainer(currentContainerId++, true, 3, true); containers.put(c3.getContainerId(), c3); ContainerExecutor ex = createContainerExecutor(containers); @@ -974,7 +1239,7 @@ private static ContainerId createContainerId(int id) { } private static Container createContainer(int containerId, - boolean guaranteed, long launchTime) { + boolean guaranteed, long launchTime, boolean running) { Container c1 = mock(Container.class); ContainerId cid1 = createContainerId(containerId); when(c1.getContainerId()).thenReturn(cid1); @@ -987,6 +1252,7 @@ private static Container createContainer(int containerId, when(c1.getResource()).thenReturn(Resource.newInstance(10, 1)); when(c1.getContainerLaunchTime()).thenReturn(launchTime); + when(c1.isRunning()).thenReturn(running); return c1; }