diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java index a648fef..0f15bbc 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java @@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api.records; import java.nio.ByteBuffer; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -55,6 +56,24 @@ @Stable public abstract class ContainerLaunchContext { + /** + * ContainerLaunchContext constant to start an unmanaged container. + *

+ * An unmanaged container is a container which does not start any process, it + * only reserves the allocated resources. An unmanaged container is useful + * when the allocated resources are to be used out of band from Yarn. Special + * care must be taken by the system using the resources out of band to ensure + * the resources being utilized on behalf of the unmanaged container do not + * exceed the allocated resources and that if the Yarn container terminates + * (lost, preempted, killed) the resources are immediately released. + */ + @Public + @Stable + public static final ContainerLaunchContext UNMANAGED_CONTAINER = + newInstance(Collections.EMPTY_MAP, Collections.EMPTY_MAP, + Collections.EMPTY_LIST, Collections.EMPTY_MAP, null, + Collections.EMPTY_MAP); + @Public @Stable public static ContainerLaunchContext newInstance( diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java index 126dfcb..dad8d67 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java @@ -203,10 +203,21 @@ private void stopNmClient(boolean stopContainers) { @Test (timeout = 180000) public void testNMClientNoCleanupOnStop() throws YarnException, IOException { + testNMClientNoCleanupOnStop(false); + } - rmClient.registerApplicationMaster("Host", 10000, ""); + @Test(timeout = 180000) + public void testNMClientNoCleanupOnStopUnmanagedContainers() + throws YarnException, IOException { + testNMClientNoCleanupOnStop(true); + } - testContainerManagement(nmClient, allocateContainers(rmClient, 5)); + private void testNMClientNoCleanupOnStop (boolean unmanagedContainers) + throws YarnException, IOException { + rmClient.registerApplicationMaster("Host", 10000, ""); + + testContainerManagement(nmClient, allocateContainers(rmClient, 5), + unmanagedContainers); rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null); @@ -221,9 +232,21 @@ public void testNMClientNoCleanupOnStop() @Test (timeout = 200000) public void testNMClient() throws YarnException, IOException { + testNMClient(false); + } + + @Test(timeout = 200000) + public void testNMClientUnmanaged() + throws YarnException, IOException { + testNMClient(true); + } + + private void testNMClient(boolean unmanagedContainers) + throws YarnException, IOException { rmClient.registerApplicationMaster("Host", 10000, ""); - testContainerManagement(nmClient, allocateContainers(rmClient, 5)); + testContainerManagement(nmClient, allocateContainers(rmClient, 5), + unmanagedContainers); rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, null); @@ -283,7 +306,8 @@ public void testNMClient() } private void testContainerManagement(NMClientImpl nmClient, - Set containers) throws YarnException, IOException { + Set containers, boolean useUnmanagedContainers) + throws YarnException, IOException { int size = containers.size(); int i = 0; for (Container container : containers) { @@ -311,14 +335,19 @@ private void testContainerManagement(NMClientImpl nmClient, } } - Credentials ts = new Credentials(); - DataOutputBuffer dob = new DataOutputBuffer(); - ts.writeTokenStorageToStream(dob); - ByteBuffer securityTokens = - ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); - ContainerLaunchContext clc = - Records.newRecord(ContainerLaunchContext.class); - clc.setTokens(securityTokens); + ContainerLaunchContext clc; + if (!useUnmanagedContainers) { + Credentials ts = new Credentials(); + DataOutputBuffer dob = new DataOutputBuffer(); + ts.writeTokenStorageToStream(dob); + ByteBuffer securityTokens = + ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); + clc = Records.newRecord(ContainerLaunchContext.class); + clc.setTokens(securityTokens); + } else { + clc = ContainerLaunchContext.UNMANAGED_CONTAINER; + } + try { nmClient.startContainer(container, clc); } catch (YarnException e) { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java index e69e61a..4e8f2ae 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java @@ -21,6 +21,7 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.Credentials; import org.apache.hadoop.yarn.api.records.ContainerId; @@ -38,6 +39,8 @@ ContainerTokenIdentifier getContainerTokenIdentifier(); + Configuration getDaemonConf(); + String getUser(); ContainerState getContainerState(); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index c2d32b5..76432ac 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -321,6 +321,15 @@ public ContainerImpl(Configuration conf, Dispatcher dispatcher, } } + public Configuration getDaemonConf() { + this.readLock.lock(); + try { + return daemonConf; + } finally { + this.readLock.unlock(); + } + } + @Override public String getUser() { this.readLock.lock(); @@ -446,7 +455,7 @@ private void finished() { EventHandler eventHandler = dispatcher.getEventHandler(); eventHandler.handle(new ApplicationContainerFinishedEvent(containerId)); // Remove the container from the resource-monitor - eventHandler.handle(new ContainerStopMonitoringEvent(containerId)); + eventHandler.handle(new ContainerStopMonitoringEvent(this)); // Tell the logService too eventHandler.handle(new LogHandlerContainerFinishedEvent( containerId, exitCode)); @@ -626,16 +635,8 @@ public ContainerState transition(ContainerImpl container, public void transition(ContainerImpl container, ContainerEvent event) { // Inform the ContainersMonitor to start monitoring the container's // resource usage. - long pmemBytes = - container.getResource().getMemory() * 1024 * 1024L; - float pmemRatio = container.daemonConf.getFloat( - YarnConfiguration.NM_VMEM_PMEM_RATIO, - YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO); - long vmemBytes = (long) (pmemRatio * pmemBytes); - container.dispatcher.getEventHandler().handle( - new ContainerStartMonitoringEvent(container.containerId, - vmemBytes, pmemBytes)); + new ContainerStartMonitoringEvent(container)); container.metrics.runningContainer(); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index edc3146..b81975b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -34,6 +34,7 @@ import java.util.Map; import java.util.Map.Entry; import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; @@ -104,6 +105,8 @@ private final LocalDirsHandlerService dirsHandler; + private final CountDownLatch unmanagedContainerLatch = new CountDownLatch(1); + public ContainerLaunch(Context context, Configuration configuration, Dispatcher dispatcher, ContainerExecutor exec, Application app, Container container, LocalDirsHandlerService dirsHandler, @@ -146,6 +149,27 @@ public Integer call() { return 0; } + if (launchContext.equals(ContainerLaunchContext.UNMANAGED_CONTAINER)) { + // Unmanaged containers don't need any setup/localization, thus + // we simply dispatch a CONTAINER_LAUNCHED and then wait for the + // notify from cleanupContainer() + dispatcher.getEventHandler().handle(new ContainerEvent( + containerID, ContainerEventType.CONTAINER_LAUNCHED)); + LOG.debug("Launched unmanaged container " + containerIdStr); + try { + unmanagedContainerLatch.await(); + } catch (InterruptedException ex) { + LOG.warn("Unmanaged container " + containerIdStr + + " execution interrupted"); + } + LOG.info("Unmanaged container " + containerIdStr + " succeeded "); + dispatcher.getEventHandler().handle( + new ContainerEvent(containerID, + ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS)); + //unmanaged container exit code is always zero (no real process) + return 0; + } + try { localResources = container.getLocalizedResources(); if (localResources == null) { @@ -336,8 +360,15 @@ public Integer call() { */ @SuppressWarnings("unchecked") // dispatcher not typed public void cleanupContainer() throws IOException { + ContainerLaunchContext launchContext = container.getLaunchContext(); ContainerId containerId = container.getContainerId(); String containerIdStr = ConverterUtils.toString(containerId); + if (launchContext.equals(ContainerLaunchContext.UNMANAGED_CONTAINER)) { + //for unmanaged containers we just need to release the blocking call() + unmanagedContainerLatch.countDown(); + LOG.info("Cleaning up unmanaged container " + containerIdStr); + return; + } LOG.info("Cleaning up container " + containerIdStr); // launch flag will be set to true if process already launched diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java index 407ada5..12b11aa 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java @@ -18,18 +18,21 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor; -import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; public class ContainerStartMonitoringEvent extends ContainersMonitorEvent { private final long vmemLimit; private final long pmemLimit; - public ContainerStartMonitoringEvent(ContainerId containerId, - long vmemLimit, long pmemLimit) { - super(containerId, ContainersMonitorEventType.START_MONITORING_CONTAINER); - this.vmemLimit = vmemLimit; - this.pmemLimit = pmemLimit; + public ContainerStartMonitoringEvent(Container container) { + super(container, ContainersMonitorEventType.START_MONITORING_CONTAINER); + pmemLimit = container.getResource().getMemory() * 1024 * 1024L; + float pmemRatio = container.getDaemonConf().getFloat( + YarnConfiguration.NM_VMEM_PMEM_RATIO, + YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO); + vmemLimit = (long) (pmemRatio * pmemLimit); } public long getVmemLimit() { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStopMonitoringEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStopMonitoringEvent.java index 240c5c0..877905f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStopMonitoringEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStopMonitoringEvent.java @@ -18,12 +18,13 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor; -import org.apache.hadoop.yarn.api.records.ContainerId; + +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; public class ContainerStopMonitoringEvent extends ContainersMonitorEvent { - public ContainerStopMonitoringEvent(ContainerId containerId) { - super(containerId, ContainersMonitorEventType.STOP_MONITORING_CONTAINER); + public ContainerStopMonitoringEvent(Container container) { + super(container, ContainersMonitorEventType.STOP_MONITORING_CONTAINER); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEvent.java index 56e578b..9459002 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEvent.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEvent.java @@ -19,21 +19,31 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor; import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.event.AbstractEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; public class ContainersMonitorEvent extends AbstractEvent { - private final ContainerId containerId; + private final Container container; - public ContainersMonitorEvent(ContainerId containerId, + public ContainersMonitorEvent(Container container, ContainersMonitorEventType eventType) { super(eventType); - this.containerId = containerId; + this.container = container; + } + + protected Container getContainer() { + return container; } public ContainerId getContainerId() { - return this.containerId; + return this.container.getContainerId(); } + public boolean isUnmanagedContainer() { + return container.getLaunchContext().equals( + ContainerLaunchContext.UNMANAGED_CONTAINER); + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index b681b34..3f6e52b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -530,6 +530,11 @@ public void handle(ContainersMonitorEvent monitoringEvent) { return; } + if (monitoringEvent.isUnmanagedContainer()) { + //for unmanaged containers there is not process to monitor + return; + } + ContainerId containerId = monitoringEvent.getContainerId(); switch (monitoringEvent.getType()) { case START_MONITORING_CONTAINER: diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java index ebc400a..d6d4e1d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java @@ -26,6 +26,7 @@ import static org.mockito.Mockito.mock; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyZeroInteractions; import static org.mockito.Mockito.when; import java.io.IOException; @@ -45,6 +46,7 @@ import java.util.Random; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; @@ -515,7 +517,65 @@ public void testLaunchAfterKillRequest() throws Exception { } } } - + + @Test + public void testLaunchCleanUpUnmanagedContainer() throws Exception { + WrappedContainer wc = null; + try { + final CountDownLatch launchLatch = new CountDownLatch(1); + final CountDownLatch cleanUpLatch = new CountDownLatch(1); + wc = new WrappedContainer(14, 314159265358979L, 4344, "yak", + ContainerLaunchContext.UNMANAGED_CONTAINER); + wc.initContainer(); + wc.localizeResources(); + assertTrue(wc.c.getLocalizedResources().isEmpty()); + final WrappedContainer fwc = wc; + Thread t = new Thread() { + @Override + public void run() { + try { + launchLatch.countDown(); + fwc.launchContainer(); + cleanUpLatch.await(); + } catch (Throwable ex) { + Assert.fail(ex.toString()); + } + } + }; + t.setDaemon(true); + t.start(); + launchLatch.await(); + cleanUpLatch.countDown(); + t.join(); + verifyZeroInteractions(wc.localizerBus); + } finally { + if (wc != null) { + wc.finished(); + } + } + } + + @Test + public void testLaunchAfterKillUnmanagedContainer() throws Exception { + WrappedContainer wc = null; + try { + wc = new WrappedContainer(14, 314159265358979L, 4344, "yak", + ContainerLaunchContext.UNMANAGED_CONTAINER); + wc.initContainer(); + wc.localizeResources(); + wc.killContainer(); + assertEquals(ContainerState.KILLING, wc.c.getContainerState()); + wc.launchContainer(); + assertEquals(ContainerState.KILLING, wc.c.getContainerState()); + wc.containerKilledOnRequest(); + verifyCleanupCall(wc); + } finally { + if (wc != null) { + wc.finished(); + } + } + } + private void verifyCleanupCall(WrappedContainer wc) throws Exception { ResourcesReleasedMatcher matchesReq = new ResourcesReleasedMatcher(wc.localResources, EnumSet.of( @@ -634,6 +694,36 @@ public boolean matches(Object o) { return serviceData; } + private static ContainerLaunchContext createMockLaunchContext( + boolean withLocalRes, + boolean withServiceData) { + ContainerLaunchContext ctxt = mock(ContainerLaunchContext.class); + Map localResources; + if (withLocalRes) { + Random r = new Random(); + long seed = r.nextLong(); + r.setSeed(seed); + System.out.println("WrappedContainerLocalResource seed: " + seed); + localResources = createLocalResources(r); + } else { + localResources = Collections.emptyMap(); + } + when(ctxt.getLocalResources()).thenReturn(localResources); + + Map serviceData; + if (withServiceData) { + Random r = new Random(); + long seed = r.nextLong(); + r.setSeed(seed); + System.out.println("ServiceData seed: " + seed); + serviceData = createServiceData(r); + } else { + serviceData = Collections.emptyMap(); + } + when(ctxt.getServiceData()).thenReturn(serviceData); + return ctxt; + } + @SuppressWarnings("unchecked") private class WrappedContainer { final DrainDispatcher dispatcher; @@ -656,9 +746,16 @@ public boolean matches(Object o) { this(appId, timestamp, id, user, true, false); } - @SuppressWarnings("rawtypes") WrappedContainer(int appId, long timestamp, int id, String user, boolean withLocalRes, boolean withServiceData) throws IOException { + this(appId, timestamp, id, user, createMockLaunchContext(withLocalRes, + withServiceData)); + } + + @SuppressWarnings("rawtypes") + WrappedContainer(int appId, long timestamp, int id, String user, + ContainerLaunchContext launchCtx) + throws IOException { dispatcher = new DrainDispatcher(); dispatcher.init(new Configuration()); @@ -693,7 +790,6 @@ public boolean matches(Object o) { launcher.start(); dispatcher.register(ContainersLauncherEventType.class, launcher); - ctxt = mock(ContainerLaunchContext.class); org.apache.hadoop.yarn.api.records.Container mockContainer = mock(org.apache.hadoop.yarn.api.records.Container.class); cId = BuilderUtils.newContainerId(appId, 1, timestamp, id); @@ -711,27 +807,10 @@ public boolean matches(Object o) { BuilderUtils.newContainerToken(BuilderUtils.newNodeId(host, port), "password".getBytes(), identifier); when(mockContainer.getContainerToken()).thenReturn(token); - if (withLocalRes) { - Random r = new Random(); - long seed = r.nextLong(); - r.setSeed(seed); - System.out.println("WrappedContainerLocalResource seed: " + seed); - localResources = createLocalResources(r); - } else { - localResources = Collections. emptyMap(); - } - when(ctxt.getLocalResources()).thenReturn(localResources); - - if (withServiceData) { - Random r = new Random(); - long seed = r.nextLong(); - r.setSeed(seed); - System.out.println("ServiceData seed: " + seed); - serviceData = createServiceData(r); - } else { - serviceData = Collections. emptyMap(); - } - when(ctxt.getServiceData()).thenReturn(serviceData); + + ctxt = launchCtx; + localResources = ctxt.getLocalResources(); + serviceData = ctxt.getServiceData(); c = new ContainerImpl(conf, dispatcher, ctxt, null, metrics, identifier); dispatcher.register(ContainerEventType.class, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index e0a4bfe..3b5860d 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verifyZeroInteractions; import java.io.BufferedReader; import java.io.File; @@ -40,6 +41,7 @@ import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; +import org.apache.hadoop.io.retry.AtMostOnce; import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest; import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest; @@ -64,6 +66,7 @@ import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.utils.BuilderUtils; import org.apache.hadoop.yarn.util.ConverterUtils; import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin; @@ -72,6 +75,8 @@ import org.apache.hadoop.yarn.util.TestProcfsBasedProcessTree; import org.junit.Before; import org.junit.Test; +import org.mockito.Mockito; +import org.mockito.internal.verification.AtMost; public class TestContainersMonitor extends BaseContainerManagerTest { @@ -339,4 +344,34 @@ private YarnConfiguration getConfForCM(boolean pMemEnabled, conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, vMemToPMemRatio); return conf; } + + @Test + public void testUnmanagedContainers() throws Exception { + //we simply verify nothing is done as part of a handle() invocation + ContainersMonitor cm = + new ContainersMonitorImpl(mock(ContainerExecutor.class), + mock(AsyncDispatcher.class), mock(Context.class)); + cm.init(getConfForCM(false, false, 8192, 2.1f)); + + Container container = mock(Container.class); + Mockito.when(container.getLaunchContext()).thenReturn( + ContainerLaunchContext.UNMANAGED_CONTAINER); + Resource r = BuilderUtils.newResource(1024, 1); + Mockito.when(container.getResource()).thenReturn(r); + Mockito.when(container.getDaemonConf()).thenReturn(new YarnConfiguration()); + ContainersMonitorEvent event = new ContainerStartMonitoringEvent(container); + event = Mockito.spy(event); + Mockito.when(event.getType()).thenReturn( + ContainersMonitorEventType.START_MONITORING_CONTAINER); + cm.handle(event); + Mockito.verify(event, new AtMost(0)).getType(); + + event = new ContainerStopMonitoringEvent(container); + event = Mockito.spy(event); + Mockito.when(event.getType()).thenReturn( + ContainersMonitorEventType.STOP_MONITORING_CONTAINER); + cm.handle(event); + Mockito.verify(event, new AtMost(0)).getType(); + + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index a021214..490ef56 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.api.records.ContainerStatus; import org.apache.hadoop.yarn.api.records.Resource; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.Dispatcher; import org.apache.hadoop.yarn.factories.RecordFactory; import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; @@ -76,6 +77,11 @@ public void setState(ContainerState state) { } @Override + public Configuration getDaemonConf() { + return new YarnConfiguration(); + } + + @Override public String getUser() { return user; }