diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
index a648fef..0f15bbc 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerLaunchContext.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.api.records;
import java.nio.ByteBuffer;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -55,6 +56,24 @@
@Stable
public abstract class ContainerLaunchContext {
+ /**
+ * ContainerLaunchContext constant to start an unmanaged container.
+ *
+ * An unmanaged container is a container which does not start any process, it
+ * only reserves the allocated resources. An unmanaged container is useful
+ * when the allocated resources are to be used out of band from Yarn. Special
+ * care must be taken by the system using the resources out of band to ensure
+ * the resources being utilized on behalf of the unmanaged container do not
+ * exceed the allocated resources and that if the Yarn container terminates
+ * (lost, preempted, killed) the resources are immediately released.
+ */
+ @Public
+ @Stable
+ public static final ContainerLaunchContext UNMANAGED_CONTAINER =
+ newInstance(Collections.EMPTY_MAP, Collections.EMPTY_MAP,
+ Collections.EMPTY_LIST, Collections.EMPTY_MAP, null,
+ Collections.EMPTY_MAP);
+
@Public
@Stable
public static ContainerLaunchContext newInstance(
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
index 126dfcb..dad8d67 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
@@ -203,10 +203,21 @@ private void stopNmClient(boolean stopContainers) {
@Test (timeout = 180000)
public void testNMClientNoCleanupOnStop()
throws YarnException, IOException {
+ testNMClientNoCleanupOnStop(false);
+ }
- rmClient.registerApplicationMaster("Host", 10000, "");
+ @Test(timeout = 180000)
+ public void testNMClientNoCleanupOnStopUnmanagedContainers()
+ throws YarnException, IOException {
+ testNMClientNoCleanupOnStop(true);
+ }
- testContainerManagement(nmClient, allocateContainers(rmClient, 5));
+ private void testNMClientNoCleanupOnStop (boolean unmanagedContainers)
+ throws YarnException, IOException {
+ rmClient.registerApplicationMaster("Host", 10000, "");
+
+ testContainerManagement(nmClient, allocateContainers(rmClient, 5),
+ unmanagedContainers);
rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
null, null);
@@ -221,9 +232,21 @@ public void testNMClientNoCleanupOnStop()
@Test (timeout = 200000)
public void testNMClient()
throws YarnException, IOException {
+ testNMClient(false);
+ }
+
+ @Test(timeout = 200000)
+ public void testNMClientUnmanaged()
+ throws YarnException, IOException {
+ testNMClient(true);
+ }
+
+ private void testNMClient(boolean unmanagedContainers)
+ throws YarnException, IOException {
rmClient.registerApplicationMaster("Host", 10000, "");
- testContainerManagement(nmClient, allocateContainers(rmClient, 5));
+ testContainerManagement(nmClient, allocateContainers(rmClient, 5),
+ unmanagedContainers);
rmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
null, null);
@@ -283,7 +306,8 @@ public void testNMClient()
}
private void testContainerManagement(NMClientImpl nmClient,
- Set containers) throws YarnException, IOException {
+ Set containers, boolean useUnmanagedContainers)
+ throws YarnException, IOException {
int size = containers.size();
int i = 0;
for (Container container : containers) {
@@ -311,14 +335,19 @@ private void testContainerManagement(NMClientImpl nmClient,
}
}
- Credentials ts = new Credentials();
- DataOutputBuffer dob = new DataOutputBuffer();
- ts.writeTokenStorageToStream(dob);
- ByteBuffer securityTokens =
- ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
- ContainerLaunchContext clc =
- Records.newRecord(ContainerLaunchContext.class);
- clc.setTokens(securityTokens);
+ ContainerLaunchContext clc;
+ if (!useUnmanagedContainers) {
+ Credentials ts = new Credentials();
+ DataOutputBuffer dob = new DataOutputBuffer();
+ ts.writeTokenStorageToStream(dob);
+ ByteBuffer securityTokens =
+ ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ clc = Records.newRecord(ContainerLaunchContext.class);
+ clc.setTokens(securityTokens);
+ } else {
+ clc = ContainerLaunchContext.UNMANAGED_CONTAINER;
+ }
+
try {
nmClient.startContainer(container, clc);
} catch (YarnException e) {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index e69e61a..4e8f2ae 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -21,6 +21,7 @@
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -38,6 +39,8 @@
ContainerTokenIdentifier getContainerTokenIdentifier();
+ Configuration getDaemonConf();
+
String getUser();
ContainerState getContainerState();
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index c2d32b5..76432ac 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -321,6 +321,15 @@ public ContainerImpl(Configuration conf, Dispatcher dispatcher,
}
}
+ public Configuration getDaemonConf() {
+ this.readLock.lock();
+ try {
+ return daemonConf;
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
@Override
public String getUser() {
this.readLock.lock();
@@ -446,7 +455,7 @@ private void finished() {
EventHandler eventHandler = dispatcher.getEventHandler();
eventHandler.handle(new ApplicationContainerFinishedEvent(containerId));
// Remove the container from the resource-monitor
- eventHandler.handle(new ContainerStopMonitoringEvent(containerId));
+ eventHandler.handle(new ContainerStopMonitoringEvent(this));
// Tell the logService too
eventHandler.handle(new LogHandlerContainerFinishedEvent(
containerId, exitCode));
@@ -626,16 +635,8 @@ public ContainerState transition(ContainerImpl container,
public void transition(ContainerImpl container, ContainerEvent event) {
// Inform the ContainersMonitor to start monitoring the container's
// resource usage.
- long pmemBytes =
- container.getResource().getMemory() * 1024 * 1024L;
- float pmemRatio = container.daemonConf.getFloat(
- YarnConfiguration.NM_VMEM_PMEM_RATIO,
- YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
- long vmemBytes = (long) (pmemRatio * pmemBytes);
-
container.dispatcher.getEventHandler().handle(
- new ContainerStartMonitoringEvent(container.containerId,
- vmemBytes, pmemBytes));
+ new ContainerStartMonitoringEvent(container));
container.metrics.runningContainer();
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index edc3146..b81975b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -34,6 +34,7 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
@@ -104,6 +105,8 @@
private final LocalDirsHandlerService dirsHandler;
+ private final CountDownLatch unmanagedContainerLatch = new CountDownLatch(1);
+
public ContainerLaunch(Context context, Configuration configuration,
Dispatcher dispatcher, ContainerExecutor exec, Application app,
Container container, LocalDirsHandlerService dirsHandler,
@@ -146,6 +149,27 @@ public Integer call() {
return 0;
}
+ if (launchContext.equals(ContainerLaunchContext.UNMANAGED_CONTAINER)) {
+ // Unmanaged containers don't need any setup/localization, thus
+ // we simply dispatch a CONTAINER_LAUNCHED and then wait for the
+ // notify from cleanupContainer()
+ dispatcher.getEventHandler().handle(new ContainerEvent(
+ containerID, ContainerEventType.CONTAINER_LAUNCHED));
+ LOG.debug("Launched unmanaged container " + containerIdStr);
+ try {
+ unmanagedContainerLatch.await();
+ } catch (InterruptedException ex) {
+ LOG.warn("Unmanaged container " + containerIdStr +
+ " execution interrupted");
+ }
+ LOG.info("Unmanaged container " + containerIdStr + " succeeded ");
+ dispatcher.getEventHandler().handle(
+ new ContainerEvent(containerID,
+ ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS));
+ //unmanaged container exit code is always zero (no real process)
+ return 0;
+ }
+
try {
localResources = container.getLocalizedResources();
if (localResources == null) {
@@ -336,8 +360,15 @@ public Integer call() {
*/
@SuppressWarnings("unchecked") // dispatcher not typed
public void cleanupContainer() throws IOException {
+ ContainerLaunchContext launchContext = container.getLaunchContext();
ContainerId containerId = container.getContainerId();
String containerIdStr = ConverterUtils.toString(containerId);
+ if (launchContext.equals(ContainerLaunchContext.UNMANAGED_CONTAINER)) {
+ //for unmanaged containers we just need to release the blocking call()
+ unmanagedContainerLatch.countDown();
+ LOG.info("Cleaning up unmanaged container " + containerIdStr);
+ return;
+ }
LOG.info("Cleaning up container " + containerIdStr);
// launch flag will be set to true if process already launched
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java
index 407ada5..12b11aa 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStartMonitoringEvent.java
@@ -18,18 +18,21 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
-import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
public class ContainerStartMonitoringEvent extends ContainersMonitorEvent {
private final long vmemLimit;
private final long pmemLimit;
- public ContainerStartMonitoringEvent(ContainerId containerId,
- long vmemLimit, long pmemLimit) {
- super(containerId, ContainersMonitorEventType.START_MONITORING_CONTAINER);
- this.vmemLimit = vmemLimit;
- this.pmemLimit = pmemLimit;
+ public ContainerStartMonitoringEvent(Container container) {
+ super(container, ContainersMonitorEventType.START_MONITORING_CONTAINER);
+ pmemLimit = container.getResource().getMemory() * 1024 * 1024L;
+ float pmemRatio = container.getDaemonConf().getFloat(
+ YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
+ vmemLimit = (long) (pmemRatio * pmemLimit);
}
public long getVmemLimit() {
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStopMonitoringEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStopMonitoringEvent.java
index 240c5c0..877905f 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStopMonitoringEvent.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainerStopMonitoringEvent.java
@@ -18,12 +18,13 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
-import org.apache.hadoop.yarn.api.records.ContainerId;
+
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
public class ContainerStopMonitoringEvent extends ContainersMonitorEvent {
- public ContainerStopMonitoringEvent(ContainerId containerId) {
- super(containerId, ContainersMonitorEventType.STOP_MONITORING_CONTAINER);
+ public ContainerStopMonitoringEvent(Container container) {
+ super(container, ContainersMonitorEventType.STOP_MONITORING_CONTAINER);
}
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEvent.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEvent.java
index 56e578b..9459002 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEvent.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorEvent.java
@@ -19,21 +19,31 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
public class ContainersMonitorEvent extends
AbstractEvent {
- private final ContainerId containerId;
+ private final Container container;
- public ContainersMonitorEvent(ContainerId containerId,
+ public ContainersMonitorEvent(Container container,
ContainersMonitorEventType eventType) {
super(eventType);
- this.containerId = containerId;
+ this.container = container;
+ }
+
+ protected Container getContainer() {
+ return container;
}
public ContainerId getContainerId() {
- return this.containerId;
+ return this.container.getContainerId();
}
+ public boolean isUnmanagedContainer() {
+ return container.getLaunchContext().equals(
+ ContainerLaunchContext.UNMANAGED_CONTAINER);
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index b681b34..3f6e52b 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -530,6 +530,11 @@ public void handle(ContainersMonitorEvent monitoringEvent) {
return;
}
+ if (monitoringEvent.isUnmanagedContainer()) {
+ //for unmanaged containers there is not process to monitor
+ return;
+ }
+
ContainerId containerId = monitoringEvent.getContainerId();
switch (monitoringEvent.getType()) {
case START_MONITORING_CONTAINER:
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index ebc400a..d6d4e1d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -26,6 +26,7 @@
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
import static org.mockito.Mockito.when;
import java.io.IOException;
@@ -45,6 +46,7 @@
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
@@ -515,7 +517,65 @@ public void testLaunchAfterKillRequest() throws Exception {
}
}
}
-
+
+ @Test
+ public void testLaunchCleanUpUnmanagedContainer() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ final CountDownLatch launchLatch = new CountDownLatch(1);
+ final CountDownLatch cleanUpLatch = new CountDownLatch(1);
+ wc = new WrappedContainer(14, 314159265358979L, 4344, "yak",
+ ContainerLaunchContext.UNMANAGED_CONTAINER);
+ wc.initContainer();
+ wc.localizeResources();
+ assertTrue(wc.c.getLocalizedResources().isEmpty());
+ final WrappedContainer fwc = wc;
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ launchLatch.countDown();
+ fwc.launchContainer();
+ cleanUpLatch.await();
+ } catch (Throwable ex) {
+ Assert.fail(ex.toString());
+ }
+ }
+ };
+ t.setDaemon(true);
+ t.start();
+ launchLatch.await();
+ cleanUpLatch.countDown();
+ t.join();
+ verifyZeroInteractions(wc.localizerBus);
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
+
+ @Test
+ public void testLaunchAfterKillUnmanagedContainer() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(14, 314159265358979L, 4344, "yak",
+ ContainerLaunchContext.UNMANAGED_CONTAINER);
+ wc.initContainer();
+ wc.localizeResources();
+ wc.killContainer();
+ assertEquals(ContainerState.KILLING, wc.c.getContainerState());
+ wc.launchContainer();
+ assertEquals(ContainerState.KILLING, wc.c.getContainerState());
+ wc.containerKilledOnRequest();
+ verifyCleanupCall(wc);
+ } finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
+
private void verifyCleanupCall(WrappedContainer wc) throws Exception {
ResourcesReleasedMatcher matchesReq =
new ResourcesReleasedMatcher(wc.localResources, EnumSet.of(
@@ -634,6 +694,36 @@ public boolean matches(Object o) {
return serviceData;
}
+ private static ContainerLaunchContext createMockLaunchContext(
+ boolean withLocalRes,
+ boolean withServiceData) {
+ ContainerLaunchContext ctxt = mock(ContainerLaunchContext.class);
+ Map localResources;
+ if (withLocalRes) {
+ Random r = new Random();
+ long seed = r.nextLong();
+ r.setSeed(seed);
+ System.out.println("WrappedContainerLocalResource seed: " + seed);
+ localResources = createLocalResources(r);
+ } else {
+ localResources = Collections.emptyMap();
+ }
+ when(ctxt.getLocalResources()).thenReturn(localResources);
+
+ Map serviceData;
+ if (withServiceData) {
+ Random r = new Random();
+ long seed = r.nextLong();
+ r.setSeed(seed);
+ System.out.println("ServiceData seed: " + seed);
+ serviceData = createServiceData(r);
+ } else {
+ serviceData = Collections.emptyMap();
+ }
+ when(ctxt.getServiceData()).thenReturn(serviceData);
+ return ctxt;
+ }
+
@SuppressWarnings("unchecked")
private class WrappedContainer {
final DrainDispatcher dispatcher;
@@ -656,9 +746,16 @@ public boolean matches(Object o) {
this(appId, timestamp, id, user, true, false);
}
- @SuppressWarnings("rawtypes")
WrappedContainer(int appId, long timestamp, int id, String user,
boolean withLocalRes, boolean withServiceData) throws IOException {
+ this(appId, timestamp, id, user, createMockLaunchContext(withLocalRes,
+ withServiceData));
+ }
+
+ @SuppressWarnings("rawtypes")
+ WrappedContainer(int appId, long timestamp, int id, String user,
+ ContainerLaunchContext launchCtx)
+ throws IOException {
dispatcher = new DrainDispatcher();
dispatcher.init(new Configuration());
@@ -693,7 +790,6 @@ public boolean matches(Object o) {
launcher.start();
dispatcher.register(ContainersLauncherEventType.class, launcher);
- ctxt = mock(ContainerLaunchContext.class);
org.apache.hadoop.yarn.api.records.Container mockContainer =
mock(org.apache.hadoop.yarn.api.records.Container.class);
cId = BuilderUtils.newContainerId(appId, 1, timestamp, id);
@@ -711,27 +807,10 @@ public boolean matches(Object o) {
BuilderUtils.newContainerToken(BuilderUtils.newNodeId(host, port),
"password".getBytes(), identifier);
when(mockContainer.getContainerToken()).thenReturn(token);
- if (withLocalRes) {
- Random r = new Random();
- long seed = r.nextLong();
- r.setSeed(seed);
- System.out.println("WrappedContainerLocalResource seed: " + seed);
- localResources = createLocalResources(r);
- } else {
- localResources = Collections. emptyMap();
- }
- when(ctxt.getLocalResources()).thenReturn(localResources);
-
- if (withServiceData) {
- Random r = new Random();
- long seed = r.nextLong();
- r.setSeed(seed);
- System.out.println("ServiceData seed: " + seed);
- serviceData = createServiceData(r);
- } else {
- serviceData = Collections. emptyMap();
- }
- when(ctxt.getServiceData()).thenReturn(serviceData);
+
+ ctxt = launchCtx;
+ localResources = ctxt.getLocalResources();
+ serviceData = ctxt.getServiceData();
c = new ContainerImpl(conf, dispatcher, ctxt, null, metrics, identifier);
dispatcher.register(ContainerEventType.class,
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
index e0a4bfe..3b5860d 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
@@ -22,6 +22,7 @@
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verifyZeroInteractions;
import java.io.BufferedReader;
import java.io.File;
@@ -40,6 +41,7 @@
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.io.retry.AtMostOnce;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
@@ -64,6 +66,7 @@
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
@@ -72,6 +75,8 @@
import org.apache.hadoop.yarn.util.TestProcfsBasedProcessTree;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.internal.verification.AtMost;
public class TestContainersMonitor extends BaseContainerManagerTest {
@@ -339,4 +344,34 @@ private YarnConfiguration getConfForCM(boolean pMemEnabled,
conf.setFloat(YarnConfiguration.NM_VMEM_PMEM_RATIO, vMemToPMemRatio);
return conf;
}
+
+ @Test
+ public void testUnmanagedContainers() throws Exception {
+ //we simply verify nothing is done as part of a handle() invocation
+ ContainersMonitor cm =
+ new ContainersMonitorImpl(mock(ContainerExecutor.class),
+ mock(AsyncDispatcher.class), mock(Context.class));
+ cm.init(getConfForCM(false, false, 8192, 2.1f));
+
+ Container container = mock(Container.class);
+ Mockito.when(container.getLaunchContext()).thenReturn(
+ ContainerLaunchContext.UNMANAGED_CONTAINER);
+ Resource r = BuilderUtils.newResource(1024, 1);
+ Mockito.when(container.getResource()).thenReturn(r);
+ Mockito.when(container.getDaemonConf()).thenReturn(new YarnConfiguration());
+ ContainersMonitorEvent event = new ContainerStartMonitoringEvent(container);
+ event = Mockito.spy(event);
+ Mockito.when(event.getType()).thenReturn(
+ ContainersMonitorEventType.START_MONITORING_CONTAINER);
+ cm.handle(event);
+ Mockito.verify(event, new AtMost(0)).getType();
+
+ event = new ContainerStopMonitoringEvent(container);
+ event = Mockito.spy(event);
+ Mockito.when(event.getType()).thenReturn(
+ ContainersMonitorEventType.STOP_MONITORING_CONTAINER);
+ cm.handle(event);
+ Mockito.verify(event, new AtMost(0)).getType();
+
+ }
}
diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index a021214..490ef56 100644
--- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -32,6 +32,7 @@
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -76,6 +77,11 @@ public void setState(ContainerState state) {
}
@Override
+ public Configuration getDaemonConf() {
+ return new YarnConfiguration();
+ }
+
+ @Override
public String getUser() {
return user;
}