commit c0c644686e909400e243ff62d42a806e2b91dd8a Author: Vinod Kumar Vavilapalli Date: Tue May 19 16:33:35 2015 -0700 PID files manager. diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 79f9b0d..f7cd68c 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -26,11 +26,6 @@ import java.util.Arrays; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; -import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; @@ -39,6 +34,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.api.records.Resource; import org.apache.hadoop.yarn.conf.YarnConfiguration; @@ -53,8 +50,6 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext; import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext; import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader; -import org.apache.hadoop.util.Shell; -import org.apache.hadoop.util.StringUtils; public abstract class ContainerExecutor implements Configurable { @@ -64,12 +59,11 @@ private Configuration conf; - private ConcurrentMap pidFiles = - new ConcurrentHashMap(); - - private ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - private final ReadLock readLock = lock.readLock(); - private final WriteLock writeLock = lock.writeLock(); + // This doesn't really need to belong here anymore, but it is still here since + // the interaction between the executor, ContainersMonitor, ContainerLaunch is + // a little weird/non-linear. + public final ContainerPidFilesManager containerPidFilesManager = + new ContainerPidFilesManager(); @Override public void setConf(Configuration conf) { @@ -177,8 +171,7 @@ public int reacquireContainer(ContainerReacquisitionContext ctx) String user = ctx.getUser(); ContainerId containerId = ctx.getContainerId(); - - Path pidPath = getPidFilePath(containerId); + Path pidPath = containerPidFilesManager.getPidFilePath(containerId); if (pidPath == null) { LOG.warn(containerId + " is not active, returning terminated error"); return ExitCode.TERMINATED.getExitCode(); @@ -206,7 +199,7 @@ public int reacquireContainer(ContainerReacquisitionContext ctx) final int sleepMsec = 100; int msecLeft = 2000; while (!file.exists() && msecLeft >= 0) { - if (!isContainerActive(containerId)) { + if (!containerPidFilesManager.isContainerActive(containerId)) { LOG.info(containerId + " was deactivated"); return ExitCode.TERMINATED.getExitCode(); } @@ -319,20 +312,6 @@ protected void logOutput(String output) { } } - /** - * Get the pidFile of the container. - * @param containerId - * @return the path of the pid-file for the given containerId. - */ - protected Path getPidFilePath(ContainerId containerId) { - try { - readLock.lock(); - return (this.pidFiles.get(containerId)); - } finally { - readLock.unlock(); - } - } - protected String[] getRunCommand(String command, String groupId, String userName, Path pidFile, Configuration conf) { return getRunCommand(command, groupId, userName, pidFile, conf, null); @@ -400,74 +379,6 @@ protected Path getPidFilePath(ContainerId containerId) { } - /** - * Is the container still active? - * @param containerId - * @return true if the container is active else false. - */ - protected boolean isContainerActive(ContainerId containerId) { - try { - readLock.lock(); - return (this.pidFiles.containsKey(containerId)); - } finally { - readLock.unlock(); - } - } - - /** - * Mark the container as active - * - * @param containerId - * the ContainerId - * @param pidFilePath - * Path where the executor should write the pid of the launched - * process - */ - public void activateContainer(ContainerId containerId, Path pidFilePath) { - try { - writeLock.lock(); - this.pidFiles.put(containerId, pidFilePath); - } finally { - writeLock.unlock(); - } - } - - /** - * Mark the container as inactive. - * Done iff the container is still active. Else treat it as - * a no-op - */ - public void deactivateContainer(ContainerId containerId) { - try { - writeLock.lock(); - this.pidFiles.remove(containerId); - } finally { - writeLock.unlock(); - } - } - - /** - * Get the process-identifier for the container - * - * @param containerID - * @return the processid of the container if it has already launched, - * otherwise return null - */ - public String getProcessId(ContainerId containerID) { - String pid = null; - Path pidFile = pidFiles.get(containerID); - if (pidFile == null) { - // This container isn't even launched yet. - return pid; - } - try { - pid = ProcessIdFileReader.getProcessId(pidFile); - } catch (IOException e) { - LOG.error("Got exception reading pid from pid-file " + pidFile, e); - } - return pid; - } - public static class DelayedProcessKiller extends Thread { private Container container; private final String user; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerPidFilesManager.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerPidFilesManager.java new file mode 100644 index 0000000..b430686 --- /dev/null +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerPidFilesManager.java @@ -0,0 +1,117 @@ +package org.apache.hadoop.yarn.server.nodemanager; + +import java.io.IOException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock; +import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.yarn.api.records.ContainerId; +import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader; + +/** + * Memorize the PID-files corresponding to each container. More than remembering + * the path itself (which can be generated), this is a way of figuring out if a + * container really started or not. + */ +public class ContainerPidFilesManager { + + private static final Log LOG = LogFactory.getLog(ContainerPidFilesManager.class); + + ConcurrentMap pidFiles = + new ConcurrentHashMap(); + + private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); + private final ReadLock readLock = lock.readLock(); + private final WriteLock writeLock = lock.writeLock(); + + public ContainerPidFilesManager() { + super(); + } + + /** + * Get the pidFile of the container. + * @param containerId + * @return the path of the pid-file for the given containerId. + */ + protected Path getPidFilePath(ContainerId containerId) { + try { + readLock.lock(); + return (this.pidFiles.get(containerId)); + } finally { + readLock.unlock(); + } + } + + /** + * Is the container still active? + * @param containerId + * @return true if the container is active else false. + */ + protected boolean isContainerActive(ContainerId containerId) { + try { + readLock.lock(); + return (this.pidFiles.containsKey(containerId)); + } finally { + readLock.unlock(); + } + } + + /** + * Mark the container as active + * + * @param containerId + * the ContainerId + * @param pidFilePath + * Path where the executor should write the pid of the launched + * process + */ + public void activateContainer(ContainerId containerId, Path pidFilePath) { + try { + writeLock.lock(); + this.pidFiles.put(containerId, pidFilePath); + } finally { + writeLock.unlock(); + } + } + + /** + * Mark the container as inactive. + * Done iff the container is still active. Else treat it as + * a no-op + */ + public void deactivateContainer(ContainerId containerId) { + try { + writeLock.lock(); + this.pidFiles.remove(containerId); + } finally { + writeLock.unlock(); + } + } + + /** + * Get the process-identifier for the container + * + * @param containerID + * @return the processid of the container if it has already launched, + * otherwise return null + */ + public String getProcessId(ContainerId containerID) { + String pid = null; + Path pidFile = pidFiles.get(containerID); + if (pidFile == null) { + // This container isn't even launched yet. + return pid; + } + try { + pid = ProcessIdFileReader.getProcessId(pidFile); + } catch (IOException e) { + LOG.error("Got exception reading pid from pid-file " + pidFile, e); + } + return pid; + } +} \ No newline at end of file diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index b9be2b1..4723e22 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -199,7 +199,7 @@ public int launchContainer(ContainerStartContext ctx) throws IOException { WIN_MAX_PATH, YarnConfiguration.NM_LOCAL_DIRS)); } - Path pidFile = getPidFilePath(containerId); + Path pidFile = containerPidFilesManager.getPidFilePath(containerId); if (pidFile != null) { sb.writeLocalWrapperScript(launchDst, pidFile); } else { @@ -220,7 +220,7 @@ public int launchContainer(ContainerStartContext ctx) throws IOException { new File(containerWorkDir.toUri().getPath()), container.getLaunchContext().getEnvironment()); - if (isContainerActive(containerId)) { + if (containerPidFilesManager.isContainerActive(containerId)) { shExec.execute(); } else { diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java index d3b5d0a..c8a5fd3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java @@ -255,7 +255,7 @@ public int launchContainer(ContainerStartContext ctx) throws IOException { // Create new local launch wrapper script LocalWrapperScriptBuilder sb = new UnixLocalWrapperScriptBuilder( containerWorkDir, commandStr, dockerPidScript); - Path pidFile = getPidFilePath(containerId); + Path pidFile = containerPidFilesManager.getPidFilePath(containerId); if (pidFile != null) { sb.writeLocalWrapperScript(launchDst, pidFile); } else { @@ -284,7 +284,7 @@ public int launchContainer(ContainerStartContext ctx) throws IOException { command, new File(containerWorkDir.toUri().getPath()), container.getLaunchContext().getEnvironment()); // sanitized env - if (isContainerActive(containerId)) { + if (containerPidFilesManager.isContainerActive(containerId)) { shExec.execute(); } else { LOG.info("Container " + containerIdStr + diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index dbe257d..16defc8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -352,7 +352,7 @@ public int launchContainer(ContainerStartContext ctx) throws IOException { ShellCommandExecutor shExec = null; try { - Path pidFilePath = getPidFilePath(containerId); + Path pidFilePath = containerPidFilesManager.getPidFilePath(containerId); if (pidFilePath != null) { List command = new ArrayList(); addSchedPriorityCommand(command); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java index 494fa8f..50800f5 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java @@ -206,7 +206,7 @@ public ContainerManagerImpl(Context context, ContainerExecutor exec, addService(auxiliaryServices); this.containersMonitor = - new ContainersMonitorImpl(exec, dispatcher, this.context); + new ContainersMonitorImpl(exec.containerPidFilesManager, dispatcher); addService(this.containersMonitor); dispatcher.register(ContainerEventType.class, diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 0b7f762..4cfab28 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -24,7 +24,6 @@ import java.io.DataOutputStream; import java.io.File; import java.io.IOException; -import java.io.OutputStream; import java.io.PrintStream; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -61,6 +60,8 @@ import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService; +import org.apache.hadoop.yarn.server.nodemanager.ContainerPidFilesManager; +import org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl; import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -71,7 +72,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService; -import org.apache.hadoop.yarn.server.nodemanager.WindowsSecureContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext; import org.apache.hadoop.yarn.server.nodemanager.util.ProcessIdFileReader; @@ -94,6 +94,7 @@ protected final Dispatcher dispatcher; protected final ContainerExecutor exec; + private final ContainerPidFilesManager pidFilesManager; private final Application app; protected final Container container; private final Configuration conf; @@ -118,6 +119,7 @@ public ContainerLaunch(Context context, Configuration configuration, this.conf = configuration; this.app = app; this.exec = exec; + this.pidFilesManager = exec.containerPidFilesManager; this.container = container; this.dispatcher = dispatcher; this.dirsHandler = dirsHandler; @@ -300,7 +302,7 @@ public Integer call() { ret = ExitCode.TERMINATED.getExitCode(); } else { - exec.activateContainer(containerID, pidFilePath); + pidFilesManager.activateContainer(containerID, pidFilePath); ret = exec.launchContainer(new ContainerStartContext.Builder() .setContainer(container) .setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath) @@ -320,7 +322,7 @@ public Integer call() { return ret; } finally { completed.set(true); - exec.deactivateContainer(containerID); + pidFilesManager.deactivateContainer(containerID); try { context.getNMStateStore().storeContainerCompleted(containerID, ret); } catch (IOException e) { @@ -396,7 +398,7 @@ public void cleanupContainer() throws IOException { LOG.debug("Marking container " + containerIdStr + " as inactive"); // this should ensure that if the container process has not launched // by this time, it will never be launched - exec.deactivateContainer(containerId); + pidFilesManager.deactivateContainer(containerId); if (LOG.isDebugEnabled()) { LOG.debug("Getting pid for container " + containerIdStr + " to kill" diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java index fb10f22..c890d9f 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java @@ -80,7 +80,7 @@ public Integer call() { if (pidFile != null) { String pidPathStr = pidFile.getPath(); pidFilePath = new Path(pidPathStr); - exec.activateContainer(containerId, pidFilePath); + pidFilesManager.activateContainer(containerId, pidFilePath); retCode = exec.reacquireContainer( new ContainerReacquisitionContext.Builder() .setUser(container.getUser()) @@ -97,7 +97,7 @@ public Integer call() { } finally { if (notInterrupted) { this.completed.set(true); - exec.deactivateContainer(containerId); + pidFilesManager.deactivateContainer(containerId); try { getContext().getNMStateStore().storeContainerCompleted(containerId, retCode); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java index b5f154d..abaab0b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java @@ -35,12 +35,11 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.event.AsyncDispatcher; import org.apache.hadoop.yarn.event.Dispatcher; -import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; -import org.apache.hadoop.yarn.server.nodemanager.Context; +import org.apache.hadoop.yarn.server.nodemanager.ContainerPidFilesManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent; import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils; -import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin; +import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree; import com.google.common.base.Preconditions; @@ -60,9 +59,8 @@ Map trackingContainers = new HashMap(); - final ContainerExecutor containerExecutor; + final ContainerPidFilesManager pidFilesManager; private final Dispatcher eventDispatcher; - private final Context context; private ResourceCalculatorPlugin resourceCalculatorPlugin; private Configuration conf; private Class processTreeClass; @@ -78,13 +76,12 @@ private static final long UNKNOWN_MEMORY_LIMIT = -1L; private int nodeCpuPercentageForYARN; - public ContainersMonitorImpl(ContainerExecutor exec, - AsyncDispatcher dispatcher, Context context) { + public ContainersMonitorImpl(ContainerPidFilesManager pidFilesManager, + AsyncDispatcher dispatcher) { super("containers-monitor"); - this.containerExecutor = exec; + this.pidFilesManager = pidFilesManager; this.eventDispatcher = dispatcher; - this.context = context; this.containersToBeAdded = new HashMap(); this.containersToBeRemoved = new ArrayList(); @@ -402,7 +399,7 @@ public void run() { // Initialize any uninitialized processTrees if (pId == null) { // get pid from ContainerId - pId = containerExecutor.getProcessId(ptInfo.getContainerId()); + pId = pidFilesManager.getProcessId(ptInfo.getContainerId()); if (pId != null) { // pId will be null, either if the container is not spawned yet // or if the container's pid is removed from ContainerExecutor diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java index 58debc9..b56d3b1 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java @@ -327,7 +327,7 @@ private int runAndBlock(ContainerId cId, String... cmd) throws IOException { Path workDir = new Path(workSpace.getAbsolutePath()); Path pidFile = new Path(workDir, "pid.txt"); - exec.activateContainer(cId, pidFile); + exec.containerPidFilesManager.activateContainer(cId, pidFile); return exec.launchContainer(new ContainerStartContext.Builder() .setContainer(container) .setNmPrivateContainerScriptPath(scriptPath) @@ -460,7 +460,8 @@ public void run() { String pid = null; int count = 10; - while ((pid = exec.getProcessId(sleepId)) == null && count > 0) { + while ((pid = exec.containerPidFilesManager.getProcessId(sleepId)) == null + && count > 0) { LOG.info("Sleeping for 200 ms before checking for pid "); Thread.sleep(200); count--; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index d48ce13..7a00ff9 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -53,8 +53,8 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext; import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext; import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext; -import org.junit.Assert; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java index 909a962..00294b3 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java @@ -61,7 +61,6 @@ import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal; -import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerSignalContext; import org.apache.hadoop.yarn.server.utils.BuilderUtils; @@ -138,7 +137,7 @@ public void testProcessTreeLimits() throws IOException { // vmem limit long limit = 700000; - ContainersMonitorImpl test = new ContainersMonitorImpl(null, null, null); + ContainersMonitorImpl test = new ContainersMonitorImpl(null, null); // create process trees // tree rooted at 100 is over limit immediately, as it is @@ -304,7 +303,7 @@ public void testContainerMonitorMemFlags() { long expVmem = (long) (expPmem * 2.1f); cm = new ContainersMonitorImpl(mock(ContainerExecutor.class), - mock(AsyncDispatcher.class), mock(Context.class)); + mock(AsyncDispatcher.class)); cm.init(getConfForCM(false, false, 8192, 2.1f)); assertEquals(expPmem, cm.getPmemAllocatedForContainers()); assertEquals(expVmem, cm.getVmemAllocatedForContainers()); @@ -312,7 +311,7 @@ public void testContainerMonitorMemFlags() { assertEquals(false, cm.isVmemCheckEnabled()); cm = new ContainersMonitorImpl(mock(ContainerExecutor.class), - mock(AsyncDispatcher.class), mock(Context.class)); + mock(AsyncDispatcher.class)); cm.init(getConfForCM(true, false, 8192, 2.1f)); assertEquals(expPmem, cm.getPmemAllocatedForContainers()); assertEquals(expVmem, cm.getVmemAllocatedForContainers()); @@ -320,7 +319,7 @@ public void testContainerMonitorMemFlags() { assertEquals(false, cm.isVmemCheckEnabled()); cm = new ContainersMonitorImpl(mock(ContainerExecutor.class), - mock(AsyncDispatcher.class), mock(Context.class)); + mock(AsyncDispatcher.class)); cm.init(getConfForCM(true, true, 8192, 2.1f)); assertEquals(expPmem, cm.getPmemAllocatedForContainers()); assertEquals(expVmem, cm.getVmemAllocatedForContainers()); @@ -328,7 +327,7 @@ public void testContainerMonitorMemFlags() { assertEquals(true, cm.isVmemCheckEnabled()); cm = new ContainersMonitorImpl(mock(ContainerExecutor.class), - mock(AsyncDispatcher.class), mock(Context.class)); + mock(AsyncDispatcher.class)); cm.init(getConfForCM(false, true, 8192, 2.1f)); assertEquals(expPmem, cm.getPmemAllocatedForContainers()); assertEquals(expVmem, cm.getVmemAllocatedForContainers());