diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 8c83fea..82f8f57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -18,11 +18,6 @@ package org.apache.hadoop.yarn.conf; -import java.net.InetSocketAddress; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - import org.apache.hadoop.HadoopIllegalArgumentException; import org.apache.hadoop.classification.InterfaceAudience.Private; import org.apache.hadoop.classification.InterfaceAudience.Public; @@ -34,6 +29,11 @@ import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; +import java.net.InetSocketAddress; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + @Public @Evolving public class YarnConfiguration extends Configuration { @@ -931,13 +931,24 @@ private static void addDeprecatedKeys() { NM_PREFIX + "docker-container-executor.image-name"; /** The name of the docker executor (For DockerContainerExecutor).*/ - public static final String NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME = - NM_PREFIX + "docker-container-executor.exec-name"; + public static final String NM_DOCKER_CONTAINER_EXECUTOR_DOCKER_URL = + NM_PREFIX + "docker-container-executor.docker-url"; /** The default docker executor (For DockerContainerExecutor).*/ - public static final String NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME = - "/usr/bin/docker"; - + public static final String NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_DOCKER_URL = + "tcp://localhost:4243"; + /** + * True if linux-container-executor should limit itself to one user + * when running in non-secure mode. + */ + public static final String NM_DOCKER_NONSECURE_MODE_LIMIT_USERS = NM_PREFIX + + "docker-container-executor.nonsecure-mode.limit-users"; + /** + * The UNIX user that containers will run as when Docker-container-executor + * is used in nonsecure mode (a use case for this is using cgroups). + */ + public static final String NM_DOCKER_NONSECURE_MODE_LOCAL_USER_KEY = NM_PREFIX + + "docker-container-executor.nonsecure-mode.local-user"; /** The path to the Linux container executor.*/ public static final String NM_LINUX_CONTAINER_EXECUTOR_PATH = NM_PREFIX + "linux-container-executor.path"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java index 4e00a1c..0b05b1d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java @@ -304,7 +304,7 @@ public void run() { } } if (error) { - setSuccess(!error); + setSuccess(!error); } fileDeletionTaskFinished(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java index c854173..b378510 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java @@ -18,69 +18,51 @@ package org.apache.hadoop.yarn.server.nodemanager; -import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Joiner; import com.google.common.base.Preconditions; import com.google.common.base.Strings; - -import org.apache.commons.lang.math.RandomUtils; +import com.google.common.collect.Lists; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.UnsupportedFileSystemException; -import org.apache.hadoop.fs.permission.FsPermission; -import org.apache.hadoop.io.IOUtils; -import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.Shell.ShellCommandExecutor; import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.yarn.api.ApplicationConstants; import org.apache.hadoop.yarn.api.records.ContainerId; import org.apache.hadoop.yarn.conf.YarnConfiguration; -import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; -import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer; import org.apache.hadoop.yarn.util.ConverterUtils; import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.File; import java.io.IOException; import java.io.OutputStream; import java.io.PrintStream; import java.util.ArrayList; -import java.util.Collections; -import java.util.EnumSet; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Random; import java.util.Set; import java.util.regex.Pattern; -import java.net.InetSocketAddress; -import static org.apache.hadoop.fs.CreateFlag.CREATE; -import static org.apache.hadoop.fs.CreateFlag.OVERWRITE; /** * This executor will launch a docker container and run the task inside the container. */ -public class DockerContainerExecutor extends ContainerExecutor { +public class DockerContainerExecutor extends LinuxContainerExecutor { private static final Log LOG = LogFactory .getLog(DockerContainerExecutor.class); - public static final String DOCKER_CONTAINER_EXECUTOR_SCRIPT = "docker_container_executor"; - public static final String DOCKER_CONTAINER_EXECUTOR_SESSION_SCRIPT = "docker_container_executor_session"; - // This validates that the image is a proper docker image and would not crash docker. public static final String DOCKER_IMAGE_PATTERN = "^(([\\w\\.-]+)(:\\d+)*\\/)?[\\w\\.:-]+$"; private final FileContext lfs; private final Pattern dockerImagePattern; - public DockerContainerExecutor() { try { this.lfs = FileContext.getLocalFSFileContext(); @@ -96,47 +78,20 @@ protected void copyFile(Path src, Path dst, String owner) throws IOException { @Override public void init() throws IOException { + super.init(); String auth = getConf().get(CommonConfigurationKeys.HADOOP_SECURITY_AUTHENTICATION); if (auth != null && !auth.equals("simple")) { throw new IllegalStateException("DockerContainerExecutor only works with simple authentication mode"); } - String dockerExecutor = getConf().get(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME, - YarnConfiguration.NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME); - if (!new File(dockerExecutor).exists()) { - throw new IllegalStateException("Invalid docker exec path: " + dockerExecutor); + String dockerUrl = getConf().get(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_DOCKER_URL, + YarnConfiguration.NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_DOCKER_URL); + if (LOG.isDebugEnabled()) { + LOG.debug("dockerUrl: " + dockerUrl); } - } - - @Override - public synchronized void startLocalizer(Path nmPrivateContainerTokensPath, - InetSocketAddress nmAddr, String user, String appId, String locId, - LocalDirsHandlerService dirsHandler) - throws IOException, InterruptedException { - - List localDirs = dirsHandler.getLocalDirs(); - List logDirs = dirsHandler.getLogDirs(); - - ContainerLocalizer localizer = - new ContainerLocalizer(lfs, user, appId, locId, getPaths(localDirs), - RecordFactoryProvider.getRecordFactory(getConf())); - - createUserLocalDirs(localDirs, user); - createUserCacheDirs(localDirs, user); - createAppDirs(localDirs, user, appId); - createAppLogDirs(appId, logDirs, user); - - // randomly choose the local directory - Path appStorageDir = getWorkingDir(localDirs, user, appId); - - String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, locId); - Path tokenDst = new Path(appStorageDir, tokenFn); - copyFile(nmPrivateContainerTokensPath, tokenDst, user); - LOG.info("Copying from " + nmPrivateContainerTokensPath + " to " + tokenDst); - lfs.setWorkingDirectory(appStorageDir); - LOG.info("CWD set to " + appStorageDir + " = " + lfs.getWorkingDirectory()); - // TODO: DO it over RPC for maintaining similarity? - localizer.runLocalization(nmAddr); - } + if (Strings.isNullOrEmpty(dockerUrl)) { + throw new IllegalStateException("DockerUrl must be configured"); + } + } @Override @@ -153,95 +108,58 @@ public int launchContainer(Container container, containerImageName = containerImageName.replaceAll("['\"]", ""); Preconditions.checkArgument(saneDockerImage(containerImageName), "Image: " + containerImageName + " is not a proper docker image"); - String dockerExecutor = getConf().get(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME, - YarnConfiguration.NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME); + String dockerUrl = getConf().get(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_DOCKER_URL, + YarnConfiguration.NM_DEFAULT_DOCKER_CONTAINER_EXECUTOR_DOCKER_URL); - FsPermission dirPerm = new FsPermission(APPDIR_PERM); ContainerId containerId = container.getContainerId(); - // create container dirs on all disks String containerIdStr = ConverterUtils.toString(containerId); - String appIdStr = - ConverterUtils.toString( - containerId.getApplicationAttemptId(). - getApplicationId()); - for (String sLocalDir : localDirs) { - Path usersdir = new Path(sLocalDir, ContainerLocalizer.USERCACHE); - Path userdir = new Path(usersdir, userName); - Path appCacheDir = new Path(userdir, ContainerLocalizer.APPCACHE); - Path appDir = new Path(appCacheDir, appIdStr); - Path containerDir = new Path(appDir, containerIdStr); - createDir(containerDir, dirPerm, true, userName); - } - - // Create the container log-dirs on all disks - createContainerLogDirs(appIdStr, containerIdStr, logDirs, userName); - Path tmpDir = new Path(containerWorkDir, - YarnConfiguration.DEFAULT_CONTAINER_TEMP_DIR); - createDir(tmpDir, dirPerm, false, userName); - - // copy launch script to work dir Path launchDst = new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT); - lfs.util().copy(nmPrivateContainerScriptPath, launchDst); - - // copy container tokens to work dir - Path tokenDst = - new Path(containerWorkDir, ContainerLaunch.FINAL_CONTAINER_TOKENS_FILE); - lfs.util().copy(nmPrivateTokensPath, tokenDst); - - String localDirMount = toMount(localDirs); String logDirMount = toMount(logDirs); - String containerWorkDirMount = toMount(Collections.singletonList(containerWorkDir.toUri().getPath())); - StringBuilder commands = new StringBuilder(); - String commandStr = commands.append(dockerExecutor) - .append(" ") - .append("run") - .append(" ") - .append("--rm --net=host") - .append(" ") - .append(" --name " + containerIdStr) - .append(localDirMount) - .append(logDirMount) - .append(containerWorkDirMount) - .append(" ") - .append(containerImageName) - .toString(); - String dockerPidScript = "`" + dockerExecutor + " inspect --format {{.State.Pid}} " + containerIdStr + "`"; - // Create new local launch wrapper script - LocalWrapperScriptBuilder sb = - new UnixLocalWrapperScriptBuilder(containerWorkDir, commandStr, dockerPidScript); - Path pidFile = getPidFilePath(containerId); - if (pidFile != null) { - sb.writeLocalWrapperScript(launchDst, pidFile); - } else { - LOG.info("Container " + containerIdStr - + " was marked as inactive. Returning terminated error"); - return ExitCode.TERMINATED.getExitCode(); - } - + + String[] localMounts = localDirMount.trim().split("\\s+"); + String[] logMounts = logDirMount.trim().split("\\s+"); + List commandStr = Lists.newArrayList("docker", "-H", dockerUrl, "run", "--rm", + "--net", "host", "--name", containerIdStr, "--user", userName, "--workdir", + containerWorkDir.toUri().getPath(), "-v", "/etc/passwd:/etc/passwd:ro"); + commandStr.addAll(Arrays.asList(localMounts)); + commandStr.addAll(Arrays.asList(logMounts)); + commandStr.add(containerImageName.trim()); + commandStr.add("bash"); + commandStr.add(launchDst.toUri().getPath()); + ShellCommandExecutor shExec = null; try { - lfs.setPermission(launchDst, - ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION); - lfs.setPermission(sb.getWrapperScriptPath(), - ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION); - // Setup command to run - String[] command = getRunCommand(sb.getWrapperScriptPath().toString(), - containerIdStr, userName, pidFile, this.getConf()); if (LOG.isDebugEnabled()) { - LOG.debug("launchContainer: " + commandStr + " " + Joiner.on(" ").join(command)); + LOG.debug("launchContainer: " + Joiner.on(" ").join(commandStr)); + } + + List createDirCommand = new ArrayList(); + createDirCommand.addAll(Arrays.asList( + containerExecutorExe, userName, userName, Integer + .toString(Commands.LAUNCH_DOCKER_CONTAINER.getValue()), appId, + containerIdStr, containerWorkDir.toString(), + nmPrivateContainerScriptPath.toUri().getPath().toString(), + nmPrivateTokensPath.toUri().getPath().toString(), + StringUtils.join(",", localDirs), + StringUtils.join(",", logDirs))); + createDirCommand.addAll(commandStr); + shExec = new ShellCommandExecutor(createDirCommand.toArray(new String[createDirCommand.size()]) + , null, // NM's cwd + container.getLaunchContext().getEnvironment()); // sanitized env + if (LOG.isDebugEnabled()) { + LOG.debug("createDirCommand: " + createDirCommand); } - shExec = new ShellCommandExecutor( - command, - new File(containerWorkDir.toUri().getPath()), - container.getLaunchContext().getEnvironment()); // sanitized env - if (isContainerActive(containerId)) { + if (isContainerActive(containerId)) { shExec.execute(); + if (LOG.isDebugEnabled()) { + logOutput(shExec.getOutput()); + } } else { LOG.info("Container " + containerIdStr + " was marked as inactive. Returning terminated error"); @@ -278,7 +196,7 @@ public int launchContainer(Container container, return 0; } - @Override +@Override public void writeLaunchEnv(OutputStream out, Map environment, Map> resources, List command) throws IOException { ContainerLaunch.ShellScriptBuilder sb = ContainerLaunch.ShellScriptBuilder.create(); @@ -335,85 +253,6 @@ private boolean saneDockerImage(String containerImageName) { return dockerImagePattern.matcher(containerImageName).matches(); } - @Override - public boolean signalContainer(String user, String pid, Signal signal) - throws IOException { - if (LOG.isDebugEnabled()) { - LOG.debug("Sending signal " + signal.getValue() + " to pid " + pid - + " as user " + user); - } - if (!containerIsAlive(pid)) { - return false; - } - try { - killContainer(pid, signal); - } catch (IOException e) { - if (!containerIsAlive(pid)) { - return false; - } - throw e; - } - return true; - } - - @Override - public boolean isContainerProcessAlive(String user, String pid) - throws IOException { - return containerIsAlive(pid); - } - - /** - * Returns true if the process with the specified pid is alive. - * - * @param pid String pid - * @return boolean true if the process is alive - */ - @VisibleForTesting - public static boolean containerIsAlive(String pid) throws IOException { - try { - new ShellCommandExecutor(Shell.getCheckProcessIsAliveCommand(pid)) - .execute(); - // successful execution means process is alive - return true; - } - catch (Shell.ExitCodeException e) { - // failure (non-zero exit code) means process is not alive - return false; - } - } - - /** - * Send a specified signal to the specified pid - * - * @param pid the pid of the process [group] to signal. - * @param signal signal to send - * (for logging). - */ - protected void killContainer(String pid, Signal signal) throws IOException { - new ShellCommandExecutor(Shell.getSignalKillCommand(signal.getValue(), pid)) - .execute(); - } - - @Override - public void deleteAsUser(String user, Path subDir, Path... baseDirs) - throws IOException, InterruptedException { - if (baseDirs == null || baseDirs.length == 0) { - LOG.info("Deleting absolute path : " + subDir); - if (!lfs.delete(subDir, true)) { - //Maybe retry - LOG.warn("delete returned false for path: [" + subDir + "]"); - } - return; - } - for (Path baseDir : baseDirs) { - Path del = subDir == null ? baseDir : new Path(baseDir, subDir); - LOG.info("Deleting path : " + del); - if (!lfs.delete(del, true)) { - LOG.warn("delete returned false for path: [" + del + "]"); - } - } - } - /** * Converts a directory list to a docker mount string * @param dirs @@ -427,368 +266,4 @@ private String toMount(List dirs) { return builder.toString(); } - private abstract class LocalWrapperScriptBuilder { - - private final Path wrapperScriptPath; - - public Path getWrapperScriptPath() { - return wrapperScriptPath; - } - - public void writeLocalWrapperScript(Path launchDst, Path pidFile) throws IOException { - DataOutputStream out = null; - PrintStream pout = null; - - try { - out = lfs.create(wrapperScriptPath, EnumSet.of(CREATE, OVERWRITE)); - pout = new PrintStream(out, false, "UTF-8"); - writeLocalWrapperScript(launchDst, pidFile, pout); - } finally { - IOUtils.cleanup(LOG, pout, out); - } - } - - protected abstract void writeLocalWrapperScript(Path launchDst, Path pidFile, - PrintStream pout); - - protected LocalWrapperScriptBuilder(Path containerWorkDir) { - this.wrapperScriptPath = new Path(containerWorkDir, - Shell.appendScriptExtension(DOCKER_CONTAINER_EXECUTOR_SCRIPT)); - } - } - - private final class UnixLocalWrapperScriptBuilder - extends LocalWrapperScriptBuilder { - private final Path sessionScriptPath; - private final String dockerCommand; - private final String dockerPidScript; - - public UnixLocalWrapperScriptBuilder(Path containerWorkDir, String dockerCommand, String dockerPidScript) { - super(containerWorkDir); - this.dockerCommand = dockerCommand; - this.dockerPidScript = dockerPidScript; - this.sessionScriptPath = new Path(containerWorkDir, - Shell.appendScriptExtension(DOCKER_CONTAINER_EXECUTOR_SESSION_SCRIPT)); - } - - @Override - public void writeLocalWrapperScript(Path launchDst, Path pidFile) - throws IOException { - writeSessionScript(launchDst, pidFile); - super.writeLocalWrapperScript(launchDst, pidFile); - } - - @Override - public void writeLocalWrapperScript(Path launchDst, Path pidFile, - PrintStream pout) { - - String exitCodeFile = ContainerLaunch.getExitCodeFile( - pidFile.toString()); - String tmpFile = exitCodeFile + ".tmp"; - pout.println("#!/usr/bin/env bash"); - pout.println("bash \"" + sessionScriptPath.toString() + "\""); - pout.println("rc=$?"); - pout.println("echo $rc > \"" + tmpFile + "\""); - pout.println("mv -f \"" + tmpFile + "\" \"" + exitCodeFile + "\""); - pout.println("exit $rc"); - } - - private void writeSessionScript(Path launchDst, Path pidFile) - throws IOException { - DataOutputStream out = null; - PrintStream pout = null; - try { - out = lfs.create(sessionScriptPath, EnumSet.of(CREATE, OVERWRITE)); - pout = new PrintStream(out, false, "UTF-8"); - // We need to do a move as writing to a file is not atomic - // Process reading a file being written to may get garbled data - // hence write pid to tmp file first followed by a mv - pout.println("#!/usr/bin/env bash"); - pout.println(); - pout.println("echo "+ dockerPidScript +" > " + pidFile.toString() + ".tmp"); - pout.println("/bin/mv -f " + pidFile.toString() + ".tmp " + pidFile); - pout.println(dockerCommand + " bash \"" + - launchDst.toUri().getPath().toString() + "\""); - } finally { - IOUtils.cleanup(LOG, pout, out); - } - lfs.setPermission(sessionScriptPath, - ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION); - } - } - - protected void createDir(Path dirPath, FsPermission perms, - boolean createParent, String user) throws IOException { - lfs.mkdir(dirPath, perms, createParent); - if (!perms.equals(perms.applyUMask(lfs.getUMask()))) { - lfs.setPermission(dirPath, perms); - } - } - - /** - * Initialize the local directories for a particular user. - *
    .mkdir - *
  • $local.dir/usercache/$user
  • - *
- */ - void createUserLocalDirs(List localDirs, String user) - throws IOException { - boolean userDirStatus = false; - FsPermission userperms = new FsPermission(USER_PERM); - for (String localDir : localDirs) { - // create $local.dir/usercache/$user and its immediate parent - try { - createDir(getUserCacheDir(new Path(localDir), user), userperms, true, user); - } catch (IOException e) { - LOG.warn("Unable to create the user directory : " + localDir, e); - continue; - } - userDirStatus = true; - } - if (!userDirStatus) { - throw new IOException("Not able to initialize user directories " - + "in any of the configured local directories for user " + user); - } - } - - - /** - * Initialize the local cache directories for a particular user. - *
    - *
  • $local.dir/usercache/$user
  • - *
  • $local.dir/usercache/$user/appcache
  • - *
  • $local.dir/usercache/$user/filecache
  • - *
- */ - void createUserCacheDirs(List localDirs, String user) - throws IOException { - LOG.info("Initializing user " + user); - - boolean appcacheDirStatus = false; - boolean distributedCacheDirStatus = false; - FsPermission appCachePerms = new FsPermission(APPCACHE_PERM); - FsPermission fileperms = new FsPermission(FILECACHE_PERM); - - for (String localDir : localDirs) { - // create $local.dir/usercache/$user/appcache - Path localDirPath = new Path(localDir); - final Path appDir = getAppcacheDir(localDirPath, user); - try { - createDir(appDir, appCachePerms, true, user); - appcacheDirStatus = true; - } catch (IOException e) { - LOG.warn("Unable to create app cache directory : " + appDir, e); - } - // create $local.dir/usercache/$user/filecache - final Path distDir = getFileCacheDir(localDirPath, user); - try { - createDir(distDir, fileperms, true, user); - distributedCacheDirStatus = true; - } catch (IOException e) { - LOG.warn("Unable to create file cache directory : " + distDir, e); - } - } - if (!appcacheDirStatus) { - throw new IOException("Not able to initialize app-cache directories " - + "in any of the configured local directories for user " + user); - } - if (!distributedCacheDirStatus) { - throw new IOException( - "Not able to initialize distributed-cache directories " - + "in any of the configured local directories for user " - + user); - } - } - - /** - * Initialize the local directories for a particular user. - *
    - *
  • $local.dir/usercache/$user/appcache/$appid
  • - *
- * @param localDirs - */ - void createAppDirs(List localDirs, String user, String appId) - throws IOException { - boolean initAppDirStatus = false; - FsPermission appperms = new FsPermission(APPDIR_PERM); - for (String localDir : localDirs) { - Path fullAppDir = getApplicationDir(new Path(localDir), user, appId); - // create $local.dir/usercache/$user/appcache/$appId - try { - createDir(fullAppDir, appperms, true, user); - initAppDirStatus = true; - } catch (IOException e) { - LOG.warn("Unable to create app directory " + fullAppDir.toString(), e); - } - } - if (!initAppDirStatus) { - throw new IOException("Not able to initialize app directories " - + "in any of the configured local directories for app " - + appId.toString()); - } - } - - - /** - * Create application log directories on all disks. - */ - void createContainerLogDirs(String appId, String containerId, - List logDirs, String user) throws IOException { - - boolean containerLogDirStatus = false; - FsPermission containerLogDirPerms = new FsPermission(LOGDIR_PERM); - for (String rootLogDir : logDirs) { - // create $log.dir/$appid/$containerid - Path appLogDir = new Path(rootLogDir, appId); - Path containerLogDir = new Path(appLogDir, containerId); - try { - createDir(containerLogDir, containerLogDirPerms, true, user); - } catch (IOException e) { - LOG.warn("Unable to create the container-log directory : " - + appLogDir, e); - continue; - } - containerLogDirStatus = true; - } - if (!containerLogDirStatus) { - throw new IOException( - "Not able to initialize container-log directories " - + "in any of the configured local directories for container " - + containerId); - } - } - - /** - * Permissions for user dir. - * $local.dir/usercache/$user - */ - static final short USER_PERM = (short) 0750; - /** - * Permissions for user appcache dir. - * $local.dir/usercache/$user/appcache - */ - static final short APPCACHE_PERM = (short) 0710; - /** - * Permissions for user filecache dir. - * $local.dir/usercache/$user/filecache - */ - static final short FILECACHE_PERM = (short) 0710; - /** - * Permissions for user app dir. - * $local.dir/usercache/$user/appcache/$appId - */ - static final short APPDIR_PERM = (short) 0710; - /** - * Permissions for user log dir. - * $logdir/$user/$appId - */ - static final short LOGDIR_PERM = (short) 0710; - - private long getDiskFreeSpace(Path base) throws IOException { - return lfs.getFsStatus(base).getRemaining(); - } - - private Path getApplicationDir(Path base, String user, String appId) { - return new Path(getAppcacheDir(base, user), appId); - } - - private Path getUserCacheDir(Path base, String user) { - return new Path(new Path(base, ContainerLocalizer.USERCACHE), user); - } - - private Path getAppcacheDir(Path base, String user) { - return new Path(getUserCacheDir(base, user), - ContainerLocalizer.APPCACHE); - } - - private Path getFileCacheDir(Path base, String user) { - return new Path(getUserCacheDir(base, user), - ContainerLocalizer.FILECACHE); - } - - protected Path getWorkingDir(List localDirs, String user, - String appId) throws IOException { - Path appStorageDir = null; - long totalAvailable = 0L; - long[] availableOnDisk = new long[localDirs.size()]; - int i = 0; - // randomly choose the app directory - // the chance of picking a directory is proportional to - // the available space on the directory. - // firstly calculate the sum of all available space on these directories - for (String localDir : localDirs) { - Path curBase = getApplicationDir(new Path(localDir), - user, appId); - long space = 0L; - try { - space = getDiskFreeSpace(curBase); - } catch (IOException e) { - LOG.warn("Unable to get Free Space for " + curBase.toString(), e); - } - availableOnDisk[i++] = space; - totalAvailable += space; - } - - // throw an IOException if totalAvailable is 0. - if (totalAvailable <= 0L) { - throw new IOException("Not able to find a working directory for " - + user); - } - - // make probability to pick a directory proportional to - // the available space on the directory. - long randomPosition = RandomUtils.nextLong() % totalAvailable; - int dir = 0; - // skip zero available space directory, - // because totalAvailable is greater than 0 and randomPosition - // is less than totalAvailable, we can find a valid directory - // with nonzero available space. - while (availableOnDisk[dir] == 0L) { - dir++; - } - while (randomPosition > availableOnDisk[dir]) { - randomPosition -= availableOnDisk[dir++]; - } - appStorageDir = getApplicationDir(new Path(localDirs.get(dir)), - user, appId); - - return appStorageDir; - } - - /** - * Create application log directories on all disks. - */ - void createAppLogDirs(String appId, List logDirs, String user) - throws IOException { - - boolean appLogDirStatus = false; - FsPermission appLogDirPerms = new FsPermission(LOGDIR_PERM); - for (String rootLogDir : logDirs) { - // create $log.dir/$appid - Path appLogDir = new Path(rootLogDir, appId); - try { - createDir(appLogDir, appLogDirPerms, true, user); - } catch (IOException e) { - LOG.warn("Unable to create the app-log directory : " + appLogDir, e); - continue; - } - appLogDirStatus = true; - } - if (!appLogDirStatus) { - throw new IOException("Not able to initialize app-log directories " - + "in any of the configured local directories for app " + appId); - } - } - - /** - * @return the list of paths of given local directories - */ - private static List getPaths(List dirs) { - List paths = new ArrayList(dirs.size()); - for (int i = 0; i < dirs.size(); i++) { - paths.add(new Path(dirs.get(i))); - } - return paths; - } - } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index d6e6894..9213587 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -55,7 +55,7 @@ private String nonsecureLocalUser; private Pattern nonsecureLocalUserPattern; - private String containerExecutorExe; + protected String containerExecutorExe; private LCEResourcesHandler resourcesHandler; private boolean containerSchedPriorityIsSet = false; private int containerSchedPriorityAdjustment = 0; @@ -88,7 +88,7 @@ public void setConf(Configuration conf) { YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LIMIT_USERS); if (!containerLimitUsers) { LOG.warn(YarnConfiguration.NM_NONSECURE_MODE_LIMIT_USERS + - ": impersonation without authentication enabled"); + ": impersonation without authentication enabled"); } } @@ -116,7 +116,8 @@ String getRunAsUser(String user) { INITIALIZE_CONTAINER(0), LAUNCH_CONTAINER(1), SIGNAL_CONTAINER(2), - DELETE_AS_USER(3); + DELETE_AS_USER(3), + LAUNCH_DOCKER_CONTAINER(4); private int value; Commands(int value) { @@ -460,5 +461,5 @@ public void mountCgroups(List cgroupKVs, String hierarchy) throw new IOException("Problem mounting cgroups " + cgroupKVs + "; exit code = " + ret_code + " and output: " + shExec.getOutput(), e); } - } + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index edfd25f..9a1b14c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -1005,6 +1005,108 @@ int initialize_app(const char *user, const char *app_id, return -1; } +int launch_docker_container_as_user(const char *user,const char *app_id, + const char *container_id,const char *work_dir, + const char *script_name, const char *cred_file, + char* const* local_dirs,char* const* log_dirs, + char* const* args) { + int exit_code = -1; + char *script_file_dest = NULL; + char *cred_file_dest = NULL; + char *exit_code_file = NULL; + + script_file_dest = get_container_launcher_file(work_dir); + if (script_file_dest == NULL) { + exit_code = OUT_OF_MEMORY; + goto cleanup; + } + fprintf(LOGFILE, "dest file %s docker_args: ", script_file_dest); + int i = 0; + for(i = 0; args[i] != '\0'; i++) + { + fprintf(LOGFILE, "%s ", args[i]); + } + cred_file_dest = get_container_credentials_file(work_dir); + if (NULL == cred_file_dest) { + exit_code = OUT_OF_MEMORY; + goto cleanup; + } + // open launch script + int container_file_source = open_file_as_nm(script_name); + if (container_file_source == -1) { + exit_code = INVALID_NM_ROOT_DIRS; + goto cleanup; + } + // open credentials + int cred_file_source = open_file_as_nm(cred_file); + if (cred_file_source == -1) { + exit_code = INVALID_ARGUMENT_NUMBER; + goto cleanup; + } + // create the user directory on all disks + int result = initialize_user(user, local_dirs); + if (result != 0) { + return result; + } + // initializing log dirs + int log_create_result = create_log_dirs(app_id, log_dirs); + if (log_create_result != 0) { + return log_create_result; + } + // give up root privs + if (change_user(user_detail->pw_uid, user_detail->pw_gid) != 0) { + exit_code = SETUID_OPER_FAILED; + goto cleanup; + } + // Create container specific directories as user. If there are no resources + // to localize for this container, app-directories and log-directories are + // also created automatically as part of this call. + if (create_container_directories(user, app_id, container_id, local_dirs, + log_dirs, work_dir) != 0) { + fprintf(LOGFILE, "Could not create container dirs"); + goto cleanup; + } + + // 700 + if (copy_file(container_file_source, script_name, script_file_dest,S_IRWXU) != 0) { + exit_code = INVALID_COMMAND_PROVIDED; + goto cleanup; + } + // 600 + if (copy_file(cred_file_source, cred_file, cred_file_dest, + S_IRUSR | S_IWUSR) != 0) { + exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + goto cleanup; + } +#if HAVE_FCLOSEALL + fcloseall(); +#else + // only those fds are opened assuming no bug + fclose(LOGFILE); + fclose(ERRORFILE); + fclose(stdin); + fclose(stdout); + fclose(stderr); +#endif +if (chdir(work_dir) != 0) { + fprintf(LOGFILE, "Can't change directory to %s -%s\n", work_dir, + strerror(errno)); + goto cleanup; + } + if (execvp(args[0], args) != 0) { + fprintf(LOGFILE, "Couldn't execute the container launch with args %s - %s", + args[0], strerror(errno)); + exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + goto cleanup; + } + exit_code = 0; +cleanup: + free(exit_code_file); + free(script_file_dest); + free(cred_file_dest); + return exit_code; +} + int launch_container_as_user(const char *user, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, @@ -1136,7 +1238,7 @@ int launch_container_as_user(const char *user, const char *app_id, goto cleanup; } if (execlp(script_file_dest, script_file_dest, NULL) != 0) { - fprintf(LOGFILE, "Couldn't execute the container launch file %s - %s", + fprintf(LOGFILE, "Couldn't execute the container launch file %s - %s", script_file_dest, strerror(errno)); exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; goto cleanup; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index b1efd6a..ea2030d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -25,6 +25,7 @@ enum command { LAUNCH_CONTAINER = 1, SIGNAL_CONTAINER = 2, DELETE_AS_USER = 3, + LAUNCH_DOCKER_CONTAINER = 4 }; enum errorcodes { @@ -97,6 +98,32 @@ int initialize_app(const char *user, const char *app_id, char* const* log_dirs, char* const* args); /* + * Function used to launch a docker container as the provided user. It does the following : + * 1) Creates container work dir and log dir to be accessible by the child + * 2) Copies the script file from the NM to the work directory + * 3) Sets up the environment + * 4) Does an execlp on the same in order to replace the current image with + * container image. + * @param user the user to become + * @param app_id the application id + * @param container_id the container id + * @param work_dir the working directory for the container. + * @param script_name the name of the script to be run to launch the container. + * @param cred_file the credentials file that needs to be compied to the + * working directory. + * @param pid_file file where pid of process should be written to + * @param local_dirs nodemanager-local-directories to be used + * @param log_dirs nodemanager-log-directories to be used + * @param resources_key type of resource enforcement (none, cgroups) + * @param resources_value values needed to apply resource enforcement + * @return -1 or errorcode enum value on error (should never return on success). + */ +int launch_docker_container_as_user(const char *user,const char *app_id, + const char *container_id,const char *work_dir, + const char *script_name, const char *cred_file, + char* const* local_dirs,char* const* log_dirs, + char* const* args); +/* * Function used to launch a container as the provided user. It does the following : * 1) Creates container work dir and log dir to be accessible by the child * 2) Copies the script file from the NM to the work directory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 9b5e784..b5e7c57 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -53,6 +53,10 @@ void display_usage(FILE *stream) { fprintf(stream, "Commands:\n"); fprintf(stream, " initialize container: %2d appid tokens " \ "nm-local-dirs nm-log-dirs cmd app...\n", INITIALIZE_CONTAINER); + fprintf(stream, + " launch docker container: %2d appid containerid workdir "\ + "container-script tokens nm-local-dirs nm-log-dirs cmd app...\n", + LAUNCH_DOCKER_CONTAINER); fprintf(stream, " launch container: %2d appid containerid workdir "\ "container-script tokens pidfile nm-local-dirs nm-log-dirs resources\n", @@ -213,6 +217,25 @@ int main(int argc, char **argv) { extract_values(local_dirs), extract_values(log_dirs), argv + optind); break; + case LAUNCH_DOCKER_CONTAINER: + if (argc < 11) { + fprintf(ERRORFILE, "Wrong number of arguments (%d vs 9) for create launch docker container\n", + argc); + fflush(ERRORFILE); + return INVALID_ARGUMENT_NUMBER; + } + app_id = argv[optind++]; + container_id = argv[optind++]; + current_dir = argv[optind++]; + script_file = argv[optind++]; + cred_file = argv[optind++]; + local_dirs = argv[optind++];// good local dirs as a comma separated list + log_dirs = argv[optind++];// good log dirs as a comma separated list + exit_code = launch_docker_container_as_user(yarn_user_name, app_id, + container_id, current_dir, script_file, cred_file, + extract_values(local_dirs), + extract_values(log_dirs), argv + optind); + break; case LAUNCH_CONTAINER: if (argc != 13) { fprintf(ERRORFILE, "Wrong number of arguments (%d vs 13) for launch container\n", diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java index e43ac2e..b68c547 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutor.java @@ -31,25 +31,21 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext; import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; -import org.apache.hadoop.yarn.util.ConverterUtils; import org.junit.After; import org.junit.Before; import org.junit.Test; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; -import java.io.FileReader; import java.io.IOException; -import java.io.LineNumberReader; import java.io.PrintWriter; -import java.util.ArrayList; import java.util.HashMap; -import java.util.List; +import java.util.HashSet; import java.util.Map; +import java.util.Set; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeTrue; import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -58,12 +54,43 @@ * This is intended to test the DockerContainerExecutor code, but it requires docker * to be installed. *
    + *
  1. Compile the code with container-executor.conf.dir set to the location you + * want for testing. + *
    
    + * > mvn clean install -Pnative -Dcontainer-executor.conf.dir=/etc/hadoop
    + *                          -DskipTests
    + * 
    + * + *
  2. Set up ${container-executor.conf.dir}/container-executor.cfg + * container-executor.cfg needs to be owned by root and have in it the proper + * config values. + *
    
    + * > cat /etc/hadoop/container-executor.cfg
    + * yarn.nodemanager.linux-container-executor.group=yarn
    + * #depending on the user id of the application.submitter option
    + * min.user.id=1
    + * > sudo chown root:root /etc/hadoop/container-executor.cfg
    + * > sudo chmod 444 /etc/hadoop/container-executor.cfg
    + * 
    + * + *
  3. Move the binary and set proper permissions on it. It needs to be owned + * by root, the group needs to be the group configured in container-executor.cfg, + * and it needs the setuid bit set. (The build will also overwrite it so you + * need to move it to a place that you can support it. + *
    
    + * > cp ./hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/c/container-executor/container-executor /tmp/
    + * > sudo chown root:yarn /tmp/container-executor
    + * > sudo chmod 4550 /tmp/container-executor
    + * 
    + * + *
  4. Run the tests with the execution enabled (The user you run the tests as + * needs to be part of the group from the config. *
  5. Install docker, and Compile the code with docker-service-url set to the host and port * where docker service is running. *
    
    - * > mvn clean install -Ddocker-service-url=tcp://0.0.0.0:4243
    - *                          -DskipTests
    + * mvn test -Dtest=TestDockerContainerExecutor -Dapplication.submitter=nobody -Dcontainer-executor.path=/tmp/container-executor -Ddocker-service-url=tcp://0.0.0.0:4243
      * 
    + *
*/ public class TestDockerContainerExecutor { private static final Log LOG = LogFactory @@ -79,9 +106,6 @@ private String appSubmitter; private String dockerUrl; private String testImage = "centos"; - private String dockerExec; - private String containerIdStr; - private ContainerId getNextContainerId() { ContainerId cId = mock(ContainerId.class, RETURNS_DEEP_STUBS); @@ -112,9 +136,9 @@ public void setup() { return; } dockerUrl = " -H " + dockerUrl; - dockerExec = "docker " + dockerUrl; + conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, yarnImage); - conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME, dockerExec); + conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_DOCKER_URL, dockerUrl); exec = new DockerContainerExecutor(); dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); @@ -123,7 +147,7 @@ public void setup() { if (appSubmitter == null || appSubmitter.isEmpty()) { appSubmitter = "nobody"; } - shellExec(dockerExec + " pull " + testImage); + shellExec("docker" + dockerUrl + " pull " + testImage); } @@ -171,16 +195,35 @@ private int runAndBlock(ContainerId cId, Map launchCtxEnv, Strin private String writeScriptFile(Map launchCtxEnv, String... cmd) throws IOException { File f = File.createTempFile("TestDockerContainerExecutor", ".sh"); f.deleteOnExit(); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter p = new PrintWriter(new FileOutputStream(f)); + PrintWriter q = new PrintWriter(baos); + Set exclusionSet = new HashSet(); + exclusionSet.add(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME); + exclusionSet.add(ApplicationConstants.Environment.HADOOP_YARN_HOME.name()); + exclusionSet.add(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name()); + exclusionSet.add(ApplicationConstants.Environment.HADOOP_HDFS_HOME.name()); + exclusionSet.add(ApplicationConstants.Environment.HADOOP_CONF_DIR.name()); + exclusionSet.add(ApplicationConstants.Environment.JAVA_HOME.name()); for(Map.Entry entry: launchCtxEnv.entrySet()) { - p.println("export " + entry.getKey() + "=\"" + entry.getValue() + "\""); + if (!exclusionSet.contains(entry.getKey())) { + p.println("export " + entry.getKey() + "=\"" + entry.getValue() + "\""); + q.println("export " + entry.getKey() + "=\"" + entry.getValue() + "\""); + } } for (String part : cmd) { p.print(part.replace("\\", "\\\\").replace("'", "\\'")); p.print(" "); + q.print(part.replace("\\", "\\\\").replace("'", "\\'")); + q.print(" "); } p.println(); p.close(); + q.println(); + q.close(); + if (LOG.isDebugEnabled()) { + LOG.debug("Launch script: " + baos.toString("UTF-8")); + } return f.getAbsolutePath(); } @@ -203,10 +246,9 @@ public void testLaunchContainer() throws IOException { Map env = new HashMap(); env.put(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, testImage); String touchFileName = "touch-file-" + System.currentTimeMillis(); - File touchFile = new File(dirsHandler.getLocalDirs().get(0), touchFileName); ContainerId cId = getNextContainerId(); int ret = runAndBlock( - cId, env, "touch", touchFile.getAbsolutePath(), "&&", "cp", touchFile.getAbsolutePath(), "/"); + cId, env, "touch", "/tmp/" + touchFileName); assertEquals(0, ret); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java index 3584fed..6f989b8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDockerContainerExecutorWithMocks.java @@ -18,7 +18,6 @@ package org.apache.hadoop.yarn.server.nodemanager; -import com.google.common.base.Strings; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -42,16 +41,15 @@ import java.io.LineNumberReader; import java.util.ArrayList; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; import static org.junit.Assume.assumeTrue; +import static org.mockito.Mockito.RETURNS_DEEP_STUBS; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; -import static org.mockito.Mockito.RETURNS_DEEP_STUBS; /** * Mock tests for docker container executor @@ -60,8 +58,9 @@ private static final Log LOG = LogFactory .getLog(TestDockerContainerExecutorWithMocks.class); - public static final String DOCKER_LAUNCH_COMMAND = "/bin/true"; + public static final String DOCKER_URL = "localhost:4243"; private DockerContainerExecutor dockerContainerExecutor = null; + private final File mockParamFile = new File("./params.txt"); private LocalDirsHandlerService dirsHandler; private Path workDir; private FileContext lfs; @@ -82,7 +81,7 @@ public void setup() { conf.set(YarnConfiguration.NM_LOCAL_DIRS, "/tmp/nm-local-dir" + time); conf.set(YarnConfiguration.NM_LOG_DIRS, "/tmp/userlogs" + time); conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME, yarnImage); - conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_EXEC_NAME , DOCKER_LAUNCH_COMMAND); + conf.set(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_DOCKER_URL, DOCKER_URL); dockerContainerExecutor = new DockerContainerExecutor(); dirsHandler = new LocalDirsHandlerService(); dirsHandler.init(conf); @@ -104,6 +103,7 @@ public void tearDown() { if (lfs != null) { lfs.delete(workDir, true); } + deleteMockParamFile(); } catch (IOException e) { throw new RuntimeException(e); } @@ -210,44 +210,43 @@ public void testContainerLaunch() throws IOException { dirsHandler.getLogDirs()); assertEquals(0, ret); //get the script - Path sessionScriptPath = new Path(workDir, - Shell.appendScriptExtension( - DockerContainerExecutor.DOCKER_CONTAINER_EXECUTOR_SESSION_SCRIPT)); - LineNumberReader lnr = new LineNumberReader(new FileReader(sessionScriptPath.toString())); + boolean cmdFound = false; List localDirs = dirsToMount(dirsHandler.getLocalDirs()); List logDirs = dirsToMount(dirsHandler.getLogDirs()); - List workDirMount = dirsToMount(Collections.singletonList(workDir.toUri().getPath())); - List expectedCommands = new ArrayList( - Arrays.asList(DOCKER_LAUNCH_COMMAND, "run", "--rm", "--net=host", "--name", containerId)); - expectedCommands.addAll(localDirs); - expectedCommands.addAll(logDirs); - expectedCommands.addAll(workDirMount); + List expectedParams = new ArrayList( + Arrays.asList(appSubmitter, appSubmitter, LinuxContainerExecutor.Commands.LAUNCH_DOCKER_CONTAINER.getValue() + "", + appId, containerId, workDir.toUri().getPath(), scriptPath.toUri().getPath(), + tokensPath.toUri().getPath())); + expectedParams.addAll(dirsHandler.getLocalDirs()); + expectedParams.addAll(dirsHandler.getLogDirs()); + expectedParams.addAll(Arrays.asList( + "docker", "-H", DOCKER_URL, "run", "--rm", "--net", "host", "--name", + containerId, "--user", "nobody", "--workdir", workDir.toUri().getPath(), + "-v", "/etc/passwd:/etc/passwd:ro")); + expectedParams.addAll(localDirs); + expectedParams.addAll(logDirs); String shellScript = workDir + "/launch_container.sh"; - expectedCommands.addAll(Arrays.asList(testImage.replaceAll("['\"]", ""), "bash","\"" + shellScript + "\"")); - - String expectedPidString = "echo `/bin/true inspect --format {{.State.Pid}} " + containerId+"` > "+ pidFile.toString() + ".tmp"; - boolean pidSetterFound = false; - while(lnr.ready()){ - String line = lnr.readLine(); - LOG.debug("line: " + line); - if (line.startsWith(DOCKER_LAUNCH_COMMAND)){ - List command = new ArrayList(); - for( String s :line.split("\\s+")){ - command.add(s.trim()); - } - - assertEquals(expectedCommands, command); - cmdFound = true; - } else if (line.startsWith("echo")) { - assertEquals(expectedPidString, line); - pidSetterFound = true; - } + expectedParams.addAll(Arrays.asList(testImage.replaceAll("['\"]", ""), "bash", shellScript)); + assertEquals(expectedParams, readMockParams()); + } + private List readMockParams() throws IOException { + LinkedList ret = new LinkedList(); + LineNumberReader reader = new LineNumberReader(new FileReader( + mockParamFile)); + String line; + while((line = reader.readLine()) != null) { + ret.add(line); + } + reader.close(); + return ret; + } + private void deleteMockParamFile() { + if(mockParamFile.exists()) { + mockParamFile.delete(); } - assertTrue(cmdFound); - assertTrue(pidSetterFound); } private List dirsToMount(List dirs) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java index 98ab8e0..68a8784 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java @@ -18,26 +18,6 @@ package org.apache.hadoop.yarn.server.nodemanager; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assume.assumeTrue; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.doAnswer; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.when; - -import java.io.File; -import java.io.FileReader; -import java.io.IOException; -import java.io.LineNumberReader; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; - import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; @@ -49,13 +29,33 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; -import org.junit.Assert; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.io.LineNumberReader; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; + public class TestLinuxContainerExecutorWithMocks { private static final Log LOG = LogFactory diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/log4j.properties b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/log4j.properties index 531b68b..63353bd 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/log4j.properties +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/resources/log4j.properties @@ -12,7 +12,7 @@ # log4j configuration used during build and unit tests -log4j.rootLogger=info,stdout +log4j.rootLogger=debug,stdout log4j.threshhold=ALL log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout