diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java index 1390214..b2f746e 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java @@ -74,7 +74,13 @@ * currently only supports simple authentication mode. It shares a lot of code * with the DefaultContainerExecutor (and it may make sense to pull out those * common pieces later). + * @deprecated The {@code DockerContainerExecutor} class has several + * limitations, not the least of which is that if used, all containers + * are launched in Docker containers. The {@link LinuxContainerExecutor} + * supports selectively launching containers in Docker containers and should + * be used instead. */ +@Deprecated public class DockerContainerExecutor extends ContainerExecutor { private static final Log LOG = LogFactory .getLog(DockerContainerExecutor.class); @@ -324,11 +330,12 @@ public int launchContainer(ContainerStartContext ctx) throws IOException { return 0; } - @Override /** * Filter the environment variables that may conflict with the ones set in * the docker image and write them out to an OutputStream. + * @throws IOException if there's an issue writing out the launch file. */ + @Override public void writeLaunchEnv(OutputStream out, Map environment, Map> resources, List command, Path logDir, String user) throws IOException { @@ -336,7 +343,7 @@ public void writeLaunchEnv(OutputStream out, Map environment, ContainerLaunch.ShellScriptBuilder.create(); //Remove environments that may conflict with the ones in Docker image. - Set exclusionSet = new HashSet(); + Set exclusionSet = new HashSet<>(); exclusionSet.add(YarnConfiguration.NM_DOCKER_CONTAINER_EXECUTOR_IMAGE_NAME); exclusionSet.add(ApplicationConstants.Environment.HADOOP_YARN_HOME.name()); exclusionSet.add(ApplicationConstants.Environment.HADOOP_COMMON_HOME.name()); @@ -347,14 +354,23 @@ public void writeLaunchEnv(OutputStream out, Map environment, if (environment != null) { for (Map.Entry env : environment.entrySet()) { if (!exclusionSet.contains(env.getKey())) { - sb.env(env.getKey().toString(), env.getValue().toString()); + sb.env(env.getKey(), env.getValue()); } } } if (resources != null) { for (Map.Entry> entry : resources.entrySet()) { for (String linkName : entry.getValue()) { - sb.symlink(entry.getKey(), new Path(linkName)); + if (new Path(linkName).getName().equals(WILDCARD)) { + // If this is a wildcarded path, link to everything in the + // directory from the working directory + for (File wildLink : readDirAsUser(user, entry.getKey())) { + sb.symlink(new Path(wildLink.toString()), + new Path(wildLink.getName())); + } + } else { + sb.symlink(entry.getKey(), new Path(linkName)); + } } } } @@ -370,26 +386,17 @@ public void writeLaunchEnv(OutputStream out, Map environment, sb.command(command); - PrintStream pout = null; - PrintStream ps = null; - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try { - pout = new PrintStream(out, false, "UTF-8"); - if (LOG.isDebugEnabled()) { - ps = new PrintStream(baos, false, "UTF-8"); - sb.write(ps); - } + try (PrintStream pout = new PrintStream(out, false, "UTF-8")) { sb.write(pout); - - } finally { - if (out != null) { - out.close(); - } - if (ps != null) { - ps.close(); - } } + if (LOG.isDebugEnabled()) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + + try (PrintStream ps = new PrintStream(baos, false, "UTF-8")) { + sb.write(ps); + } + LOG.debug("Script: " + baos.toString("UTF-8")); } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm index fbfe04b..1fa94ca 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainerExecutor.md.vm @@ -22,6 +22,15 @@ Docker Container Executor * [Docker Image Requirements](#Docker_Image_Requirements) * [Working example of yarn launched docker containers](#Working_example_of_yarn_launched_docker_containers) +DEPRECATED +---------- + +The Docker Container Executor is not a recommended solution for providing Docker +support in a YARN cluster. The Linux Container Executor should be used instead. +The documentation on [using CGroups](NodeManagerCgroups.html) and +[secure containers](SecureContainer.html) contain basic information about +configuring the Linux Container Executor. + Overview --------