commit aa80e3bc4db85e38ad6f4e0686b5d2ecd0500f57 Author: eyang Date: Fri Oct 13 18:17:53 2017 -0400 YARN-7066. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 2013306..0425246 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -49,6 +49,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; +import org.codehaus.jettison.json.JSONArray; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import java.nio.file.Files; import java.nio.file.Paths; @@ -128,6 +131,12 @@ * source is an absolute path that is not a symlink and that points to a * localized resource. * + *
  • + * {@code YARN_CONTAINER_RUNTIME_DOCKER_USER_MOUNTS} adds additional + * volume mounts to the Docker container. The value of the environment + * variable should be a JSONArray compose of JSONObject which + * contains source, target, and option. + *
  • * */ @InterfaceAudience.Private @@ -170,6 +179,18 @@ @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS = "YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_USER_MOUNTS = + "MOUNTS"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_MOUNT_SOURCE = + "source"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_MOUNT_TARGET = + "target"; + @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_MOUNT_OPTION = + "option"; private Configuration conf; private DockerClient dockerClient; @@ -596,6 +617,26 @@ public void launchContainer(ContainerRuntimeContext ctx) } } + if(environment.containsKey(ENV_DOCKER_CONTAINER_USER_MOUNTS)) { + String blob = environment.get(ENV_DOCKER_CONTAINER_USER_MOUNTS); + JSONArray list; + try { + list = new JSONArray(blob); + for (int i = 0; i < list.length(); i++) { + JSONObject mount = (JSONObject) list.get(i); + String src = mount.getString(ENV_DOCKER_CONTAINER_MOUNT_SOURCE); + String dst = mount.getString(ENV_DOCKER_CONTAINER_MOUNT_TARGET); + if (mount.has(ENV_DOCKER_CONTAINER_MOUNT_OPTION)) { + dst = dst + ":" + mount.getString(ENV_DOCKER_CONTAINER_MOUNT_OPTION); + } + runCommand.addMountLocation(src, dst, true); + } + } catch (JSONException e) { + throw new ContainerExecutionException("Invalid mount : " + + e); + } + } + if (allowPrivilegedContainerExecution(container)) { runCommand.setPrivileged(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index fbfee54..f5c3796 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -1002,6 +1002,36 @@ public void testMountMultiple() } @Test + public void testUserDefinedMounts() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf); + env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_USER_MOUNTS, "[{ \"source\":\"/home/"+runAsUser+"\", \"target\":\"/mnt/hdfs/user/"+runAsUser+"\", \"option\":\"ro\" },{ \"source\":\"/tmp/abc/data\", \"target\":\"/mnt/hdfs/tmp/abc/data\" }]"); + runtime.launchContainer(builder.setExecutionAttribute(RUN_AS_USER, runAsUser).build()); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines(Paths.get + (dockerCommandFile), Charset.forName("UTF-8")); + + Assert.assertEquals(1, dockerCommands.size()); + + String command = dockerCommands.get(0); + + Assert.assertTrue("Did not find expected " + + "/home/"+runAsUser+":/mnt/hdfs/user/"+runAsUser+":ro in docker " + + "run args : " + command, + command.contains(" -v /home/"+runAsUser+":/mnt/hdfs/user/"+runAsUser+":ro")); + Assert.assertTrue("Did not find expected " + + "/tmp/abc/data:/mnt/hdfs/tmp/abc/data in docker " + + "run args : " + command, + command.contains(" -v /tmp/abc/data:/mnt/hdfs/tmp/abc/data")); + } + + @Test public void testContainerLivelinessCheck() throws ContainerExecutionException, PrivilegedOperationException {