diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 768382145b1..281ad015cb2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1945,6 +1945,16 @@ public static boolean isAclEnabled(Configuration conf) { */ public static final boolean DEFAULT_NM_DOCKER_ALLOW_DELAYED_REMOVAL = false; + /** The default list of read-only mounts to be bind-mounted into all + * Docker containers that use DockerContainerRuntime. */ + public static final String NM_DOCKER_DEFAULT_RO_MOUNTS = + DOCKER_CONTAINER_RUNTIME_PREFIX + "default-ro-mounts"; + + /** The default list of read-write mounts to be bind-mounted into all + * Docker containers that use DockerContainerRuntime. */ + public static final String NM_DOCKER_DEFAULT_RW_MOUNTS = + DOCKER_CONTAINER_RUNTIME_PREFIX + "default-rw-mounts"; + /** The mode in which the Java Container Sandbox should run detailed by * the JavaSandboxLinuxContainerRuntime. */ public static final String YARN_CONTAINER_SANDBOX = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 25c24d36156..a4ed307c52d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1787,6 +1787,20 @@ + The default list of read-only mounts to be bind-mounted + into all Docker containers that use DockerContainerRuntime. + yarn.nodemanager.runtime.linux.docker.default-ro-mounts + + + + + The default list of read-write mounts to be bind-mounted + into all Docker containers that use DockerContainerRuntime. + yarn.nodemanager.runtime.linux.docker.default-rw-mounts + + + + The mode in which the Java Container Sandbox should run detailed by the JavaSandboxLinuxContainerRuntime. yarn.nodemanager.runtime.linux.sandbox-mode diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 379bad4bbbf..be73b82677c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -237,6 +237,8 @@ private int userRemappingGidThreshold; private Set capabilities; private boolean delayedRemovalAllowed; + private Set defaultROMounts = new HashSet<>(); + private Set defaultRWMounts = new HashSet<>(); /** * Return whether the given environment variables indicate that the operation @@ -299,6 +301,8 @@ public void initialize(Configuration conf, Context nmContext) this.conf = conf; dockerClient = new DockerClient(conf); allowedNetworks.clear(); + defaultROMounts.clear(); + defaultRWMounts.clear(); allowedNetworks.addAll(Arrays.asList( conf.getTrimmedStrings( YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, @@ -340,6 +344,14 @@ public void initialize(Configuration conf, Context nmContext) delayedRemovalAllowed = conf.getBoolean( YarnConfiguration.NM_DOCKER_ALLOW_DELAYED_REMOVAL, YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_DELAYED_REMOVAL); + + defaultROMounts.addAll(Arrays.asList( + conf.getTrimmedStrings( + YarnConfiguration.NM_DOCKER_DEFAULT_RO_MOUNTS))); + + defaultRWMounts.addAll(Arrays.asList( + conf.getTrimmedStrings( + YarnConfiguration.NM_DOCKER_DEFAULT_RW_MOUNTS))); } private Set getDockerCapabilitiesFromConf() throws @@ -833,6 +845,32 @@ public void launchContainer(ContainerRuntimeContext ctx) } } + if(defaultROMounts != null && !defaultROMounts.isEmpty()) { + for (String mount : defaultROMounts) { + String[] dir = StringUtils.split(mount, ':'); + if (dir.length != 2) { + throw new ContainerExecutionException("Invalid mount : " + + mount); + } + String src = dir[0]; + String dst = dir[1]; + runCommand.addReadOnlyMountLocation(src, dst); + } + } + + if(defaultRWMounts != null && !defaultRWMounts.isEmpty()) { + for (String mount : defaultRWMounts) { + String[] dir = StringUtils.split(mount, ':'); + if (dir.length != 2) { + throw new ContainerExecutionException("Invalid mount : " + + mount); + } + String src = dir[0]; + String dst = dir[1]; + runCommand.addReadWriteMountLocation(src, dst); + } + } + if (allowHostPidNamespace(container)) { runCommand.setPidNamespace("host"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 8c0a6554705..3695840f34d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -82,6 +82,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentMap; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_DOCKER_DEFAULT_RO_MOUNTS; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_DOCKER_DEFAULT_RW_MOUNTS; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.APPID; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.APPLICATION_LOCAL_DIRS; @@ -1327,6 +1329,142 @@ public void testUserMountModeNulInvalid() throws ContainerExecutionException { } @Test + public void testDefaultROMounts() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + conf.setStrings(NM_DOCKER_DEFAULT_RO_MOUNTS, + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + runtime.launchContainer(builder.build()); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines( + Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 14; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" group-add=" + String.join(",", groups), + dockerCommands.get(counter++)); + Assert.assertEquals(" image=busybox:latest", + dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals( + " name=container_e11_1518975676334_14532816_01_000001", + dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir," + + "/test_user_filecache_dir:/test_user_filecache_dir," + + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar", + dockerCommands.get(counter++)); + Assert.assertEquals( + " rw-mounts=/test_container_log_dir:/test_container_log_dir," + + "/test_application_local_dir:/test_application_local_dir", + dockerCommands.get(counter++)); + Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter)); + } + + @Test + public void testDefaultROMountsInvalid() throws ContainerExecutionException { + conf.setStrings(NM_DOCKER_DEFAULT_RO_MOUNTS, + "source,target"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + try { + runtime.launchContainer(builder.build()); + Assert.fail("Expected a launch container failure due to invalid mount."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + + @Test + public void testDefaultRWMounts() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + conf.setStrings(NM_DOCKER_DEFAULT_RW_MOUNTS, + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + runtime.launchContainer(builder.build()); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines( + Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 14; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" group-add=" + String.join(",", groups), + dockerCommands.get(counter++)); + Assert.assertEquals(" image=busybox:latest", + dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals( + " name=container_e11_1518975676334_14532816_01_000001", + dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir," + + "/test_user_filecache_dir:/test_user_filecache_dir", + dockerCommands.get(counter++)); + Assert.assertEquals( + " rw-mounts=/test_container_log_dir:/test_container_log_dir," + + "/test_application_local_dir:/test_application_local_dir," + + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar", + dockerCommands.get(counter++)); + Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter)); + } + + @Test + public void testDefaultRWMountsInvalid() throws ContainerExecutionException { + conf.setStrings(NM_DOCKER_DEFAULT_RW_MOUNTS, + "source,target"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + try { + runtime.launchContainer(builder.build()); + Assert.fail("Expected a launch container failure due to invalid mount."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + + @Test public void testContainerLivelinessCheck() throws ContainerExecutionException, PrivilegedOperationException {