diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 5ba2e05cfeb..7d0b647db80 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1969,6 +1969,16 @@ public static boolean isAclEnabled(Configuration conf) { */ public static final int DEFAULT_NM_DOCKER_STOP_GRACE_PERIOD = 10; + /** The default list of read-only mounts to be bind-mounted into all + * Docker containers that use DockerContainerRuntime. */ + public static final String NM_DOCKER_DEFAULT_RO_MOUNTS = + DOCKER_CONTAINER_RUNTIME_PREFIX + "default-ro-mounts"; + + /** The default list of read-write mounts to be bind-mounted into all + * Docker containers that use DockerContainerRuntime. */ + public static final String NM_DOCKER_DEFAULT_RW_MOUNTS = + DOCKER_CONTAINER_RUNTIME_PREFIX + "default-rw-mounts"; + /** The mode in which the Java Container Sandbox should run detailed by * the JavaSandboxLinuxContainerRuntime. */ public static final String YARN_CONTAINER_SANDBOX = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 4eb509f26fa..7e7cfa45c90 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1802,6 +1802,20 @@ + The default list of read-only mounts to be bind-mounted + into all Docker containers that use DockerContainerRuntime. + yarn.nodemanager.runtime.linux.docker.default-ro-mounts + + + + + The default list of read-write mounts to be bind-mounted + into all Docker containers that use DockerContainerRuntime. + yarn.nodemanager.runtime.linux.docker.default-rw-mounts + + + + The mode in which the Java Container Sandbox should run detailed by the JavaSandboxLinuxContainerRuntime. yarn.nodemanager.runtime.linux.sandbox-mode diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index a14b0853093..8e84421160b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -249,6 +249,8 @@ private Set capabilities; private boolean delayedRemovalAllowed; private int dockerStopGracePeriod; + private Set defaultROMounts = new HashSet<>(); + private Set defaultRWMounts = new HashSet<>(); /** * Return whether the given environment variables indicate that the operation @@ -311,6 +313,8 @@ public void initialize(Configuration conf, Context nmContext) this.conf = conf; dockerClient = new DockerClient(conf); allowedNetworks.clear(); + defaultROMounts.clear(); + defaultRWMounts.clear(); allowedNetworks.addAll(Arrays.asList( conf.getTrimmedStrings( YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, @@ -356,6 +360,14 @@ public void initialize(Configuration conf, Context nmContext) dockerStopGracePeriod = conf.getInt( YarnConfiguration.NM_DOCKER_STOP_GRACE_PERIOD, YarnConfiguration.DEFAULT_NM_DOCKER_STOP_GRACE_PERIOD); + + defaultROMounts.addAll(Arrays.asList( + conf.getTrimmedStrings( + YarnConfiguration.NM_DOCKER_DEFAULT_RO_MOUNTS))); + + defaultRWMounts.addAll(Arrays.asList( + conf.getTrimmedStrings( + YarnConfiguration.NM_DOCKER_DEFAULT_RW_MOUNTS))); } private Set getDockerCapabilitiesFromConf() throws @@ -860,6 +872,32 @@ public void launchContainer(ContainerRuntimeContext ctx) } } + if(defaultROMounts != null && !defaultROMounts.isEmpty()) { + for (String mount : defaultROMounts) { + String[] dir = StringUtils.split(mount, ':'); + if (dir.length != 2) { + throw new ContainerExecutionException("Invalid mount : " + + mount); + } + String src = dir[0]; + String dst = dir[1]; + runCommand.addReadOnlyMountLocation(src, dst); + } + } + + if(defaultRWMounts != null && !defaultRWMounts.isEmpty()) { + for (String mount : defaultRWMounts) { + String[] dir = StringUtils.split(mount, ':'); + if (dir.length != 2) { + throw new ContainerExecutionException("Invalid mount : " + + mount); + } + String src = dir[0]; + String dst = dir[1]; + runCommand.addReadWriteMountLocation(src, dst); + } + } + if (allowHostPidNamespace(container)) { runCommand.setPidNamespace("host"); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 6ad35b2a43b..c3045ba47e3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -88,6 +88,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentMap; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_DOCKER_DEFAULT_RO_MOUNTS; +import static org.apache.hadoop.yarn.conf.YarnConfiguration.NM_DOCKER_DEFAULT_RW_MOUNTS; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.APPID; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.APPLICATION_LOCAL_DIRS; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.CONTAINER_ID_STR; @@ -1349,6 +1351,142 @@ public void testUserMountModeNulInvalid() throws ContainerExecutionException { } @Test + public void testDefaultROMounts() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + conf.setStrings(NM_DOCKER_DEFAULT_RO_MOUNTS, + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + runtime.launchContainer(builder.build()); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines( + Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 14; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" group-add=" + String.join(",", groups), + dockerCommands.get(counter++)); + Assert.assertEquals(" image=busybox:latest", + dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals( + " name=container_e11_1518975676334_14532816_01_000001", + dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir," + + "/test_user_filecache_dir:/test_user_filecache_dir," + + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar", + dockerCommands.get(counter++)); + Assert.assertEquals( + " rw-mounts=/test_container_log_dir:/test_container_log_dir," + + "/test_application_local_dir:/test_application_local_dir", + dockerCommands.get(counter++)); + Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter)); + } + + @Test + public void testDefaultROMountsInvalid() throws ContainerExecutionException { + conf.setStrings(NM_DOCKER_DEFAULT_RO_MOUNTS, + "source,target"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + try { + runtime.launchContainer(builder.build()); + Assert.fail("Expected a launch container failure due to invalid mount."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + + @Test + public void testDefaultRWMounts() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + conf.setStrings(NM_DOCKER_DEFAULT_RW_MOUNTS, + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + runtime.launchContainer(builder.build()); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines( + Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 14; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" group-add=" + String.join(",", groups), + dockerCommands.get(counter++)); + Assert.assertEquals(" image=busybox:latest", + dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals( + " name=container_e11_1518975676334_14532816_01_000001", + dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" ro-mounts=/test_filecache_dir:/test_filecache_dir," + + "/test_user_filecache_dir:/test_user_filecache_dir", + dockerCommands.get(counter++)); + Assert.assertEquals( + " rw-mounts=/test_container_log_dir:/test_container_log_dir," + + "/test_application_local_dir:/test_application_local_dir," + + "/tmp/foo:/tmp/foo,/tmp/bar:/tmp/bar", + dockerCommands.get(counter++)); + Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter)); + } + + @Test + public void testDefaultRWMountsInvalid() throws ContainerExecutionException { + conf.setStrings(NM_DOCKER_DEFAULT_RW_MOUNTS, + "source,target"); + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + try { + runtime.launchContainer(builder.build()); + Assert.fail("Expected a launch container failure due to invalid mount."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + + @Test public void testContainerLivelinessCheck() throws ContainerExecutionException, PrivilegedOperationException {