diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 93437e3..a827dff 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1502,6 +1502,17 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK = "host"; + /** Comma separated list containing the allowed mounts for Docker containers. + * The format of the value is "/src:mode". /src cannot be a symlink, must be + * an absolute path, and must already exist on the NodeManager. Mode is used + * to control whether the mount is read-only or read-write. The allowed values + * are "ro" and "rw". Local Resources are automatically added to the mount + * whitelist at runtime, and are not required to be added by the + * administrator. + */ + public static final String DOCKER_WHITE_LIST_VOLUME_MOUNTS = + DOCKER_CONTAINER_RUNTIME_PREFIX + "white-list-volume-mounts"; + /** The mode in which the Java Container Sandbox should run detailed by * the JavaSandboxLinuxContainerRuntime. */ public static final String YARN_CONTAINER_SANDBOX = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index e058d6e..006b4c1 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -44,19 +44,18 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerInspectCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerStopCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.util.DockerBindMountUtils; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeConstants; import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext; -import java.nio.file.Files; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Map.Entry; import java.util.Set; import java.util.regex.Pattern; @@ -120,12 +119,22 @@ * setting it to false. * *
  • - * {@code YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS} adds - * additional volume mounts to the Docker container. The value of the + * {@code YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS} allows users to specify + * additional volume mounts for the Docker container. The value of the * environment variable should be a comma-separated list of mounts. - * All such mounts must be given as {@code source:dest}, where the - * source is an absolute path that is not a symlink and that points to a - * localized resource. + * All such mounts must be given as {@code source:dest:mode}, and meet the + * following criteria. + *
      + *
    1. The source path cannot reference parent directories via the .. + * notation.
    2. + *
    3. The source path must exist on the node manager.
    4. + *
    5. The source path cannot be a symlink.
    6. + *
    7. The mount mode must be "ro" or "rw".
    8. + *
    9. The mount mode must be allowed by the admin whitelist.
    10. + *
    11. The mount must be in the admin whitelist defined by + * {@code yarn.nodemanager.runtime.linux.docker.white-list-volume-mounts} + * or a localized resource.
    12. + *
    *
  • * */ @@ -164,8 +173,8 @@ public static final String ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER = "YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER"; @InterfaceAudience.Private - public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS = - "YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS"; + public static final String ENV_DOCKER_CONTAINER_MOUNTS = + "YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS"; static final String CGROUPS_ROOT_DIRECTORY = "/sys/fs/cgroup"; @@ -176,6 +185,7 @@ private String defaultNetwork; private CGroupsHandler cGroupsHandler; private AccessControlList privilegedContainersAcl; + private Map adminMountWhitelist = new HashMap<>(); /** * Return whether the given environment variables indicate that the operation @@ -260,6 +270,9 @@ public void initialize(Configuration conf) privilegedContainersAcl = new AccessControlList(conf.getTrimmed( YarnConfiguration.NM_DOCKER_PRIVILEGED_CONTAINERS_ACL, YarnConfiguration.DEFAULT_NM_DOCKER_PRIVILEGED_CONTAINERS_ACL)); + + adminMountWhitelist = DockerBindMountUtils.parseAdminWhitelist( + conf.getTrimmed(YarnConfiguration.DOCKER_WHITE_LIST_VOLUME_MOUNTS, "")); } @Override @@ -414,28 +427,6 @@ private boolean allowPrivilegedContainerExecution(Container container) return true; } - @VisibleForTesting - protected String validateMount(String mount, - Map> localizedResources) - throws ContainerExecutionException { - for (Entry> resource : localizedResources.entrySet()) { - if (resource.getValue().contains(mount)) { - java.nio.file.Path path = Paths.get(resource.getKey().toString()); - if (!path.isAbsolute()) { - throw new ContainerExecutionException("Mount must be absolute: " + - mount); - } - if (Files.isSymbolicLink(path)) { - throw new ContainerExecutionException("Mount cannot be a symlink: " + - mount); - } - return path.toString(); - } - } - throw new ContainerExecutionException("Mount must be a localized " + - "resource: " + mount); - } - @Override public void launchContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { @@ -499,21 +490,9 @@ public void launchContainer(ContainerRuntimeContext ctx) runCommand.addMountLocation(dir, dir, true); } - if (environment.containsKey(ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS)) { - String mounts = environment.get( - ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS); - if (!mounts.isEmpty()) { - for (String mount : StringUtils.split(mounts)) { - String[] dir = StringUtils.split(mount, ':'); - if (dir.length != 2) { - throw new ContainerExecutionException("Invalid mount : " + - mount); - } - String src = validateMount(dir[0], localizedResources); - String dst = dir[1]; - runCommand.addMountLocation(src, dst + ":ro", true); - } - } + if (environment.containsKey(ENV_DOCKER_CONTAINER_MOUNTS)) { + DockerBindMountUtils.addUserMounts(environment, adminMountWhitelist, + localizedResources, runCommand); } if (allowPrivilegedContainerExecution(container)) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index b645754..0896d59 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -69,6 +69,13 @@ public DockerRunCommand addMountLocation(String sourcePath, String return this; } + public DockerRunCommand addMountLocation(String sourcePath, String + destinationPath, String mode) { + super.addCommandArguments("-v", sourcePath + ":" + destinationPath + + ":" + mode); + return this; + } + public DockerRunCommand setCGroupParent(String parentPath) { super.addCommandArguments("--cgroup-parent=" + parentPath); return this; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/util/DockerBindMountUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/util/DockerBindMountUtils.java new file mode 100644 index 0000000..6370751 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/util/DockerBindMountUtils.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.util; + +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +final public class DockerBindMountUtils { + + private static final Pattern USER_MOUNT_PATTERN = Pattern.compile( + "(?<=^|,)([\\s/.a-zA-Z0-9_-]+):([\\s/.a-zA-Z0-9_-]+):([a-z]+)"); + private static final Pattern ADMIN_MOUNT_WHITELIST_PATTERN = Pattern.compile( + "(?<=^|,)([\\s/.a-zA-Z0-9_-]+):([a-z]+)"); + + /** + * Parse the admin supplied whitelist configuration. + * + * @param adminWhiteList the whitelist from the YARN configuration. + * @return the parsed whitelist. + * @throws ContainerExecutionException if the configuration is invalid. + */ + public static Map parseAdminWhitelist(String adminWhiteList) + throws ContainerExecutionException { + Matcher m = ADMIN_MOUNT_WHITELIST_PATTERN.matcher(adminWhiteList); + Map parsedAdminWhitelist = new HashMap<>(); + while (m.find()) { + if (m.groupCount() != 2) { + throw new ContainerExecutionException( + "Invalid mount in whitelist: " + m.group(1)); + } + if (!validateMountModeString(m.group(2))) { + throw new ContainerExecutionException( + "Invalid mount mode for mount: " + m.group(1)); + } + parsedAdminWhitelist.put(Paths.get(m.group(1)), m.group(2)); + } + return parsedAdminWhitelist; + } + + /** + * Add the volume bind mounts to the Docker run command. + * + * @param environment the environment for the container. + * @param adminMountWhitelist the admin defined whitelist + * @param localizedResources resources localized for the container. + * @param runCommand the Docker run command to augment with volumes. + * @throws ContainerExecutionException if the configuration is invalid. + */ + public static void addUserMounts(Map environment, + Map adminMountWhitelist, + Map> localizedResources, + DockerRunCommand runCommand) throws ContainerExecutionException { + Matcher parsedUserMounts = USER_MOUNT_PATTERN.matcher(environment + .get(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS)); + while (parsedUserMounts.find()) { + if (parsedUserMounts.groupCount() != 3) { + throw new ContainerExecutionException( + "Invalid mount : " + parsedUserMounts.group()); + } + java.nio.file.Path src = Paths.get(parsedUserMounts.group(1)); + String dst = parsedUserMounts.group(2); + String mode = parsedUserMounts.group(3); + if (validateUserMount(src, mode, adminMountWhitelist, + localizedResources)) { + runCommand.addMountLocation(src.toString(), dst, mode); + } + } + } + + /** + * Validate the user supplied mount against the following conditions. + *
      + *
    1. The source path cannot reference parent directories via the .. + * notation.
    2. + *
    3. The source path must exist on the node manager.
    4. + *
    5. The source path cannot be a symlink.
    6. + *
    7. The mount mode must be "ro" or "rw".
    8. + *
    9. The mount mode must be allowed by the admin whitelist.
    10. + *
    11. The mount must be in the admin whitelist defined by + * {@code yarn.nodemanager.runtime.linux.docker.white-list-volume-mounts} + * or a localized resource.
    12. + *
    + * + * @param userMountSrc The source mount. + * @param mode The mode used for the mount. ro and rw allowed. + * @param localizedResources The collection of localized resources. + * @return true if the mount meets the validation criteria. + * @throws ContainerExecutionException any validation failure. + */ + private static boolean validateUserMount(Path userMountSrc, String mode, + Map adminMountWhitelist, + Map> localizedResources) + throws ContainerExecutionException { + + if (userMountSrc.toString().contains("..")) { + throw new ContainerExecutionException( + "Mount cannot reference parent directories: " + userMountSrc); + } + if (!userMountSrc.toFile().exists()) { + throw new ContainerExecutionException("Source path does not already " + + "exist: " + userMountSrc); + } + if (!userMountSrc.isAbsolute()) { + throw new ContainerExecutionException("Mount must be absolute: " + + userMountSrc); + } + if (Files.isSymbolicLink(userMountSrc)) { + throw new ContainerExecutionException("Mount cannot be a symlink: " + + userMountSrc); + } + if (!validateMountModeString(mode)) { + throw new ContainerExecutionException("Invalid mount mode requested: " + + mode); + } + Map matchingMountFromWhitelist = + getMatchingMountFromWhitelist(userMountSrc, adminMountWhitelist); + if (matchingMountFromWhitelist.size() == 1) { + Map.Entry matchingMount = + matchingMountFromWhitelist.entrySet().iterator().next(); + if (validateUserMountModeAllowed(mode, matchingMount.getValue())) { + return true; + } + } + for (Map.Entry> resource : localizedResources.entrySet()) { + if (resource.getKey().toString().contains(userMountSrc.toString())) { + if (!mode.equals("ro")) { + throw new ContainerExecutionException("Local resources can only be " + + "mounted read-only"); + } + return true; + } + } + throw new ContainerExecutionException("Mount was not found in the " + + "configured whitelist and is not a local resource: " + userMountSrc); + } + + /** + * Check the requested mode against the admin allowed mode. + * + * @param requestedMode user requested mode for the mount. + * @param allowedMode admin allowed mode for the mount. + * @return true if the mount is allowed. + */ + private static boolean validateUserMountModeAllowed(String requestedMode, + String allowedMode) { + return requestedMode.equals("ro") || + (requestedMode.equals("rw") && allowedMode.equals("rw")); + } + + /** + * Validate that the supplied mount mode is a valid value. + * + * @param mode the mode to check. + * @return true if the mount mode is a valid value. + */ + private static boolean validateMountModeString(String mode) { + return mode.equals("ro") || mode.equals("rw"); + } + + /** + * Check the admin supplied whitelist for the user supplied source mount. The + * user supplied mount must be an exact match or a subdirectory of the admin + * supplied whitelist. + * + * @param userSrcMount the user supplied source mount. + * @param adminMountWhitelist the admin supplied whitelist. + * @return if the mount is found in the whitelist, return the whitelist entry. + */ + private static Map getMatchingMountFromWhitelist( + Path userSrcMount, Map adminMountWhitelist) { + Map matchingMountFromWhitelist = new HashMap<>(); + if (adminMountWhitelist.containsKey(userSrcMount)) { + matchingMountFromWhitelist.put(userSrcMount, + adminMountWhitelist.get(userSrcMount)); + return matchingMountFromWhitelist; + } + Path userSrcPathParent = userSrcMount.getParent(); + while (userSrcPathParent != null) { + if (adminMountWhitelist.containsKey(userSrcPathParent)) { + matchingMountFromWhitelist.put( + userSrcMount, adminMountWhitelist.get(userSrcMount)); + return matchingMountFromWhitelist; + } + userSrcPathParent = userSrcPathParent.getParent(); + } + return matchingMountFromWhitelist; + } +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 9894dcd..7c770f8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -709,114 +709,6 @@ public void testCGroupParent() throws ContainerExecutionException { } @Test - public void testMountSourceOnly() - throws ContainerExecutionException, PrivilegedOperationException, - IOException{ - DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( - mockExecutor, mockCGroupsHandler); - runtime.initialize(conf); - - env.put( - DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS, - "source"); - - try { - runtime.launchContainer(builder.build()); - Assert.fail("Expected a launch container failure due to invalid mount."); - } catch (ContainerExecutionException e) { - LOG.info("Caught expected exception : " + e); - } - } - - @Test - public void testMountSourceTarget() - throws ContainerExecutionException, PrivilegedOperationException, - IOException{ - DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( - mockExecutor, mockCGroupsHandler); - runtime.initialize(conf); - - env.put( - DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS, - "test_dir/test_resource_file:test_mount"); - - runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List dockerCommands = Files.readAllLines(Paths.get - (dockerCommandFile), Charset.forName("UTF-8")); - - Assert.assertEquals(1, dockerCommands.size()); - - String command = dockerCommands.get(0); - - Assert.assertTrue("Did not find expected " + - "/test_local_dir/test_resource_file:test_mount mount in docker " + - "run args : " + command, - command.contains(" -v /test_local_dir/test_resource_file:test_mount" + - ":ro ")); - } - - @Test - public void testMountInvalid() - throws ContainerExecutionException, PrivilegedOperationException, - IOException{ - DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( - mockExecutor, mockCGroupsHandler); - runtime.initialize(conf); - - env.put( - DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS, - "source:target:other"); - - try { - runtime.launchContainer(builder.build()); - Assert.fail("Expected a launch container failure due to invalid mount."); - } catch (ContainerExecutionException e) { - LOG.info("Caught expected exception : " + e); - } - } - - @Test - public void testMountMultiple() - throws ContainerExecutionException, PrivilegedOperationException, - IOException{ - DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( - mockExecutor, mockCGroupsHandler); - runtime.initialize(conf); - - env.put( - DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS, - "test_dir/test_resource_file:test_mount1," + - "test_dir/test_resource_file:test_mount2"); - - runtime.launchContainer(builder.build()); - PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); - List args = op.getArguments(); - String dockerCommandFile = args.get(11); - - List dockerCommands = Files.readAllLines(Paths.get - (dockerCommandFile), Charset.forName("UTF-8")); - - Assert.assertEquals(1, dockerCommands.size()); - - String command = dockerCommands.get(0); - - Assert.assertTrue("Did not find expected " + - "/test_local_dir/test_resource_file:test_mount1 mount in docker " + - "run args : " + command, - command.contains(" -v /test_local_dir/test_resource_file:test_mount1" + - ":ro ")); - Assert.assertTrue("Did not find expected " + - "/test_local_dir/test_resource_file:test_mount2 mount in docker " + - "run args : " + command, - command.contains(" -v /test_local_dir/test_resource_file:test_mount2" + - ":ro ")); - } - - @Test public void testContainerLivelinessCheck() throws ContainerExecutionException, PrivilegedOperationException { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/util/TestDockerBindMountUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/util/TestDockerBindMountUtils.java new file mode 100644 index 0000000..8529d3a --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/util/TestDockerBindMountUtils.java @@ -0,0 +1,226 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.util; + +import org.apache.commons.collections.map.HashedMap; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker.DockerRunCommand; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; + +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_MOUNTS; +import static org.junit.Assert.*; + +public class TestDockerBindMountUtils { + + private static final Log LOG = LogFactory.getLog( + TestDockerBindMountUtils.class); + + private Configuration conf; + private String containerId; + private String image; + private String user; + + private Map environment = new HashMap<>(); + private Map adminMountWhitelist = new HashedMap(); + private Map> localizedResources = + new HashMap<>(); + + @Before + public void setUp() { + conf = new Configuration(); + containerId = "container_id"; + image = "busybox:latest"; + user = "user"; + environment.clear(); + adminMountWhitelist.clear(); + localizedResources.clear(); + } + + @Rule + public TemporaryFolder tempdir1 = new TemporaryFolder(); + public TemporaryFolder tempdir2 = new TemporaryFolder(); + + @Test + public void testAddUserMountsSuccess() throws Exception { + tempdir1.create(); + tempdir2.create(); + conf.set( + YarnConfiguration.DOCKER_WHITE_LIST_VOLUME_MOUNTS, + tempdir1.getRoot() + ":rw," + tempdir2.getRoot() + ":ro"); + adminMountWhitelist = DockerBindMountUtils.parseAdminWhitelist( + conf.getTrimmed(YarnConfiguration.DOCKER_WHITE_LIST_VOLUME_MOUNTS, "")); + String mountsEnv = tempdir1.getRoot() + ":/tmp/1:rw," + + tempdir2.getRoot() + ":/tmp/2:ro"; + environment.put(ENV_DOCKER_CONTAINER_MOUNTS, mountsEnv); + DockerRunCommand dockerRunCommand = + new DockerRunCommand(containerId, user, image); + DockerBindMountUtils.addUserMounts(environment, adminMountWhitelist, + localizedResources, dockerRunCommand); + String dockerRunFull = "run --name=container_id --user=user " + + "-v " + tempdir1.getRoot() + ":/tmp/1:rw " + + "-v " + tempdir2.getRoot() + ":/tmp/2:ro busybox:latest"; + assertEquals(dockerRunFull, dockerRunCommand.getCommandWithArguments()); + } + + @Test + public void testAddUserMountsLocalizedResourceSuccess() throws Exception { + tempdir1.create(); + localizedResources.put( + new org.apache.hadoop.fs.Path(tempdir1.getRoot().toString()), + Collections.singletonList(tempdir1.getRoot().toString())); + String mountsEnv = tempdir1.getRoot() + ":/tmp/1:ro"; + environment.put(ENV_DOCKER_CONTAINER_MOUNTS, mountsEnv); + DockerRunCommand dockerRunCommand = + new DockerRunCommand(containerId, user, image); + DockerBindMountUtils.addUserMounts(environment, adminMountWhitelist, + localizedResources, dockerRunCommand); + String dockerRunFull = "run --name=container_id --user=user " + + "-v " + tempdir1.getRoot() + ":/tmp/1:ro " + "busybox:latest"; + assertEquals(dockerRunFull, dockerRunCommand.getCommandWithArguments()); + } + + @Test + public void testAddUserMountsLocalizedResourceRWFailure() throws Exception { + tempdir1.create(); + localizedResources.put( + new org.apache.hadoop.fs.Path(tempdir1.getRoot().toString()), + Collections.singletonList(tempdir1.getRoot().toString())); + String mountsEnv = tempdir1.getRoot() + ":/tmp/1:rw"; + environment.put(ENV_DOCKER_CONTAINER_MOUNTS, mountsEnv); + DockerRunCommand dockerRunCommand = + new DockerRunCommand(containerId, user, image); + try { + DockerBindMountUtils.addUserMounts(environment, adminMountWhitelist, + localizedResources, dockerRunCommand); + Assert.fail("Expected failure due to attempt to mount local resource rw"); + } catch (ContainerExecutionException e) { + assertEquals("Local resources can only be mounted read-only", + e.getMessage()); + LOG.info("Caught expected exception : " + e); + } + } + + @Test + public void testAddUserMountParentCheckFailure() throws Exception { + conf.set(YarnConfiguration.DOCKER_WHITE_LIST_VOLUME_MOUNTS, + "/tmp/1:rw"); + adminMountWhitelist = DockerBindMountUtils.parseAdminWhitelist( + conf.getTrimmed(YarnConfiguration.DOCKER_WHITE_LIST_VOLUME_MOUNTS, "")); + String mountsEnv = "/tmp/1/../:/tmp/1/:rw"; + environment.put(ENV_DOCKER_CONTAINER_MOUNTS, mountsEnv); + DockerRunCommand dockerRunCommand = + new DockerRunCommand(containerId, user, image); + try { + DockerBindMountUtils.addUserMounts(environment, adminMountWhitelist, + localizedResources, dockerRunCommand); + Assert.fail("Expected failure due to parent directory reference"); + } catch (ContainerExecutionException e) { + assertEquals("Mount cannot reference parent directories: /tmp/1/..", + e.getMessage()); + LOG.info("Caught expected exception : " + e); + } + } + + @Test + public void testAddUserMountNonexistentSourceFailure() throws Exception { + conf.set(YarnConfiguration.DOCKER_WHITE_LIST_VOLUME_MOUNTS, + "/tmp/1:rw"); + adminMountWhitelist = DockerBindMountUtils.parseAdminWhitelist( + conf.getTrimmed(YarnConfiguration.DOCKER_WHITE_LIST_VOLUME_MOUNTS, "")); + String mountsEnv = "/tmp/1/:/tmp/1/:rw"; + environment.put(ENV_DOCKER_CONTAINER_MOUNTS, mountsEnv); + DockerRunCommand dockerRunCommand = + new DockerRunCommand(containerId, user, image); + try { + DockerBindMountUtils.addUserMounts(environment, adminMountWhitelist, + localizedResources, dockerRunCommand); + Assert.fail("Expected failure due to non-existent source"); + } catch (ContainerExecutionException e) { + assertEquals("Source path does not already exist: /tmp/1", + e.getMessage()); + LOG.info("Caught expected exception : " + e); + } + } + + @Test + public void testAddUserMountSymlinkSourceFailure() throws Exception { + tempdir1.create(); + tempdir2.create(); + Path symlinkTarget = Paths.get(tempdir2.getRoot() + "/symlinkTarget"); + Path symlink = Files.createSymbolicLink(symlinkTarget, + tempdir1.getRoot().toPath().toAbsolutePath()); + assertTrue(Files.exists(symlink)); + assertTrue(Files.isSymbolicLink(symlink)); + conf.set(YarnConfiguration.DOCKER_WHITE_LIST_VOLUME_MOUNTS, + "/tmp/1:rw"); + adminMountWhitelist = DockerBindMountUtils.parseAdminWhitelist( + conf.getTrimmed(YarnConfiguration.DOCKER_WHITE_LIST_VOLUME_MOUNTS, "")); + String mountsEnv = symlink.toFile().getAbsolutePath() + ":/tmp/1/:rw"; + environment.put(ENV_DOCKER_CONTAINER_MOUNTS, mountsEnv); + DockerRunCommand dockerRunCommand = + new DockerRunCommand(containerId, user, image); + try { + DockerBindMountUtils.addUserMounts(environment, adminMountWhitelist, + localizedResources, dockerRunCommand); + Assert.fail("Expected failure due to symlinked source"); + } catch (ContainerExecutionException e) { + assertEquals("Mount cannot be a symlink: " + symlink, e.getMessage()); + LOG.info("Caught expected exception : " + e); + } finally { + Files.delete(symlink); + } + } + + @Test + public void testAddUserMountNonwhitelistedSourceFailure() throws Exception { + tempdir1.create(); + tempdir2.create(); + conf.set(YarnConfiguration.DOCKER_WHITE_LIST_VOLUME_MOUNTS, "/tmp/1:rw"); + adminMountWhitelist = DockerBindMountUtils.parseAdminWhitelist( + conf.getTrimmed(YarnConfiguration.DOCKER_WHITE_LIST_VOLUME_MOUNTS, "")); + String mountsEnv = tempdir1.getRoot() + ":/tmp/1:rw"; + environment.put(ENV_DOCKER_CONTAINER_MOUNTS, mountsEnv); + DockerRunCommand dockerRunCommand = + new DockerRunCommand(containerId, user, image); + try { + DockerBindMountUtils.addUserMounts(environment, adminMountWhitelist, + localizedResources, dockerRunCommand); + Assert.fail("Expected failure due to symlinked source"); + } catch (ContainerExecutionException e) { + assertEquals("Mount was not found in the configured whitelist and is not " + + "a local resource: " + tempdir1.getRoot(), e.getMessage()); + LOG.info("Caught expected exception : " + e); + } + } +} \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index 4de0a6a..0eec07b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -133,6 +133,16 @@ The following properties should be set in yarn-site.xml: privileged contains if privileged containers are allowed. + + + yarn.nodemanager.runtime.linux.docker.white-list-volume-mounts + + + Optional. A comma-separated list of volumes provided in the form + {@code source:mode} from which applications are allowed to bind mount into + containers. + + ``` In addition, a container-executer.cfg file must exist and contain settings for @@ -223,7 +233,7 @@ environment variables in the application's environment: | `YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE` | Controls whether the Docker container's default command is overridden. When set to true, the Docker container's command will be "bash _path\_to\_launch\_script_". When unset or set to false, the Docker container's default command is used. | | `YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK` | Sets the network type to be used by the Docker container. It must be a valid value as determined by the yarn.nodemanager.runtime.linux.docker.allowed-container-networks property. | | `YARN_CONTAINER_RUNTIME_DOCKER_RUN_PRIVILEGED_CONTAINER` | Controls whether the Docker container is a privileged container. In order to use privileged containers, the yarn.nodemanager.runtime.linux.docker.privileged-containers.allowed property must be set to true, and the application owner must appear in the value of the yarn.nodemanager.runtime.linux.docker.privileged-containers.acl property. If this environment variable is set to true, a privileged Docker container will be used if allowed. No other value is allowed, so the environment variable should be left unset rather than setting it to false. | -| `YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest", where the source is an absolute path that is not a symlink and that points to a localized resource. Note that as of YARN-5298, localized directories are automatically mounted into the container as volumes. | +| `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` | Adds additional volume mounts to the Docker container. The value of the environment variable should be a comma-separated list of mounts. All such mounts must be given as "source:dest:mode". Note that as of YARN-5298, localized directories are automatically mounted into the container as volumes. | The first two are required. The remainder can be set as needed. While controlling the container type through environment variables is somewhat less @@ -236,6 +246,59 @@ the application will behave exactly as any other YARN application. Logs will be aggregated and stored in the relevant history server. The application life cycle will be the same as for a non-Docker application. +Using Docker Bind Mounted Volumes +--------------------------------- + +Files and directories from the host are commonly needed within the Docker +containers, which Docker provides through [volumes](https://docs.docker.com/engine/tutorials/dockervolumes/). +Examples include localized resources, Apache Hadoop binaries, and sockets. To +facilitate this need, YARN-5534 added the ability for administrators to set a +whitelist of host directories that are allowed to be bind mounted as volumes +into containers. + +In order to make use of this feature, the following must be configured. + +* The administrator must define the volume whitelist by setting the yarn-site.xml configuration property `yarn.nodemanager.runtime.linux.docker.white-list-volume-mounts`. +* The application submitter requests the required volumes at application submission time using the `YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS` environment variable. + +The administrator supplied whitelist is defined as a comma separated list in +the form :. The source is the file on the host. The mode is used +to restrict the allowed mount mode, which can be ro (read-only) or rw +(read-write). Note that localized resources are automatically whitelisted in +read-only mode. + +The user supplied mount list is defined as a comma separated list in the form +::. The source is the file on the host. The mode +defines the mode the user expects for the mount, which can be ro (read-only) or +rw (read-write). The destination is the path within the contatiner where the +source will be bind mounted. + +Several restrictions are in place to limit host exposure. For a mount +request to succeed, the following must be true. If any of the following +conditions are not met, the container launch will be failed. + +* The source path cannot reference parent directories via the .. notation. +* The source path must exist on the Node manager. +* The source path cannot be a symlink. +* The mount mode must be "ro" or "rw". +* The mount mode must be allowed by the admin whitelist. +* The mount must be in the admin whitelist or a localized resource. +* The requested mount mode but be less than or equal to the admin supplied mode. + +The following example outlines how to use this feature to mount the commonly +needed /sys/fs/cgroup directory into the container running on YARN. + +The administrator sets yarn.nodemanager.runtime.linux.docker.white-list-volume-mounts +in yarn-site.xml to "/sys/fs/cgroup:ro" and restarts the Node Manager. +Applications can now request that /sys/fs/cgroup be mounted from the host into +the container in read-only mode. + +At application submission time, the YARN_CONTAINER_RUNTIME_DOCKER_MOUNTS +environment variable can then be set to request this mount. In this example, +the environment variable would be set to "/sys/fs/cgroup:/sys/fs/cgroup:ro". +The destination path is not restricted, "/sys/fs/cgroup:/cgroup:ro" would also +be valid given the example admin whitelist. + Connecting to a Secure Docker Repository ----------------------------------------