From 464767e19e73e4c78a08e685a66aecfc9d22c350 Mon Sep 17 00:00:00 2001 From: Craig Condit Date: Wed, 5 Sep 2018 10:47:54 -0500 Subject: [PATCH] YARN-6456. Allow administrators to set a single ContainerRuntime for all containers. --- .../apache/hadoop/yarn/conf/YarnConfiguration.java | 12 ++ .../src/main/resources/yarn-default.xml | 20 +++ .../containermanager/container/ContainerImpl.java | 3 + .../containermanager/launcher/ContainerLaunch.java | 1 + .../resources/gpu/GpuResourceHandlerImpl.java | 3 + .../runtime/DefaultLinuxContainerRuntime.java | 6 +- .../linux/runtime/DockerLinuxContainerRuntime.java | 49 +++++-- .../resources/gpu/TestGpuResourceHandler.java | 9 +- .../linux/runtime/TestDockerContainerRuntime.java | 152 ++++++++++++++++++++- 9 files changed, 235 insertions(+), 20 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index d681c03abbd..5c936b8e0c3 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1882,9 +1882,21 @@ public static boolean isAclEnabled(Configuration conf) { public static final String[] DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES = {"default"}; + /** Default runtime to be used. */ + public static final String LINUX_CONTAINER_RUNTIME_TYPE = + LINUX_CONTAINER_RUNTIME_PREFIX + "type"; + public static final String DOCKER_CONTAINER_RUNTIME_PREFIX = LINUX_CONTAINER_RUNTIME_PREFIX + "docker."; + /** Comma-separated list of allowed docker images. */ + public static final String NM_DOCKER_ALLOWED_IMAGES = + DOCKER_CONTAINER_RUNTIME_PREFIX + "allowed-images"; + + /** Default docker image to be used. */ + public static final String NM_DOCKER_IMAGE_NAME = + DOCKER_CONTAINER_RUNTIME_PREFIX + "image-name"; + /** Capabilities allowed (and added by default) for docker containers. **/ public static final String NM_DOCKER_CONTAINER_CAPABILITIES = DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 42624365557..3357e543931 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1725,6 +1725,12 @@ default + + Default container runtime to use. + yarn.nodemanager.runtime.linux.type + + + This configuration setting determines the capabilities assigned to docker containers when they are launched. While these may not @@ -1735,6 +1741,20 @@ CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE + + Comma-separated list of allowed docker images. An empty list + is equivalent to allowing all images. + yarn.nodemanager.runtime.linux.docker.allowed-images + + + + + Default docker image to be used when the docker runtime is + selected. + yarn.nodemanager.runtime.linux.docker.image-name + + + This configuration setting determines if privileged docker containers are allowed on this cluster. Privileged containers are granted diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index e4cbfdce592..f88dfbfb716 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -1544,6 +1544,7 @@ public void transition(ContainerImpl container, ContainerEvent event) { // TODO: Add containerWorkDir to the deletion service. if (DockerLinuxContainerRuntime.isDockerContainerRequested( + container.daemonConf, container.getLaunchContext().getEnvironment())) { removeDockerContainer(container); } @@ -1584,6 +1585,7 @@ public void transition(ContainerImpl container, ContainerEvent event) { // TODO: Add containerOuputDir to the deletion service. if (DockerLinuxContainerRuntime.isDockerContainerRequested( + container.daemonConf, container.getLaunchContext().getEnvironment())) { removeDockerContainer(container); } @@ -1858,6 +1860,7 @@ public void transition(ContainerImpl container, ContainerEvent event) { } if (DockerLinuxContainerRuntime.isDockerContainerRequested( + container.daemonConf, container.getLaunchContext().getEnvironment())) { removeDockerContainer(container); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 2aca5f8f2cd..3a252680a4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -798,6 +798,7 @@ public void cleanupContainer() throws IOException { } // The Docker container may not have fully started, reap the container. if (DockerLinuxContainerRuntime.isDockerContainerRequested( + conf, container.getLaunchContext().getEnvironment())) { reapDockerContainerNoPid(user); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java index 118438296b1..e25a6b5bee7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java @@ -48,6 +48,7 @@ public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus"; public static final String CONTAINER_ID_CLI_OPTION = "--container_id"; + private Context nmContext; private GpuResourceAllocator gpuAllocator; private CGroupsHandler cGroupsHandler; private PrivilegedOperationExecutor privilegedOperationExecutor; @@ -55,6 +56,7 @@ public GpuResourceHandlerImpl(Context nmContext, CGroupsHandler cGroupsHandler, PrivilegedOperationExecutor privilegedOperationExecutor) { + this.nmContext = nmContext; this.cGroupsHandler = cGroupsHandler; this.privilegedOperationExecutor = privilegedOperationExecutor; gpuAllocator = new GpuResourceAllocator(nmContext); @@ -102,6 +104,7 @@ public GpuResourceHandlerImpl(Context nmContext, cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES, containerIdStr); if (!DockerLinuxContainerRuntime.isDockerContainerRequested( + nmContext.getConf(), container.getLaunchContext().getEnvironment())) { // Write to devices cgroup only for non-docker container. The reason is // docker engine runtime runc do the devices cgroups initialize in the diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java index 2b8e8adfba4..82ca6d91f24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -71,7 +72,10 @@ public DefaultLinuxContainerRuntime(PrivilegedOperationExecutor @Override public boolean isRuntimeRequested(Map env) { String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE); - return type == null || type.equals("default"); + if (type == null) { + type = conf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE); + } + return type == null || type.isEmpty() || type.equals("default"); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 0665269339b..9cd60b0d1da 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -236,6 +236,8 @@ private Context nmContext; private DockerClient dockerClient; private PrivilegedOperationExecutor privilegedOperationExecutor; + private Set allowedImages = new HashSet<>(); + private String defaultImageName; private Set allowedNetworks = new HashSet<>(); private String defaultNetwork; private CGroupsHandler cGroupsHandler; @@ -256,17 +258,17 @@ * called {@code YARN_CONTAINER_RUNTIME_TYPE} whose value is {@code docker}, * this method will return true. Otherwise it will return false. * + * @param daemonConf the NodeManager daemon configuration * @param env the environment variable settings for the operation * @return whether a Docker container is requested */ - public static boolean isDockerContainerRequested( + public static boolean isDockerContainerRequested(Configuration daemonConf, Map env) { - if (env == null) { - return false; + String type = (env == null) + ? null : env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE); + if (type == null) { + type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE); } - - String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE); - return type != null && type.equals("docker"); } @@ -310,10 +312,15 @@ public void initialize(Configuration conf, Context nmContext) this.nmContext = nmContext; this.conf = conf; dockerClient = new DockerClient(); + allowedImages.clear(); allowedNetworks.clear(); defaultROMounts.clear(); defaultRWMounts.clear(); defaultTmpfsMounts.clear(); + allowedImages.addAll(Arrays.asList( + conf.getTrimmedStrings(YarnConfiguration.NM_DOCKER_ALLOWED_IMAGES))); + defaultImageName = conf.getTrimmed( + YarnConfiguration.NM_DOCKER_IMAGE_NAME, ""); allowedNetworks.addAll(Arrays.asList( conf.getTrimmedStrings( YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, @@ -322,13 +329,24 @@ public void initialize(Configuration conf, Context nmContext) YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK, YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK); + if (!defaultImageName.isEmpty() + && !allowedImages.isEmpty() + && !allowedImages.contains(defaultImageName)) { + String message = "Default image: " + defaultImageName + + " is not in the set of allowed images: " + allowedImages; + + if (LOG.isWarnEnabled()) { + LOG.warn(message + ". Please check configuration"); + } + + throw new ContainerExecutionException(message); + } if(!allowedNetworks.contains(defaultNetwork)) { String message = "Default network: " + defaultNetwork + " is not in the set of allowed networks: " + allowedNetworks; if (LOG.isWarnEnabled()) { - LOG.warn(message + ". Please check " - + "configuration"); + LOG.warn(message + ". Please check configuration"); } throw new ContainerExecutionException(message); @@ -375,7 +393,7 @@ public void initialize(Configuration conf, Context nmContext) @Override public boolean isRuntimeRequested(Map env) { - return isDockerContainerRequested(env); + return isDockerContainerRequested(conf, env); } private Set getDockerCapabilitiesFromConf() throws @@ -795,6 +813,9 @@ public void launchContainer(ContainerRuntimeContext ctx) String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME); boolean useEntryPoint = checkUseEntryPoint(environment); + if (imageName == null || imageName.isEmpty()) { + imageName = defaultImageName; + } if(network == null || network.isEmpty()) { network = defaultNetwork; } @@ -803,7 +824,7 @@ public void launchContainer(ContainerRuntimeContext ctx) validateHostname(hostname); - validateImageName(imageName); + validateImageName(allowedImages, imageName); String containerIdStr = containerId.toString(); String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER); @@ -1239,8 +1260,8 @@ private PrivilegedOperation buildLaunchOp(ContainerRuntimeContext ctx, return launchOp; } - public static void validateImageName(String imageName) - throws ContainerExecutionException { + public static void validateImageName(Set allowedImages, + String imageName) throws ContainerExecutionException { if (imageName == null || imageName.isEmpty()) { throw new ContainerExecutionException( ENV_DOCKER_CONTAINER_IMAGE + " not set!"); @@ -1249,6 +1270,10 @@ public static void validateImageName(String imageName) throw new ContainerExecutionException("Image name '" + imageName + "' doesn't match docker image name pattern"); } + if (!allowedImages.isEmpty() && !allowedImages.contains(imageName)) { + throw new ContainerExecutionException("Image name '" + imageName + + "' is not allowed"); + } } private void executeLivelinessCheck(ContainerRuntimeContext ctx) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java index 9a8a4c9f284..301c3e580ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java @@ -80,8 +80,11 @@ public void setup() { mockPrivilegedExecutor = mock(PrivilegedOperationExecutor.class); mockNMStateStore = mock(NMStateStoreService.class); + Configuration conf = new Configuration(); + Context nmctx = mock(Context.class); when(nmctx.getNMStateStore()).thenReturn(mockNMStateStore); + when(nmctx.getConf()).thenReturn(conf); runningContainersMap = new ConcurrentHashMap<>(); when(nmctx.getContainers()).thenReturn(runningContainersMap); @@ -347,15 +350,17 @@ public void testAllocationStored() throws Exception { public void testAllocationStoredWithNULLStateStore() throws Exception { NMNullStateStoreService mockNMNULLStateStore = mock(NMNullStateStoreService.class); + Configuration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4"); + Context nmnctx = mock(Context.class); when(nmnctx.getNMStateStore()).thenReturn(mockNMNULLStateStore); + when(nmnctx.getConf()).thenReturn(conf); GpuResourceHandlerImpl gpuNULLStateResourceHandler = new GpuResourceHandlerImpl(nmnctx, mockCGroupsHandler, mockPrivilegedExecutor); - Configuration conf = new YarnConfiguration(); - conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4"); GpuDiscoverer.getInstance().initialize(conf); gpuNULLStateResourceHandler.bootstrap(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 2f171671f23..935f02de703 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -75,8 +75,10 @@ import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -314,11 +316,45 @@ public void testSelectDockerContainerType() { envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other"); Assert.assertEquals(false, DockerLinuxContainerRuntime - .isDockerContainerRequested(null)); + .isDockerContainerRequested(conf, null)); Assert.assertEquals(true, DockerLinuxContainerRuntime - .isDockerContainerRequested(envDockerType)); + .isDockerContainerRequested(conf, envDockerType)); Assert.assertEquals(false, DockerLinuxContainerRuntime - .isDockerContainerRequested(envOtherType)); + .isDockerContainerRequested(conf, envOtherType)); + } + + @Test + public void testSelectDockerContainerTypeWithDockerAsDefault() { + Map envDockerType = new HashMap<>(); + Map envOtherType = new HashMap<>(); + + conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "docker"); + envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker"); + envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other"); + + Assert.assertEquals(true, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, null)); + Assert.assertEquals(true, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envDockerType)); + Assert.assertEquals(false, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envOtherType)); + } + + @Test + public void testSelectDockerContainerTypeWithDefaultSet() { + Map envDockerType = new HashMap<>(); + Map envOtherType = new HashMap<>(); + + conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "default"); + envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker"); + envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other"); + + Assert.assertEquals(false, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, null)); + Assert.assertEquals(true, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envDockerType)); + Assert.assertEquals(false, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envOtherType)); } private PrivilegedOperation capturePrivilegedOperation() @@ -426,6 +462,57 @@ public void testDockerContainerLaunch() dockerCommands.get(counter)); } + @Test + public void testDockerContainerLaunchWithDefaultImage() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + conf.set(YarnConfiguration.NM_DOCKER_IMAGE_NAME, "busybox:1.2.3"); + env.remove(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE); + + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + runtime.launchContainer(builder.build()); + + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines(Paths.get + (dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 13; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" group-add=" + String.join(",", groups), + dockerCommands.get(counter++)); + Assert + .assertEquals(" image=busybox:1.2.3", dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals(" mounts=" + + "/test_container_log_dir:/test_container_log_dir:rw," + + "/test_application_local_dir:/test_application_local_dir:rw," + + "/test_filecache_dir:/test_filecache_dir:ro," + + "/test_user_filecache_dir:/test_user_filecache_dir:ro", + dockerCommands.get(counter++)); + Assert.assertEquals( + " name=container_e11_1518975676334_14532816_01_000001", + dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter)); + } + @Test public void testContainerLaunchWithUserRemapping() throws ContainerExecutionException, PrivilegedOperationException, @@ -476,6 +563,40 @@ public void testContainerLaunchWithUserRemapping() dockerCommands.get(counter)); } + @Test + public void testAllowedImagesConfiguration() throws + ContainerExecutionException { + // default configuration should be valid + DockerLinuxContainerRuntime runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + // setting a default image should be valid + conf.set(YarnConfiguration.NM_DOCKER_IMAGE_NAME, "busybox:latest"); + runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + // setting a default image which is in an allowed list should be valid + conf.setStrings(YarnConfiguration.NM_DOCKER_ALLOWED_IMAGES, + "centos", "busybox:latest", "ubuntu"); + runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + // default image not in allowed images should fail + conf.setStrings(YarnConfiguration.NM_DOCKER_ALLOWED_IMAGES, + "centos", "ubuntu"); + try { + runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + Assert.fail("Invalid default image did not trigger failure"); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + @Test public void testAllowedNetworksConfiguration() throws ContainerExecutionException { @@ -1876,18 +1997,39 @@ public void testDockerImageNamePattern() throws Exception { ":8080/ubuntu" }; + String[] allowedNames = { "ubuntu", "fedora/httpd:version1.0" }; + String[] disallowedNames = { "dangerous", "ubuntu:insecureVersion"}; + + Set emptyAllowedNames = Collections.emptySet(); + Set presentAllowedNames = new HashSet<>(); + presentAllowedNames.addAll(Arrays.asList(allowedNames)); + for (String name : validNames) { - DockerLinuxContainerRuntime.validateImageName(name); + DockerLinuxContainerRuntime.validateImageName(emptyAllowedNames, name); } for (String name : invalidNames) { try { - DockerLinuxContainerRuntime.validateImageName(name); + DockerLinuxContainerRuntime.validateImageName(emptyAllowedNames, name); Assert.fail(name + " is an invalid name and should fail the regex"); } catch (ContainerExecutionException ce) { continue; } } + + for (String name : allowedNames) { + DockerLinuxContainerRuntime.validateImageName(presentAllowedNames, name); + } + + for (String name : disallowedNames) { + try { + DockerLinuxContainerRuntime.validateImageName( + presentAllowedNames, name); + Assert.fail(name + " is not in the allowed image list and should fail"); + } catch (ContainerExecutionException ce) { + continue; + } + } } @Test -- 2.15.2 (Apple Git-101.1)