From cbf4669ab035ca664c88bfeaf29676cb9de34103 Mon Sep 17 00:00:00 2001 From: Craig Condit Date: Wed, 12 Sep 2018 10:16:11 -0500 Subject: [PATCH] YARN-6456. Allow administrators to set a single ContainerRuntime for all containers. --- .../hadoop/yarn/conf/YarnConfiguration.java | 12 ++ .../src/main/resources/yarn-default.xml | 20 +++ .../container/ContainerImpl.java | 3 + .../launcher/ContainerLaunch.java | 1 + .../resources/gpu/GpuResourceHandlerImpl.java | 3 + .../runtime/DefaultLinuxContainerRuntime.java | 6 +- .../runtime/DockerLinuxContainerRuntime.java | 55 ++++-- .../resources/gpu/TestGpuResourceHandler.java | 9 +- .../runtime/TestDockerContainerRuntime.java | 156 +++++++++++++++++- .../src/site/markdown/DockerContainers.md | 26 +++ 10 files changed, 269 insertions(+), 22 deletions(-) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index a82801d620e..0d3ac005176 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1882,9 +1882,21 @@ public static boolean isAclEnabled(Configuration conf) { public static final String[] DEFAULT_LINUX_CONTAINER_RUNTIME_ALLOWED_RUNTIMES = {"default"}; + /** Default runtime to be used. */ + public static final String LINUX_CONTAINER_RUNTIME_TYPE = + LINUX_CONTAINER_RUNTIME_PREFIX + "type"; + public static final String DOCKER_CONTAINER_RUNTIME_PREFIX = LINUX_CONTAINER_RUNTIME_PREFIX + "docker."; + /** Comma-separated list of allowed docker images. */ + public static final String NM_DOCKER_ALLOWED_IMAGES = + DOCKER_CONTAINER_RUNTIME_PREFIX + "allowed-images"; + + /** Default docker image to be used. */ + public static final String NM_DOCKER_IMAGE_NAME = + DOCKER_CONTAINER_RUNTIME_PREFIX + "image-name"; + /** Capabilities allowed (and added by default) for docker containers. **/ public static final String NM_DOCKER_CONTAINER_CAPABILITIES = DOCKER_CONTAINER_RUNTIME_PREFIX + "capabilities"; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 0700902e9cd..43f4f2d4a94 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1725,6 +1725,12 @@ default + + Default container runtime to use. + yarn.nodemanager.runtime.linux.type + + + This configuration setting determines the capabilities assigned to docker containers when they are launched. While these may not @@ -1735,6 +1741,20 @@ CHOWN,DAC_OVERRIDE,FSETID,FOWNER,MKNOD,NET_RAW,SETGID,SETUID,SETFCAP,SETPCAP,NET_BIND_SERVICE,SYS_CHROOT,KILL,AUDIT_WRITE + + Comma-separated list of allowed docker images. An empty list + is equivalent to allowing all images. + yarn.nodemanager.runtime.linux.docker.allowed-images + + + + + Default docker image to be used when the docker runtime is + selected. + yarn.nodemanager.runtime.linux.docker.image-name + + + This configuration setting determines if privileged docker containers are allowed on this cluster. Privileged containers are granted diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index e4cbfdce592..f88dfbfb716 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -1544,6 +1544,7 @@ public void transition(ContainerImpl container, ContainerEvent event) { // TODO: Add containerWorkDir to the deletion service. if (DockerLinuxContainerRuntime.isDockerContainerRequested( + container.daemonConf, container.getLaunchContext().getEnvironment())) { removeDockerContainer(container); } @@ -1584,6 +1585,7 @@ public void transition(ContainerImpl container, ContainerEvent event) { // TODO: Add containerOuputDir to the deletion service. if (DockerLinuxContainerRuntime.isDockerContainerRequested( + container.daemonConf, container.getLaunchContext().getEnvironment())) { removeDockerContainer(container); } @@ -1858,6 +1860,7 @@ public void transition(ContainerImpl container, ContainerEvent event) { } if (DockerLinuxContainerRuntime.isDockerContainerRequested( + container.daemonConf, container.getLaunchContext().getEnvironment())) { removeDockerContainer(container); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index 2aca5f8f2cd..3a252680a4b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -798,6 +798,7 @@ public void cleanupContainer() throws IOException { } // The Docker container may not have fully started, reap the container. if (DockerLinuxContainerRuntime.isDockerContainerRequested( + conf, container.getLaunchContext().getEnvironment())) { reapDockerContainerNoPid(user); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java index 118438296b1..e25a6b5bee7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/GpuResourceHandlerImpl.java @@ -48,6 +48,7 @@ public static final String EXCLUDED_GPUS_CLI_OPTION = "--excluded_gpus"; public static final String CONTAINER_ID_CLI_OPTION = "--container_id"; + private Context nmContext; private GpuResourceAllocator gpuAllocator; private CGroupsHandler cGroupsHandler; private PrivilegedOperationExecutor privilegedOperationExecutor; @@ -55,6 +56,7 @@ public GpuResourceHandlerImpl(Context nmContext, CGroupsHandler cGroupsHandler, PrivilegedOperationExecutor privilegedOperationExecutor) { + this.nmContext = nmContext; this.cGroupsHandler = cGroupsHandler; this.privilegedOperationExecutor = privilegedOperationExecutor; gpuAllocator = new GpuResourceAllocator(nmContext); @@ -102,6 +104,7 @@ public GpuResourceHandlerImpl(Context nmContext, cGroupsHandler.createCGroup(CGroupsHandler.CGroupController.DEVICES, containerIdStr); if (!DockerLinuxContainerRuntime.isDockerContainerRequested( + nmContext.getConf(), container.getLaunchContext().getEnvironment())) { // Write to devices cgroup only for non-docker container. The reason is // docker engine runtime runc do the devices cgroups initialize in the diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java index 2b8e8adfba4..82ca6d91f24 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java @@ -24,6 +24,7 @@ import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; +import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor; import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; @@ -71,7 +72,10 @@ public DefaultLinuxContainerRuntime(PrivilegedOperationExecutor @Override public boolean isRuntimeRequested(Map env) { String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE); - return type == null || type.equals("default"); + if (type == null) { + type = conf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE); + } + return type == null || type.isEmpty() || type.equals("default"); } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 8b2b4048765..9df097489c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -235,6 +235,8 @@ private Context nmContext; private DockerClient dockerClient; private PrivilegedOperationExecutor privilegedOperationExecutor; + private Set allowedImages = new HashSet<>(); + private String defaultImageName; private Set allowedNetworks = new HashSet<>(); private String defaultNetwork; private CGroupsHandler cGroupsHandler; @@ -254,17 +256,17 @@ * called {@code YARN_CONTAINER_RUNTIME_TYPE} whose value is {@code docker}, * this method will return true. Otherwise it will return false. * + * @param daemonConf the NodeManager daemon configuration * @param env the environment variable settings for the operation * @return whether a Docker container is requested */ - public static boolean isDockerContainerRequested( + public static boolean isDockerContainerRequested(Configuration daemonConf, Map env) { - if (env == null) { - return false; + String type = (env == null) + ? null : env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE); + if (type == null) { + type = daemonConf.get(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE); } - - String type = env.get(ContainerRuntimeConstants.ENV_CONTAINER_TYPE); - return type != null && type.equals("docker"); } @@ -308,10 +310,15 @@ public void initialize(Configuration conf, Context nmContext) this.nmContext = nmContext; this.conf = conf; dockerClient = new DockerClient(); + allowedImages.clear(); allowedNetworks.clear(); defaultROMounts.clear(); defaultRWMounts.clear(); defaultTmpfsMounts.clear(); + allowedImages.addAll(Arrays.asList( + conf.getTrimmedStrings(YarnConfiguration.NM_DOCKER_ALLOWED_IMAGES))); + defaultImageName = conf.getTrimmed( + YarnConfiguration.NM_DOCKER_IMAGE_NAME, ""); allowedNetworks.addAll(Arrays.asList( conf.getTrimmedStrings( YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, @@ -320,13 +327,30 @@ public void initialize(Configuration conf, Context nmContext) YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK, YarnConfiguration.DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK); + if (!defaultImageName.isEmpty() + && !allowedImages.isEmpty() + && !allowedImages.contains(defaultImageName)) { + String message = "Default image: " + defaultImageName + + " is not in the set of allowed images: " + allowedImages; + + if (LOG.isWarnEnabled()) { + LOG.warn(message + ". Please check configuration"); + } + + throw new ContainerExecutionException(message); + } + for (String imageName: allowedImages) { + if (!dockerImagePattern.matcher(imageName).matches()) { + throw new ContainerExecutionException("Image name '" + imageName + + "' doesn't match docker image name pattern"); + } + } if(!allowedNetworks.contains(defaultNetwork)) { String message = "Default network: " + defaultNetwork + " is not in the set of allowed networks: " + allowedNetworks; if (LOG.isWarnEnabled()) { - LOG.warn(message + ". Please check " - + "configuration"); + LOG.warn(message + ". Please check configuration"); } throw new ContainerExecutionException(message); @@ -369,7 +393,7 @@ public void initialize(Configuration conf, Context nmContext) @Override public boolean isRuntimeRequested(Map env) { - return isDockerContainerRequested(env); + return isDockerContainerRequested(conf, env); } private Set getDockerCapabilitiesFromConf() throws @@ -789,6 +813,9 @@ public void launchContainer(ContainerRuntimeContext ctx) String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME); boolean useEntryPoint = checkUseEntryPoint(environment); + if (imageName == null || imageName.isEmpty()) { + imageName = defaultImageName; + } if(network == null || network.isEmpty()) { network = defaultNetwork; } @@ -797,7 +824,7 @@ public void launchContainer(ContainerRuntimeContext ctx) validateHostname(hostname); - validateImageName(imageName); + validateImageName(allowedImages, imageName); String containerIdStr = containerId.toString(); String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER); @@ -1226,8 +1253,8 @@ private PrivilegedOperation buildLaunchOp(ContainerRuntimeContext ctx, return launchOp; } - public static void validateImageName(String imageName) - throws ContainerExecutionException { + public static void validateImageName(Set allowedImages, + String imageName) throws ContainerExecutionException { if (imageName == null || imageName.isEmpty()) { throw new ContainerExecutionException( ENV_DOCKER_CONTAINER_IMAGE + " not set!"); @@ -1236,6 +1263,10 @@ public static void validateImageName(String imageName) throw new ContainerExecutionException("Image name '" + imageName + "' doesn't match docker image name pattern"); } + if (!allowedImages.isEmpty() && !allowedImages.contains(imageName)) { + throw new ContainerExecutionException("Image name '" + imageName + + "' is not allowed"); + } } private void executeLivelinessCheck(ContainerRuntimeContext ctx) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java index 9a8a4c9f284..301c3e580ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/gpu/TestGpuResourceHandler.java @@ -80,8 +80,11 @@ public void setup() { mockPrivilegedExecutor = mock(PrivilegedOperationExecutor.class); mockNMStateStore = mock(NMStateStoreService.class); + Configuration conf = new Configuration(); + Context nmctx = mock(Context.class); when(nmctx.getNMStateStore()).thenReturn(mockNMStateStore); + when(nmctx.getConf()).thenReturn(conf); runningContainersMap = new ConcurrentHashMap<>(); when(nmctx.getContainers()).thenReturn(runningContainersMap); @@ -347,15 +350,17 @@ public void testAllocationStored() throws Exception { public void testAllocationStoredWithNULLStateStore() throws Exception { NMNullStateStoreService mockNMNULLStateStore = mock(NMNullStateStoreService.class); + Configuration conf = new YarnConfiguration(); + conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4"); + Context nmnctx = mock(Context.class); when(nmnctx.getNMStateStore()).thenReturn(mockNMNULLStateStore); + when(nmnctx.getConf()).thenReturn(conf); GpuResourceHandlerImpl gpuNULLStateResourceHandler = new GpuResourceHandlerImpl(nmnctx, mockCGroupsHandler, mockPrivilegedExecutor); - Configuration conf = new YarnConfiguration(); - conf.set(YarnConfiguration.NM_GPU_ALLOWED_DEVICES, "0:0,1:1,2:3,3:4"); GpuDiscoverer.getInstance().initialize(conf); gpuNULLStateResourceHandler.bootstrap(conf); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 4464496e08c..cb294192fd5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -75,8 +75,10 @@ import java.nio.file.attribute.PosixFilePermission; import java.nio.file.attribute.PosixFilePermissions; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -309,11 +311,45 @@ public void testSelectDockerContainerType() { envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other"); Assert.assertEquals(false, DockerLinuxContainerRuntime - .isDockerContainerRequested(null)); + .isDockerContainerRequested(conf, null)); Assert.assertEquals(true, DockerLinuxContainerRuntime - .isDockerContainerRequested(envDockerType)); + .isDockerContainerRequested(conf, envDockerType)); Assert.assertEquals(false, DockerLinuxContainerRuntime - .isDockerContainerRequested(envOtherType)); + .isDockerContainerRequested(conf, envOtherType)); + } + + @Test + public void testSelectDockerContainerTypeWithDockerAsDefault() { + Map envDockerType = new HashMap<>(); + Map envOtherType = new HashMap<>(); + + conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "docker"); + envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker"); + envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other"); + + Assert.assertEquals(true, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, null)); + Assert.assertEquals(true, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envDockerType)); + Assert.assertEquals(false, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envOtherType)); + } + + @Test + public void testSelectDockerContainerTypeWithDefaultSet() { + Map envDockerType = new HashMap<>(); + Map envOtherType = new HashMap<>(); + + conf.set(YarnConfiguration.LINUX_CONTAINER_RUNTIME_TYPE, "default"); + envDockerType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "docker"); + envOtherType.put(ContainerRuntimeConstants.ENV_CONTAINER_TYPE, "other"); + + Assert.assertEquals(false, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, null)); + Assert.assertEquals(true, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envDockerType)); + Assert.assertEquals(false, DockerLinuxContainerRuntime + .isDockerContainerRequested(conf, envOtherType)); } private PrivilegedOperation capturePrivilegedOperation() @@ -421,6 +457,57 @@ public void testDockerContainerLaunch() dockerCommands.get(counter)); } + @Test + public void testDockerContainerLaunchWithDefaultImage() + throws ContainerExecutionException, PrivilegedOperationException, + IOException { + conf.set(YarnConfiguration.NM_DOCKER_IMAGE_NAME, "busybox:1.2.3"); + env.remove(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE); + + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + runtime.launchContainer(builder.build()); + + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines(Paths.get + (dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 13; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" group-add=" + String.join(",", groups), + dockerCommands.get(counter++)); + Assert + .assertEquals(" image=busybox:1.2.3", dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals(" mounts=" + + "/test_container_log_dir:/test_container_log_dir:rw," + + "/test_application_local_dir:/test_application_local_dir:rw," + + "/test_filecache_dir:/test_filecache_dir:ro," + + "/test_user_filecache_dir:/test_user_filecache_dir:ro", + dockerCommands.get(counter++)); + Assert.assertEquals( + " name=container_e11_1518975676334_14532816_01_000001", + dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" user=" + uidGidPair, dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter)); + } + @Test public void testContainerLaunchWithUserRemapping() throws ContainerExecutionException, PrivilegedOperationException, @@ -471,6 +558,40 @@ public void testContainerLaunchWithUserRemapping() dockerCommands.get(counter)); } + @Test + public void testAllowedImagesConfiguration() throws + ContainerExecutionException { + // default configuration should be valid + DockerLinuxContainerRuntime runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + // setting a default image should be valid + conf.set(YarnConfiguration.NM_DOCKER_IMAGE_NAME, "busybox:latest"); + runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + // setting a default image which is in an allowed list should be valid + conf.setStrings(YarnConfiguration.NM_DOCKER_ALLOWED_IMAGES, + "centos", "busybox:latest", "ubuntu"); + runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + + // default image not in allowed images should fail + conf.setStrings(YarnConfiguration.NM_DOCKER_ALLOWED_IMAGES, + "centos", "ubuntu"); + try { + runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, nmContext); + Assert.fail("Invalid default image did not trigger failure"); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + @Test public void testAllowedNetworksConfiguration() throws ContainerExecutionException { @@ -794,8 +915,8 @@ public void testLaunchPidNamespaceContainersInvalidEnvVar() List args = op.getArguments(); String dockerCommandFile = args.get(11); - List dockerCommands = Files.readAllLines(Paths.get - (dockerCommandFile), Charset.forName("UTF-8")); + List dockerCommands = Files.readAllLines(Paths.get( + dockerCommandFile), Charset.forName("UTF-8")); int expected = 13; Assert.assertEquals(expected, dockerCommands.size()); @@ -1875,18 +1996,39 @@ public void testDockerImageNamePattern() throws Exception { ":8080/ubuntu" }; + String[] allowedNames = {"ubuntu", "fedora/httpd:version1.0"}; + String[] disallowedNames = {"dangerous", "ubuntu:insecureVersion"}; + + Set emptyAllowedNames = Collections.emptySet(); + Set presentAllowedNames = new HashSet<>(); + presentAllowedNames.addAll(Arrays.asList(allowedNames)); + for (String name : validNames) { - DockerLinuxContainerRuntime.validateImageName(name); + DockerLinuxContainerRuntime.validateImageName(emptyAllowedNames, name); } for (String name : invalidNames) { try { - DockerLinuxContainerRuntime.validateImageName(name); + DockerLinuxContainerRuntime.validateImageName(emptyAllowedNames, name); Assert.fail(name + " is an invalid name and should fail the regex"); } catch (ContainerExecutionException ce) { continue; } } + + for (String name : allowedNames) { + DockerLinuxContainerRuntime.validateImageName(presentAllowedNames, name); + } + + for (String name : disallowedNames) { + try { + DockerLinuxContainerRuntime.validateImageName( + presentAllowedNames, name); + Assert.fail(name + " is not in the allowed image list and should fail"); + } catch (ContainerExecutionException ce) { + continue; + } + } } @Test diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index a3ce93f92a1..42667fac246 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -114,6 +114,32 @@ The following properties should be set in yarn-site.xml: + + yarn.nodemanager.runtime.linux.type + + + Optional. Sets the default container runtime to use. + + + + + + yarn.nodemanager.runtime.linux.docker.allowed-images + + Optional. Comma-separated list of allowed docker images. An empty list + is equivalent to allowing all images. + + + + + yarn.nodemanager.runtime.linux.docker.image-name + + + Optional. Default docker image to be used when the docker runtime is + selected. + + + yarn.nodemanager.runtime.linux.docker.allowed-container-networks host,none,bridge -- 2.18.0