commit f8bc8a0622ebbcd203e761bea106301cd42f2cea Author: Eric Yang Date: Fri Dec 1 18:40:14 2017 -0500 YARN-7516. Implement security check for untrusted docker image. (Contributed by Eric Yang) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index 831abf5..bedb434 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -3422,6 +3422,9 @@ public static boolean areNodeLabelsEnabled( public static final String TIMELINE_XFS_OPTIONS = TIMELINE_XFS_PREFIX + "xframe-options"; + public static final String TRUSTED_DOCKER_REGISTRY = YARN_PREFIX + + "docker.trusted.registry"; + public YarnConfiguration() { super(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 2550c42..5686237 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -3599,4 +3599,13 @@ 0,1 + + + Define a list of trusted docker registry that can download certified docker image. + Without using trusted registry, docker instance is not allowed to mount external + file system for security reasons. + + yarn.docker.trusted.registry + + diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 20359ea..9579a2a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -651,10 +651,20 @@ private String getUserIdInfo(String userName) @Override public void launchContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException { + boolean trusted = false; Container container = ctx.getContainer(); Map environment = container.getLaunchContext() .getEnvironment(); String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE); + String[] trustedDockerRegistry = conf + .getStrings(YarnConfiguration.TRUSTED_DOCKER_REGISTRY); + if (trustedDockerRegistry != null) { + for (String prefix : trustedDockerRegistry) { + if (imageName.startsWith(prefix)) { + trusted = true; + } + } + } String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK); String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME); @@ -724,60 +734,62 @@ public void launchContainer(ContainerRuntimeContext ctx) cgroupsRootDirectory, false); } - List allDirs = new ArrayList<>(containerLocalDirs); - allDirs.addAll(filecacheDirs); - allDirs.add(containerWorkDir.toString()); - allDirs.addAll(containerLogDirs); - allDirs.addAll(userLocalDirs); - for (String dir: allDirs) { - runCommand.addMountLocation(dir, dir, true); - } - - if (environment.containsKey(ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS)) { - String mounts = environment.get( - ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS); - if (!mounts.isEmpty()) { - for (String mount : StringUtils.split(mounts)) { - String[] dir = StringUtils.split(mount, ':'); - if (dir.length != 2) { - throw new ContainerExecutionException("Invalid mount : " + - mount); + if (trusted) { + List allDirs = new ArrayList<>(containerLocalDirs); + allDirs.addAll(filecacheDirs); + allDirs.add(containerWorkDir.toString()); + allDirs.addAll(containerLogDirs); + allDirs.addAll(userLocalDirs); + for (String dir: allDirs) { + runCommand.addMountLocation(dir, dir, true); + } + + if (environment.containsKey(ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS)) { + String mounts = environment.get( + ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS); + if (!mounts.isEmpty()) { + for (String mount : StringUtils.split(mounts)) { + String[] dir = StringUtils.split(mount, ':'); + if (dir.length != 2) { + throw new ContainerExecutionException("Invalid mount : " + + mount); + } + String src = validateMount(dir[0], localizedResources); + String dst = dir[1]; + runCommand.addReadOnlyMountLocation(src, dst, true); } - String src = validateMount(dir[0], localizedResources); - String dst = dir[1]; - runCommand.addReadOnlyMountLocation(src, dst, true); } } - } - if (environment.containsKey(ENV_DOCKER_CONTAINER_MOUNTS)) { - Matcher parsedMounts = USER_MOUNT_PATTERN.matcher( - environment.get(ENV_DOCKER_CONTAINER_MOUNTS)); - if (!parsedMounts.find()) { - throw new ContainerExecutionException( - "Unable to parse user supplied mount list: " - + environment.get(ENV_DOCKER_CONTAINER_MOUNTS)); - } - parsedMounts.reset(); - while (parsedMounts.find()) { - String src = parsedMounts.group(1); - String dst = parsedMounts.group(2); - String mode = parsedMounts.group(3); - if (!mode.equals("ro") && !mode.equals("rw")) { + if (environment.containsKey(ENV_DOCKER_CONTAINER_MOUNTS)) { + Matcher parsedMounts = USER_MOUNT_PATTERN.matcher( + environment.get(ENV_DOCKER_CONTAINER_MOUNTS)); + if (!parsedMounts.find()) { throw new ContainerExecutionException( - "Invalid mount mode requested for mount: " - + parsedMounts.group()); + "Unable to parse user supplied mount list: " + + environment.get(ENV_DOCKER_CONTAINER_MOUNTS)); } - if (mode.equals("ro")) { - runCommand.addReadOnlyMountLocation(src, dst); - } else { - runCommand.addReadWriteMountLocation(src, dst); + parsedMounts.reset(); + while (parsedMounts.find()) { + String src = parsedMounts.group(1); + String dst = parsedMounts.group(2); + String mode = parsedMounts.group(3); + if (!mode.equals("ro") && !mode.equals("rw")) { + throw new ContainerExecutionException( + "Invalid mount mode requested for mount: " + + parsedMounts.group()); + } + if (mode.equals("ro")) { + runCommand.addReadOnlyMountLocation(src, dst); + } else { + runCommand.addReadWriteMountLocation(src, dst); + } } } - } - if (allowPrivilegedContainerExecution(container)) { - runCommand.setPrivileged(); + if (allowPrivilegedContainerExecution(container)) { + runCommand.setPrivileged(); + } } String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 4d32427..3d9c2dc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -142,6 +142,8 @@ public void setup() { conf = new Configuration(); conf.set("hadoop.tmp.dir", tmpPath); + conf.setStrings(YarnConfiguration.TRUSTED_DOCKER_REGISTRY, + "docker.example.com:5000"); mockExecutor = Mockito .mock(PrivilegedOperationExecutor.class); @@ -153,7 +155,7 @@ public void setup() { context = mock(ContainerLaunchContext.class); env = new HashMap(); env.put("FROM_CLIENT", "1"); - image = "busybox:latest"; + image = "docker.example.com:5000/busybox:latest"; env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE, image); when(container.getContainerId()).thenReturn(cId); @@ -343,8 +345,8 @@ public void testDockerContainerLaunch() Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); - Assert - .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); + Assert.assertEquals(" image=docker.example.com:5000/busybox:latest", + dockerCommands.get(counter++)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", dockerCommands.get(counter++)); @@ -431,8 +433,8 @@ public void testContainerLaunchWithUserRemapping() dockerCommands.get(counter++)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); - Assert - .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); + Assert.assertEquals(" image=docker.example.com:5000/busybox:latest", + dockerCommands.get(counter++)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", dockerCommands.get(counter++)); @@ -548,8 +550,8 @@ public void testContainerLaunchWithNetworkingDefaults() Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); Assert.assertEquals(" hostname=test.hostname", dockerCommands.get(counter++)); - Assert - .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); + Assert.assertEquals(" image=docker.example.com:5000/busybox:latest", + dockerCommands.get(counter++)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", dockerCommands.get(counter++)); @@ -613,8 +615,8 @@ public void testContainerLaunchWithCustomNetworks() Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); - Assert - .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); + Assert.assertEquals(" image=docker.example.com:5000/busybox:latest", + dockerCommands.get(counter++)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", dockerCommands.get(counter++)); @@ -656,8 +658,8 @@ public void testContainerLaunchWithCustomNetworks() Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); - Assert - .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); + Assert.assertEquals(" image=docker.example.com:5000/busybox:latest", + dockerCommands.get(counter++)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", dockerCommands.get(counter++)); @@ -828,8 +830,8 @@ public void testLaunchPrivilegedContainersWithEnabledSettingAndDefaultACL() Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); - Assert - .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); + Assert.assertEquals(" image=docker.example.com:5000/busybox:latest", + dockerCommands.get(counter++)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", dockerCommands.get(counter++)); @@ -942,7 +944,8 @@ public void testMountSourceTarget() Assert.assertEquals(" detach=true", dockerCommands.get(3)); Assert.assertEquals(" docker-command=run", dockerCommands.get(4)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5)); - Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6)); + Assert.assertEquals(" image=docker.example.com:5000/busybox:latest", + dockerCommands.get(6)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", dockerCommands.get(7)); @@ -1012,7 +1015,8 @@ public void testMountMultiple() Assert.assertEquals(" detach=true", dockerCommands.get(3)); Assert.assertEquals(" docker-command=run", dockerCommands.get(4)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5)); - Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6)); + Assert.assertEquals(" image=docker.example.com:5000/busybox:latest", + dockerCommands.get(6)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", dockerCommands.get(7)); @@ -1063,7 +1067,8 @@ public void testUserMounts() Assert.assertEquals(" detach=true", dockerCommands.get(3)); Assert.assertEquals(" docker-command=run", dockerCommands.get(4)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(5)); - Assert.assertEquals(" image=busybox:latest", dockerCommands.get(6)); + Assert.assertEquals(" image=docker.example.com:5000/busybox:latest", + dockerCommands.get(6)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", dockerCommands.get(7)); @@ -1510,8 +1515,8 @@ public void testDockerCommandPlugin() throws Exception { Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); - Assert - .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); + Assert.assertEquals(" image=docker.example.com:5000/busybox:latest", + dockerCommands.get(counter++)); Assert.assertEquals( " launch-command=bash,/test_container_work_dir/launch_container.sh", dockerCommands.get(counter++)); @@ -1576,4 +1581,90 @@ public void testDockerCapabilities() Assert.assertEquals("CHOWN", it.next()); Assert.assertEquals("DAC_OVERRIDE", it.next()); } + + /** + * Test docker image originated from trusted registry. When image + * is not from trusted registry, mount options are disabled. + * + * @throws PrivilegedOperationException + * @throws ContainerExecutionException + * @throws IOException + */ + @Test + public void testDockerTrustedRegistry() throws PrivilegedOperationException, + ContainerExecutionException, IOException { + conf.set(YarnConfiguration.TRUSTED_DOCKER_REGISTRY, "abc.example.com:1234"); + DockerLinuxContainerRuntime runtime = + new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler); + when(mockExecutor + .executePrivilegedOperation(anyList(), any(PrivilegedOperation.class), + any(File.class), anyMap(), anyBoolean(), anyBoolean())).thenReturn( + null); + when(mockExecutor + .executePrivilegedOperation(anyList(), any(PrivilegedOperation.class), + any(File.class), anyMap(), anyBoolean(), anyBoolean())).thenReturn( + "volume1,local"); + + Context nmContext = mock(Context.class); + ResourcePluginManager rpm = mock(ResourcePluginManager.class); + Map pluginsMap = new HashMap<>(); + ResourcePlugin plugin1 = mock(ResourcePlugin.class); + + // Create the docker command plugin logic, which will set volume driver + DockerCommandPlugin dockerCommandPlugin = new MockDockerCommandPlugin( + "volume1", "local"); + + when(plugin1.getDockerCommandPluginInstance()).thenReturn( + dockerCommandPlugin); + ResourcePlugin plugin2 = mock(ResourcePlugin.class); + pluginsMap.put("plugin1", plugin1); + pluginsMap.put("plugin2", plugin2); + + when(rpm.getNameToPlugins()).thenReturn(pluginsMap); + + when(nmContext.getResourcePluginManager()).thenReturn(rpm); + + runtime.initialize(conf, nmContext); + + ContainerRuntimeContext containerRuntimeContext = builder.build(); + + runtime.prepareContainer(containerRuntimeContext); + checkVolumeCreateCommand(); + + runtime.launchContainer(containerRuntimeContext); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files + .readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 14; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); + Assert.assertEquals(" image=docker.example.com:5000/busybox:latest", + dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals(" name=container_id", dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" ro-mounts=/source/path:/destination/path", + dockerCommands.get(counter++)); + Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++)); + + // Verify volume-driver is set to expected value. + Assert.assertEquals(" volume-driver=driver-1", + dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter++)); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index 1a50c92..03d46fc 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -169,6 +169,16 @@ The following properties should be set in yarn-site.xml: "none" or "NONE" + + + yarn.docker.trusted.registry + registry.example.com:5000 + + Optional. A common spearated list of trusted Docker registry servers. + Images from trusted registry can mount external filesystems or run privileged containers. + + + ```