diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java index fdb58598abd..7d7833c2126 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java @@ -1811,6 +1811,14 @@ public static boolean isAclEnabled(Configuration conf) { public static final String DEFAULT_NM_DOCKER_DEFAULT_CONTAINER_NETWORK = "host"; + /** Allow host pid namespace for containers. Use with care. */ + public static final String NM_DOCKER_ALLOW_HOST_PID_NAMESPACE = + DOCKER_CONTAINER_RUNTIME_PREFIX + "host-pid-namespace.allowed"; + + /** Host pid namespace for containers is disabled by default. */ + public static final boolean DEFAULT_NM_DOCKER_ALLOW_HOST_PID_NAMESPACE = + false; + /** * Whether or not users are allowed to request that Docker containers honor * the debug deletion delay. This is useful for troubleshooting Docker diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml index 017799a8e41..2ab2eef9fb2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml @@ -1678,6 +1678,14 @@ + This configuration setting determines whether the host's PID + namespace is allowed for docker containers on this cluster. + Use with care. + yarn.nodemanager.runtime.linux.docker.host-pid-namespace.allowed + false + + + Property to enable docker user remapping yarn.nodemanager.runtime.linux.docker.enable-userremapping.allowed false diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 6799ce27251..a94afa41e76 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -192,6 +192,9 @@ public static final String ENV_DOCKER_CONTAINER_NETWORK = "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK"; @InterfaceAudience.Private + public static final String ENV_DOCKER_CONTAINER_PID_NAMESPACE = + "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_PID_NAMESPACE"; + @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_HOSTNAME = "YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_HOSTNAME"; @InterfaceAudience.Private @@ -480,6 +483,47 @@ private void validateContainerNetworkType(String network) throw new ContainerExecutionException(msg); } + /** + * Return whether the YARN container is allowed to run using the host's PID + * namespace for the Docker container. For this to be allowed, the submitting + * user must request the feature and the feature must be enabled on the + * cluster. + * + * @param container the target YARN container + * @return whether host pid namespace is requested and allowed + * @throws ContainerExecutionException if host pid namespace is requested + * but is not allowed + */ + private boolean allowHostPidNamespace(Container container) + throws ContainerExecutionException { + Map environment = container.getLaunchContext() + .getEnvironment(); + String pidNamespace = environment.get(ENV_DOCKER_CONTAINER_PID_NAMESPACE); + + if (pidNamespace == null) { + return false; + } + + if (!pidNamespace.equalsIgnoreCase("host")) { + LOG.warn("NOT requesting PID namespace. Value of " + + ENV_DOCKER_CONTAINER_PID_NAMESPACE + "is invalid: " + pidNamespace); + return false; + } + + boolean hostPidNamespaceEnabled = conf.getBoolean( + YarnConfiguration.NM_DOCKER_ALLOW_HOST_PID_NAMESPACE, + YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_HOST_PID_NAMESPACE); + + if (!hostPidNamespaceEnabled) { + String message = "Host pid namespace being requested but this is not " + + "enabled on this cluster"; + LOG.warn(message); + throw new ContainerExecutionException(message); + } + + return true; + } + public static void validateHostname(String hostname) throws ContainerExecutionException { if (hostname != null && !hostname.isEmpty()) { @@ -798,6 +842,10 @@ public void launchContainer(ContainerRuntimeContext ctx) } } + if (allowHostPidNamespace(container)) { + runCommand.setPidNamespace("host"); + } + if (allowPrivilegedContainerExecution(container)) { runCommand.setPrivileged(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index aa7d4d57144..b7e84d7c911 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -56,6 +56,11 @@ public DockerRunCommand setNetworkType(String type) { return this; } + public DockerRunCommand setPidNamespace(String type) { + super.addCommandArguments("pid", type); + return this; + } + public DockerRunCommand addMountLocation(String sourcePath, String destinationPath, boolean createSource) { boolean sourceExists = new File(sourcePath).exists(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index b5cb5512891..afd6465c1c2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -217,6 +217,10 @@ const char *get_docker_error_message(const int error_code) { return "Invalid docker volume name"; case INVALID_DOCKER_VOLUME_COMMAND: return "Invalid docker volume command"; + case PID_HOST_DISABLED: + return "Host pid namespace is disabled"; + case INVALID_PID_NAMESPACE: + return "Invalid pid namespace"; default: return "Unknown error"; } @@ -779,6 +783,51 @@ static int set_network(const struct configuration *command_config, return ret; } +static int set_pid_namespace(const struct configuration *command_config, + const struct configuration *conf, char *out, + const size_t outlen) { + size_t tmp_buffer_size = 1024; + char *tmp_buffer = (char *) alloc_and_clear_memory(tmp_buffer_size, sizeof(char)); + char *value = get_configuration_value("pid", DOCKER_COMMAND_FILE_SECTION, + command_config); + char *pid_host_enabled = get_configuration_value("docker.pid-host.enabled", + CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf); + int ret = 0; + + if (value != NULL) { + if (strcmp(value, "host") == 0) { + if (pid_host_enabled != NULL) { + if (strcmp(pid_host_enabled, "1") == 0) { + ret = add_to_buffer(out, outlen, "--pid='host' "); + if (ret != 0) { + ret = BUFFER_TOO_SMALL; + } + } else { + fprintf(ERRORFILE, "Host pid namespace is disabled\n"); + ret = PID_HOST_DISABLED; + goto free_and_exit; + } + } else { + fprintf(ERRORFILE, "Host pid namespace is disabled\n"); + ret = PID_HOST_DISABLED; + goto free_and_exit; + } + } else { + fprintf(ERRORFILE, "Invalid pid namespace\n"); + ret = INVALID_PID_NAMESPACE; + } + } + + free_and_exit: + free(tmp_buffer); + free(value); + free(pid_host_enabled); + if (ret != 0) { + memset(out, 0, outlen); + } + return ret; +} + static int set_capabilities(const struct configuration *command_config, const struct configuration *conf, char *out, const size_t outlen) { @@ -1146,6 +1195,11 @@ int get_docker_run_command(const char *command_file, const struct configuration return ret; } + ret = set_pid_namespace(&command_config, conf, out, outlen); + if (ret != 0) { + return ret; + } + ret = add_ro_mounts(&command_config, conf, out, outlen); if (ret != 0) { return ret; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h index f98800cf0e5..a14928d9f37 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h @@ -55,7 +55,9 @@ enum docker_error_codes { INVALID_DOCKER_KILL_COMMAND, INVALID_DOCKER_VOLUME_DRIVER, INVALID_DOCKER_VOLUME_NAME, - INVALID_DOCKER_VOLUME_COMMAND + INVALID_DOCKER_VOLUME_COMMAND, + PID_HOST_DISABLED, + INVALID_PID_NAMESPACE }; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc index 416bf388e4a..b58586efd12 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc @@ -454,6 +454,98 @@ namespace ContainerExecutor { ASSERT_EQ(0, strlen(buff)); } + TEST_F(TestDockerUtil, test_set_pid_namespace) { + struct configuration container_cfg, cmd_cfg; + const int buff_len = 1024; + char buff[buff_len]; + int ret = 0; + std::string container_executor_cfg_contents[] = {"[docker]\n docker.pid-host.enabled=1", + "[docker]\n docker.pid-host.enabled=0", + "[docker]\n"}; + std::vector > file_cmd_vec; + file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n docker-command=run\n pid=host", "--pid='host' ")); + file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n docker-command=run", "")); + + std::vector > bad_file_cmd_vec; + bad_file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n docker-command=run\n pid=other", + static_cast(INVALID_PID_NAMESPACE))); + + write_container_executor_cfg(container_executor_cfg_contents[0]); + ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); + + std::vector >::const_iterator itr; + std::vector >::const_iterator itr2; + if (ret != 0) { + FAIL(); + } + for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) { + memset(buff, 0, buff_len); + write_command_file(itr->first); + ret = read_config(docker_command_file.c_str(), &cmd_cfg); + if (ret != 0) { + FAIL(); + } + ret = set_pid_namespace(&cmd_cfg, &container_cfg, buff, buff_len); + ASSERT_EQ(0, ret); + ASSERT_STREQ(itr->second.c_str(), buff); + } + for (itr2 = bad_file_cmd_vec.begin(); itr2 != bad_file_cmd_vec.end(); ++itr2) { + memset(buff, 0, buff_len); + write_command_file(itr2->first); + ret = read_config(docker_command_file.c_str(), &cmd_cfg); + if (ret != 0) { + FAIL(); + } + ret = set_pid_namespace(&cmd_cfg, &container_cfg, buff, buff_len); + ASSERT_EQ(itr2->second, ret); + ASSERT_EQ(0, strlen(buff)); + } + + // check default case and when it's turned off + for (int i = 1; i < 3; ++i) { + write_container_executor_cfg(container_executor_cfg_contents[i]); + ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); + if (ret != 0) { + FAIL(); + } + file_cmd_vec.clear(); + file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n docker-command=run", "")); + for (itr = file_cmd_vec.begin(); itr != file_cmd_vec.end(); ++itr) { + memset(buff, 0, buff_len); + write_command_file(itr->first); + ret = read_config(docker_command_file.c_str(), &cmd_cfg); + if (ret != 0) { + FAIL(); + } + ret = set_pid_namespace(&cmd_cfg, &container_cfg, buff, buff_len); + ASSERT_EQ(0, ret); + ASSERT_STREQ(itr->second.c_str(), buff); + } + bad_file_cmd_vec.clear(); + bad_file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n docker-command=run\n pid=other", + static_cast(INVALID_PID_NAMESPACE))); + bad_file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n docker-command=run\n pid=host", + static_cast(PID_HOST_DISABLED))); + for (itr2 = bad_file_cmd_vec.begin(); itr2 != bad_file_cmd_vec.end(); ++itr2) { + memset(buff, 0, buff_len); + write_command_file(itr2->first); + ret = read_config(docker_command_file.c_str(), &cmd_cfg); + if (ret != 0) { + FAIL(); + } + ret = set_pid_namespace(&cmd_cfg, &container_cfg, buff, buff_len); + ASSERT_EQ(itr2->second, ret); + ASSERT_EQ(0, strlen(buff)); + } + } + } + TEST_F(TestDockerUtil, test_check_mount_permitted) { const char *permitted_mounts[] = {"/etc", "/usr/bin/cut", "/tmp/", NULL}; std::vector > test_data; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java index 208455ff5ae..58e47420bc2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java @@ -694,7 +694,7 @@ public void testContainerLaunchWithCustomNetworks() } @Test - public void testLaunchPrivilegedContainersInvalidEnvVar() + public void testLaunchPidNamespaceContainersInvalidEnvVar() throws ContainerExecutionException, PrivilegedOperationException, IOException{ DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( @@ -702,7 +702,7 @@ public void testLaunchPrivilegedContainersInvalidEnvVar() runtime.initialize(conf, null); env.put(DockerLinuxContainerRuntime - .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "invalid-value"); + .ENV_DOCKER_CONTAINER_PID_NAMESPACE, "invalid-value"); runtime.launchContainer(builder.build()); PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); @@ -718,6 +718,107 @@ public void testLaunchPrivilegedContainersInvalidEnvVar() String command = dockerCommands.get(0); //ensure --privileged isn't in the invocation + Assert.assertTrue("Unexpected --pid in docker run args : " + command, + !command.contains("--pid")); + } + + @Test + public void testLaunchPidNamespaceContainersWithDisabledSetting() + throws ContainerExecutionException { + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, null); + + env.put(DockerLinuxContainerRuntime + .ENV_DOCKER_CONTAINER_PID_NAMESPACE, "host"); + + try { + runtime.launchContainer(builder.build()); + Assert.fail("Expected a privileged launch container failure."); + } catch (ContainerExecutionException e) { + LOG.info("Caught expected exception : " + e); + } + } + + @Test + public void testLaunchPidNamespaceContainersEnabled() + throws ContainerExecutionException, PrivilegedOperationException, + IOException{ + //Enable host pid namespace containers. + conf.setBoolean(YarnConfiguration.NM_DOCKER_ALLOW_HOST_PID_NAMESPACE, + true); + + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, null); + + env.put(DockerLinuxContainerRuntime + .ENV_DOCKER_CONTAINER_PID_NAMESPACE, "host"); + + runtime.launchContainer(builder.build()); + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines( + Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 14; + int counter = 0; + Assert.assertEquals(expected, dockerCommands.size()); + Assert.assertEquals("[docker-command-execution]", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-add=SYS_CHROOT,NET_BIND_SERVICE", + dockerCommands.get(counter++)); + Assert.assertEquals(" cap-drop=ALL", dockerCommands.get(counter++)); + Assert.assertEquals(" detach=true", dockerCommands.get(counter++)); + Assert.assertEquals(" docker-command=run", dockerCommands.get(counter++)); + Assert.assertEquals(" hostname=ctr-id", dockerCommands.get(counter++)); + Assert + .assertEquals(" image=busybox:latest", dockerCommands.get(counter++)); + Assert.assertEquals( + " launch-command=bash,/test_container_work_dir/launch_container.sh", + dockerCommands.get(counter++)); + Assert.assertEquals(" name=container_id", dockerCommands.get(counter++)); + Assert.assertEquals(" net=host", dockerCommands.get(counter++)); + Assert.assertEquals(" pid=host", dockerCommands.get(counter++)); + Assert.assertEquals( + " rw-mounts=/test_container_local_dir:/test_container_local_dir," + + "/test_filecache_dir:/test_filecache_dir," + + "/test_container_work_dir:/test_container_work_dir," + + "/test_container_log_dir:/test_container_log_dir," + + "/test_user_local_dir:/test_user_local_dir", + dockerCommands.get(counter++)); + Assert.assertEquals(" user=run_as_user", dockerCommands.get(counter++)); + Assert.assertEquals(" workdir=/test_container_work_dir", + dockerCommands.get(counter++)); + } + + @Test + public void testLaunchPrivilegedContainersInvalidEnvVar() + throws ContainerExecutionException, PrivilegedOperationException, + IOException{ + DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime( + mockExecutor, mockCGroupsHandler); + runtime.initialize(conf, null); + + env.put(DockerLinuxContainerRuntime + .ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER, "invalid-value"); + runtime.launchContainer(builder.build()); + + PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs(); + List args = op.getArguments(); + String dockerCommandFile = args.get(11); + + List dockerCommands = Files.readAllLines( + Paths.get(dockerCommandFile), Charset.forName("UTF-8")); + + int expected = 13; + Assert.assertEquals(expected, dockerCommands.size()); + + String command = dockerCommands.get(0); + + //ensure --privileged isn't in the invocation Assert.assertTrue("Unexpected --privileged in docker run args : " + command, !command.contains("--privileged")); }