commit f59454971d724155b4588c79f9c51b712d129762 Author: Eric Yang Date: Fri Jan 5 19:20:58 2018 -0500 YARN-7516. Implement security check for untrusted docker image. (Contributed by Eric Yang) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index a0138d1..b482f2d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -73,6 +73,48 @@ static int add_param_to_command(const struct configuration *command_config, cons return ret; } +int check_trusted_image(const struct configuration *command_config, const struct configuration *conf) { + int found = 0; + int i = 0; + int ret = 0; + char *image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, command_config); + char **privileged_registry = get_configuration_values_delimiter("docker.privileged-containers.registries", CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ","); + char *registry_ptr = NULL; + if (image_name == NULL) { + ret = INVALID_DOCKER_IMAGE_NAME; + goto free_and_exit; + } + if (privileged_registry != NULL) { + for (i = 0; privileged_registry[i] != NULL; i++) { + int len = strlen(privileged_registry[i]); + if (privileged_registry[i][len - 1] != '/') { + registry_ptr = (char *) alloc_and_clear_memory(len + 2, sizeof(char)); + strncpy(registry_ptr, privileged_registry[i], len); + registry_ptr[len] = '/'; + registry_ptr[len + 1] = '\0'; + } else { + registry_ptr = strdup(privileged_registry[i]); + } + if (strncmp(image_name, registry_ptr, strlen(registry_ptr))==0) { + fprintf(ERRORFILE, "image: %s is trusted in %s registry.\n", image_name, privileged_registry[i]); + found=1; + free(registry_ptr); + break; + } + free(registry_ptr); + } + } + if (found==0) { + fprintf(ERRORFILE, "image: %s is not trusted.\n", image_name); + ret = INVALID_DOCKER_IMAGE_TRUST; + } + free(image_name); + + free_and_exit: + free(privileged_registry); + return ret; +} + static int add_param_to_command_if_allowed(const struct configuration *command_config, const struct configuration *executor_cfg, const char *key, const char *allowed_key, const char *param, @@ -99,6 +141,14 @@ static int add_param_to_command_if_allowed(const struct configuration *command_c } if (values != NULL) { + // Disable capabilities, devices if image is not trusted. + if (strcmp(key, "net") != 0) { + if (check_trusted_image(command_config, executor_cfg) != 0) { + fprintf(ERRORFILE, "Disable %s for untrusted image\n", key); + return INVALID_DOCKER_IMAGE_TRUST; + } + } + if (permitted_values != NULL) { for (i = 0; values[i] != NULL; ++i) { memset(tmp_buffer, 0, tmp_buffer_size); @@ -217,6 +267,8 @@ const char *get_docker_error_message(const int error_code) { return "Invalid docker volume name"; case INVALID_DOCKER_VOLUME_COMMAND: return "Invalid docker volume command"; + case INVALID_DOCKER_IMAGE_TRUST: + return "Docker image is not trusted"; default: return "Unknown error"; } @@ -727,6 +779,7 @@ static int set_capabilities(const struct configuration *command_config, if (ret != 0) { return BUFFER_TOO_SMALL; } + ret = add_param_to_command_if_allowed(command_config, conf, "cap-add", "docker.allowed.capabilities", "--cap-add=", 1, 0, @@ -886,6 +939,18 @@ static int add_mounts(const struct configuration *command_config, const struct c } if (values != NULL) { + // Disable mount volumes if image is not trusted. + if (check_trusted_image(command_config, conf) != 0) { + fprintf(ERRORFILE, "Disable mount volume for untrusted image\n"); + memset(out, 0, outlen); + if (ro == 0) { + ret = INVALID_DOCKER_RW_MOUNT; + } else { + ret = INVALID_DOCKER_MOUNT; + } + goto free_and_exit; + } + ret = normalize_mounts(permitted_ro_mounts); ret |= normalize_mounts(permitted_rw_mounts); if (ret != 0) { @@ -986,6 +1051,12 @@ static int set_privileged(const struct configuration *command_config, const stru if (value != NULL && strcmp(value, "true") == 0) { if (privileged_container_enabled != NULL) { if (strcmp(privileged_container_enabled, "1") == 0) { + // Disable set privileged if image is not trusted. + if (check_trusted_image(command_config, conf) != 0) { + fprintf(ERRORFILE, "Privileged containers are disabled from untrusted source\n"); + ret = PRIVILEGED_CONTAINERS_DISABLED; + goto free_and_exit; + } ret = add_to_buffer(out, outlen, "--privileged "); if (ret != 0) { ret = BUFFER_TOO_SMALL; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h index 9c42abe..88371f0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h @@ -53,7 +53,8 @@ enum docker_error_codes { INVALID_DOCKER_STOP_COMMAND, INVALID_DOCKER_VOLUME_DRIVER, INVALID_DOCKER_VOLUME_NAME, - INVALID_DOCKER_VOLUME_COMMAND + INVALID_DOCKER_VOLUME_COMMAND, + INVALID_DOCKER_IMAGE_TRUST }; /** diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc index 0c1c4bf..4cca21f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc @@ -468,12 +468,12 @@ namespace ContainerExecutor { const int buff_len = 1024; char buff[buff_len]; int ret = 0; - std::string container_executor_cfg_contents[] = {"[docker]\n docker.privileged-containers.enabled=1", + std::string container_executor_cfg_contents[] = {"[docker]\n docker.privileged-containers.enabled=1\n docker.privileged-containers.registries=hadoop", "[docker]\n docker.privileged-containers.enabled=0", "[docker]\n"}; std::vector > file_cmd_vec; file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n privileged=true", "--privileged ")); + "[docker-command-execution]\n docker-command=run\n privileged=true\n image=hadoop/image", "--privileged ")); file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n docker-command=run\n privileged=false", "")); file_cmd_vec.push_back(std::make_pair( @@ -536,15 +536,19 @@ namespace ContainerExecutor { const int buff_len = 1024; char buff[buff_len]; int ret = 0; - std::string container_executor_cfg_contents = "[docker]\n docker.allowed.capabilities=CHROOT,MKNOD"; + std::string container_executor_cfg_contents = "[docker]\n" + " docker.allowed.capabilities=CHROOT,MKNOD\n" + " docker.privileged-containers.registries=hadoop\n"; std::vector > file_cmd_vec; file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n cap-add=CHROOT,MKNOD", + "[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n cap-add=CHROOT,MKNOD", "--cap-drop='ALL' --cap-add='CHROOT' --cap-add='MKNOD' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n cap-add=CHROOT", "--cap-drop='ALL' --cap-add='CHROOT' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n cap-add=CHROOT", + "--cap-drop='ALL' --cap-add='CHROOT' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run", "--cap-drop='ALL' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n", + "--cap-drop='ALL' ")); write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); @@ -563,7 +567,7 @@ namespace ContainerExecutor { ASSERT_EQ(0, ret); ASSERT_STREQ(itr->second.c_str(), buff); } - write_command_file("[docker-command-execution]\n docker-command=run\n cap-add=SETGID"); + write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n cap-add=SETGID"); ret = read_config(docker_command_file.c_str(), &cmd_cfg); if (ret != 0) { FAIL(); @@ -573,7 +577,7 @@ namespace ContainerExecutor { ASSERT_EQ(INVALID_DOCKER_CAPABILITY, ret); ASSERT_EQ(0, strlen(buff)); - container_executor_cfg_contents = "[docker]\n"; + container_executor_cfg_contents = "[docker]\n docker.privileged-containers.registries=hadoop\n"; write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); if (ret != 0) { @@ -590,20 +594,22 @@ namespace ContainerExecutor { const int buff_len = 1024; char buff[buff_len]; int ret = 0; - std::string container_executor_cfg_contents = "[docker]\n docker.allowed.devices=/dev/test-device,/dev/device2"; + std::string container_executor_cfg_contents = "[docker]\n" + " docker.privileged-containers.registries=hadoop\n" + " docker.allowed.devices=/dev/test-device,/dev/device2"; std::vector > file_cmd_vec; file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n devices=/dev/test-device:/dev/test-device", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/test-device:/dev/test-device", "--device='/dev/test-device:/dev/test-device' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n devices=/dev/device2:/dev/device2", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device2:/dev/device2", "--device='/dev/device2:/dev/device2' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n " - "devices=/dev/test-device:/dev/test-device,/dev/device2:/dev/device2", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n" + " devices=/dev/test-device:/dev/test-device,/dev/device2:/dev/device2", "--device='/dev/test-device:/dev/test-device' --device='/dev/device2:/dev/device2' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n", "")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image", "")); write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); @@ -622,7 +628,7 @@ namespace ContainerExecutor { ASSERT_EQ(0, ret); ASSERT_STREQ(itr->second.c_str(), buff); } - write_command_file("[docker-command-execution]\n docker-command=run\n devices=/dev/device3:/dev/device3"); + write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device3:/dev/device3"); ret = read_config(docker_command_file.c_str(), &cmd_cfg); if (ret != 0) { FAIL(); @@ -632,7 +638,7 @@ namespace ContainerExecutor { ASSERT_EQ(INVALID_DOCKER_DEVICE, ret); ASSERT_EQ(0, strlen(buff)); - write_command_file("[docker-command-execution]\n docker-command=run\n devices=/dev/device1"); + write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n devices=/dev/device1"); ret = read_config(docker_command_file.c_str(), &cmd_cfg); if (ret != 0) { FAIL(); @@ -660,21 +666,22 @@ namespace ContainerExecutor { const int buff_len = 1024; char buff[buff_len]; int ret = 0; - std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/opt,/var,/usr/bin/cut,..\n " + std::string container_executor_cfg_contents = "[docker]\n docker.privileged-containers.registries=hadoop\n " + "docker.allowed.rw-mounts=/opt,/var,/usr/bin/cut,..\n " "docker.allowed.ro-mounts=/etc/passwd"; std::vector > file_cmd_vec; file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n rw-mounts=/var:/var", "-v '/var:/var' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/var:/var", "-v '/var:/var' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n rw-mounts=/var/:/var/", "-v '/var/:/var/' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/var/:/var/", "-v '/var/:/var/' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n rw-mounts=/usr/bin/cut:/usr/bin/cut", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/usr/bin/cut:/usr/bin/cut", "-v '/usr/bin/cut:/usr/bin/cut' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n rw-mounts=/opt:/mydisk1,/var/log/:/mydisk2", "-v '/opt:/mydisk1' -v '/var/log/:/mydisk2' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n", "")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n", "")); write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); if (ret != 0) { @@ -710,7 +717,7 @@ namespace ContainerExecutor { static_cast(INVALID_DOCKER_RW_MOUNT))); bad_file_cmds_vec.push_back(std::make_pair( "[docker-command-execution]\n docker-command=run\n rw-mounts=/blah:/blah", - static_cast(INVALID_DOCKER_MOUNT))); + static_cast(INVALID_DOCKER_RW_MOUNT))); std::vector >::const_iterator itr2; @@ -767,31 +774,32 @@ namespace ContainerExecutor { char buff[buff_len]; int ret = 0; - std::string container_executor_cfg_contents = "[docker]\n docker.allowed.rw-mounts=/home/,/var,/usr/bin/cut,..\n " + std::string container_executor_cfg_contents = "[docker]\n docker.privileged-containers.registries=hadoop\n " + "docker.allowed.rw-mounts=/home/,/var,/usr/bin/cut,..\n " "docker.allowed.ro-mounts=/etc/passwd,/etc/group"; std::vector > file_cmd_vec; file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/var:/var", "-v '/var:/var:ro' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/var:/var", "-v '/var:/var:ro' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/var/:/var/", "-v '/var/:/var/:ro' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/var/:/var/", "-v '/var/:/var/:ro' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home", "-v '/home:/home:ro' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home:/home", "-v '/home:/home:ro' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/home/:/home", "-v '/home/:/home:ro' ")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home/:/home", "-v '/home/:/home:ro' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/usr/bin/cut:/usr/bin/cut", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/usr/bin/cut:/usr/bin/cut", "-v '/usr/bin/cut:/usr/bin/cut:ro' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/etc/group:/etc/group", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc/group:/etc/group", "-v '/etc/group:/etc/group:ro' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/etc/passwd:/etc/passwd", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc/passwd:/etc/passwd", "-v '/etc/passwd:/etc/passwd:ro' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/var/log:/mydisk1,/etc/passwd:/etc/passwd", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/var/log:/mydisk1,/etc/passwd:/etc/passwd", "-v '/var/log:/mydisk1:ro' -v '/etc/passwd:/etc/passwd:ro' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n", "")); + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n", "")); write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); if (ret != 0) { @@ -820,10 +828,10 @@ namespace ContainerExecutor { std::vector > bad_file_cmds_vec; bad_file_cmds_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/etc:/etc", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/etc:/etc", static_cast(INVALID_DOCKER_RO_MOUNT))); bad_file_cmds_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n ro-mounts=/blah:/blah", + "[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/blah:/blah", static_cast(INVALID_DOCKER_MOUNT))); std::vector >::const_iterator itr2; @@ -841,13 +849,13 @@ namespace ContainerExecutor { ASSERT_STREQ("", buff); } - container_executor_cfg_contents = "[docker]\n"; + container_executor_cfg_contents = "[docker]\n docker.privileged-containers.registries=hadoop\n"; write_container_executor_cfg(container_executor_cfg_contents); ret = read_config(container_executor_cfg_file.c_str(), &container_cfg); if (ret != 0) { FAIL(); } - write_command_file("[docker-command-execution]\n docker-command=run\n ro-mounts=/home:/home"); + write_command_file("[docker-command-execution]\n docker-command=run\n image=hadoop/image\n ro-mounts=/home:/home"); strcpy(buff, "test string"); ret = add_ro_mounts(&cmd_cfg, &container_cfg, buff, buff_len); ASSERT_EQ(INVALID_DOCKER_RO_MOUNT, ret); @@ -859,7 +867,7 @@ namespace ContainerExecutor { std::string container_executor_contents = "[docker]\n docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n" " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n " " docker.privileged-containers.enabled=1\n docker.allowed.capabilities=CHOWN,SETUID\n" - " docker.allowed.devices=/dev/test"; + " docker.allowed.devices=/dev/test\n docker.privileged-containers.registries=hadoop\n"; write_file(container_executor_cfg_file, container_executor_contents); int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg); if (ret != 0) { @@ -873,53 +881,53 @@ namespace ContainerExecutor { std::vector > file_cmd_vec; file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test", - "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' ")); + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test", + "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'hadoop/docker-image' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n" + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n" " launch-command=bash,test_script.sh,arg1,arg2", - "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); + "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'hadoop/docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'" - " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' " + " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' " "'test_script.sh' 'arg1' 'arg2' ")); file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' " - "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'" + "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash'" " 'test_script.sh' 'arg1' 'arg2' ")); file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' " - "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' " + "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' " "'bash' 'test_script.sh' 'arg1' 'arg2' ")); file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n" @@ -927,25 +935,34 @@ namespace ContainerExecutor { "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' " "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --group-add '1000' --group-add '1001' " - "--device='/dev/test:/dev/test' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); + "--device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); + file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " network=bridge\n net=bridge\n" + " detach=true\n rm=true\n group-add=1000,1001\n" + " launch-command=bash,test_script.sh,arg1,arg2", + "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' --cap-drop='ALL' " + "--hostname='host-id' --group-add '1000' --group-add '1001' " + "'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); std::vector > bad_file_cmd_vec; bad_file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n image=docker-image\n user=test", + "[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n user=test", static_cast(INVALID_DOCKER_CONTAINER_NAME))); bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n user=test\n", static_cast(INVALID_DOCKER_IMAGE_NAME))); bad_file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n", + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n", static_cast(INVALID_DOCKER_USER_NAME))); // invalid rw mount bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/var/log:/var/log\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -955,7 +972,7 @@ namespace ContainerExecutor { // invalid ro mount bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/bin:/bin,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -965,7 +982,7 @@ namespace ContainerExecutor { // invalid capability bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID,SETGID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -975,7 +992,7 @@ namespace ContainerExecutor { // invalid device bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/dev1:/dev/dev1\n privileged=true\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -985,7 +1002,7 @@ namespace ContainerExecutor { // invalid network bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n privileged=true\n net=host\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -998,11 +1015,13 @@ namespace ContainerExecutor { TEST_F(TestDockerUtil, test_docker_run_no_privileged) { std::string container_executor_contents[] = {"[docker]\n docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n" - " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n " + " docker.privileged-containers.registries=hadoop\n" + " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n" " docker.allowed.capabilities=CHOWN,SETUID\n" " docker.allowed.devices=/dev/test", "[docker]\n docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n" - " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n " + " docker.privileged-containers.registries=hadoop\n" + " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n" " docker.allowed.capabilities=CHOWN,SETUID\n" " privileged=0\n" " docker.allowed.devices=/dev/test"}; @@ -1029,32 +1048,32 @@ namespace ContainerExecutor { file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'" - " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' " + " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' " "'test_script.sh' 'arg1' 'arg2' ")); file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' " - "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'" + "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash'" " 'test_script.sh' 'arg1' 'arg2' ")); std::vector > bad_file_cmd_vec; bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index 1a50c92..1c3152a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -197,6 +197,7 @@ are allowed. It contains the following properties: | `docker.allowed.ro-mounts` | Comma separated directories that containers are allowed to mount in read-only mode. By default, no directories are allowed to mounted. | | `docker.allowed.rw-mounts` | Comma separated directories that containers are allowed to mount in read-write mode. By default, no directories are allowed to mounted. | | `docker.privileged-containers.enabled` | Set to 1 or 0 to enable or disable launching privileged containers. Default value is 0. | +| `docker.privileged-containers.registries` | Comma separated list of trusted docker registries for running trusted privileged docker containers. By default, no registries are defined. | Please note that if you wish to run Docker containers that require access to the YARN local directories, you must add them to the docker.allowed.rw-mounts list. @@ -350,6 +351,17 @@ the environment variable would be set to "/sys/fs/cgroup:/sys/fs/cgroup:ro". The destination path is not restricted, "/sys/fs/cgroup:/cgroup:ro" would also be valid given the example admin whitelist. +Privileged Container Security Consideration +------------------------------------------- + +Privileged docker container can interact with host system devices. This can cause harm to host operating system without proper care. In order to mitigate risk of allowing privileged container to run on Hadoop cluster, we implemented a controlled process to sandbox unauthorized privileged docker images. + +The default behavior is disallow any privileged docker containers. When `docker.privileged-containers.enabled` is set to enabled, docker image can run with root privileges in the docker container, but access to host level devices are disabled. This allows developer and tester to run docker images from internet without causing harm to host operating system. + +When docker images have been certified by developers and testers to be trustworthy. The trusted image can be promoted to trusted docker registry. System administrator can define `docker.privileged-containers.registries`, and setup private docker registry server to promote trusted images. + +Trusted images are allowed to mount external devices such as HDFS via NFS gateway, or host level Hadoop configuration. If system administrators allow writing to external volumes using `docker.allow.rw-mounts directive`, privileged docker container can have full control of host level files in the predefined volumes. + Connecting to a Secure Docker Repository ----------------------------------------