commit 354d99f1abb32858eca4a243e3bfca8100a8ec7f Author: Eric Yang Date: Fri Jan 5 19:20:58 2018 -0500 YARN-7516. Implement security check for untrusted docker image. (Contributed by Eric Yang) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index a0138d1..052554d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -20,6 +20,7 @@ #include #include #include +#include #include "../modules/common/module-configs.h" #include "docker-util.h" #include "string-utils.h" @@ -480,6 +481,30 @@ int get_docker_inspect_command(const char *command_file, const struct configurat return BUFFER_TOO_SMALL; } +int check_trusted_image(const struct configuration *command_config, const struct configuration *conf) { + bool found = false; + int i = 0; + int ret = 0; + char *image_name = get_configuration_value("image", DOCKER_COMMAND_FILE_SECTION, command_config); + char *value = get_configuration_value("privileged", DOCKER_COMMAND_FILE_SECTION, command_config); + char **trusted_repo = get_configuration_values_delimiter("docker.trusted.registries", CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf, ","); + if (value != NULL && strcmp(value, "true") == 0) { + if (trusted_repo != NULL) { + for (i = 0; trusted_repo[i] != NULL; i++) { + if (strncmp(image_name, trusted_repo[i], strlen(trusted_repo[i]))==0) { + fprintf(ERRORFILE, "image: %s is trusted in %s.\n", image_name, trusted_repo[i]); + found=true; + } + } + } + if (!found) { + fprintf(ERRORFILE, "image: %s is not trusted.\n", image_name); + ret = INVALID_DOCKER_IMAGE_NAME; + } + } + return ret; +} + int get_docker_load_command(const char *command_file, const struct configuration *conf, char *out, const size_t outlen) { int ret = 0; char *image_name = NULL; @@ -727,6 +752,12 @@ static int set_capabilities(const struct configuration *command_config, if (ret != 0) { return BUFFER_TOO_SMALL; } + + // Disable set capabilities if image is not trusted. + if (check_trusted_image(command_config, conf) != 0) { + return ret; + } + ret = add_param_to_command_if_allowed(command_config, conf, "cap-add", "docker.allowed.capabilities", "--cap-add=", 1, 0, @@ -743,6 +774,12 @@ static int set_capabilities(const struct configuration *command_config, static int set_devices(const struct configuration *command_config, const struct configuration *conf, char *out, const size_t outlen) { int ret = 0; + + // Disable set devices if image is not trusted. + if (check_trusted_image(command_config, conf) != 0) { + return ret; + } + ret = add_param_to_command_if_allowed(command_config, conf, "devices", "docker.allowed.devices", "--device=", 1, ':', out, outlen); if (ret != 0) { @@ -885,6 +922,11 @@ static int add_mounts(const struct configuration *command_config, const struct c ro_suffix = ":ro"; } + // Disable mount volumes if image is not trusted. + if (check_trusted_image(command_config, conf) != 0) { + return ret; + } + if (values != NULL) { ret = normalize_mounts(permitted_ro_mounts); ret |= normalize_mounts(permitted_rw_mounts); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc index 0c1c4bf..a91d1a0 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc @@ -859,7 +859,7 @@ namespace ContainerExecutor { std::string container_executor_contents = "[docker]\n docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n" " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n " " docker.privileged-containers.enabled=1\n docker.allowed.capabilities=CHOWN,SETUID\n" - " docker.allowed.devices=/dev/test"; + " docker.allowed.devices=/dev/test\n docker.trusted.registries=hadoop\n"; write_file(container_executor_cfg_file, container_executor_contents); int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg); if (ret != 0) { @@ -873,53 +873,53 @@ namespace ContainerExecutor { std::vector > file_cmd_vec; file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test", - "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' ")); + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test", + "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'hadoop/docker-image' ")); file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n" + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n" " launch-command=bash,test_script.sh,arg1,arg2", - "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); + "run --name='container_e1_12312_11111_02_000001' --user='test' --cap-drop='ALL' 'hadoop/docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN'" - " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash' " + " --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' " "'test_script.sh' 'arg1' 'arg2' ")); file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --cap-drop='ALL' --cap-add='CHOWN' " - "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' 'bash'" + "--cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash'" " 'test_script.sh' 'arg1' 'arg2' ")); file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" " launch-command=bash,test_script.sh,arg1,arg2", "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' " - "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'docker-image' " + "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --device='/dev/test:/dev/test' 'hadoop/docker-image' " "'bash' 'test_script.sh' 'arg1' 'arg2' ")); file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n" @@ -927,25 +927,36 @@ namespace ContainerExecutor { "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' -v '/var/log:/var/log:ro' -v '/var/lib:/lib:ro'" " -v '/usr/bin/cut:/usr/bin/cut:ro' -v '/tmp:/tmp' --cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' " "--cap-add='CHOWN' --cap-add='SETUID' --hostname='host-id' --group-add '1000' --group-add '1001' " - "--device='/dev/test:/dev/test' 'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); + "--device='/dev/test:/dev/test' 'hadoop/docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); + file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" + " network=bridge\n devices=/dev/test:/dev/test\n net=bridge\n privileged=true\n" + " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n group-add=1000,1001\n" + " launch-command=bash,test_script.sh,arg1,arg2", + "run --name='container_e1_12312_11111_02_000001' --user='test' -d --rm --net='bridge' " + "--cgroup-parent='ctr-cgroup' --privileged --cap-drop='ALL' " + "--hostname='host-id' --group-add '1000' --group-add '1001' " + "'docker-image' 'bash' 'test_script.sh' 'arg1' 'arg2' ")); std::vector > bad_file_cmd_vec; bad_file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n image=docker-image\n user=test", + "[docker-command-execution]\n docker-command=run\n image=hadoop/docker-image\n user=test", static_cast(INVALID_DOCKER_CONTAINER_NAME))); bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n user=test\n", static_cast(INVALID_DOCKER_IMAGE_NAME))); bad_file_cmd_vec.push_back(std::make_pair( - "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n", + "[docker-command-execution]\n docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n", static_cast(INVALID_DOCKER_USER_NAME))); // invalid rw mount bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/var/log:/var/log\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -955,7 +966,7 @@ namespace ContainerExecutor { // invalid ro mount bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/bin:/bin,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -965,7 +976,7 @@ namespace ContainerExecutor { // invalid capability bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n" " cap-add=CHOWN,SETUID,SETGID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -975,7 +986,7 @@ namespace ContainerExecutor { // invalid device bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/dev1:/dev/dev1\n privileged=true\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" @@ -985,7 +996,7 @@ namespace ContainerExecutor { // invalid network bad_file_cmd_vec.push_back(std::make_pair( "[docker-command-execution]\n" - " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=docker-image\n user=test\n hostname=host-id\n" + " docker-command=run\n name=container_e1_12312_11111_02_000001\n image=hadoop/docker-image\n user=test\n hostname=host-id\n" " ro-mounts=/var/log:/var/log,/var/lib:/lib,/usr/bin/cut:/usr/bin/cut\n rw-mounts=/tmp:/tmp\n" " network=bridge\n devices=/dev/test:/dev/test\n privileged=true\n net=host\n" " cap-add=CHOWN,SETUID\n cgroup-parent=ctr-cgroup\n detach=true\n rm=true\n" diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md index 1a50c92..b466f38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/DockerContainers.md @@ -197,6 +197,7 @@ are allowed. It contains the following properties: | `docker.allowed.ro-mounts` | Comma separated directories that containers are allowed to mount in read-only mode. By default, no directories are allowed to mounted. | | `docker.allowed.rw-mounts` | Comma separated directories that containers are allowed to mount in read-write mode. By default, no directories are allowed to mounted. | | `docker.privileged-containers.enabled` | Set to 1 or 0 to enable or disable launching privileged containers. Default value is 0. | +| `docker.trusted.registries` | Comma separated list of trusted docker registries for running trusted privileged docker containers. By default, no registries are defined. | Please note that if you wish to run Docker containers that require access to the YARN local directories, you must add them to the docker.allowed.rw-mounts list. @@ -350,6 +351,17 @@ the environment variable would be set to "/sys/fs/cgroup:/sys/fs/cgroup:ro". The destination path is not restricted, "/sys/fs/cgroup:/cgroup:ro" would also be valid given the example admin whitelist. +Privileged Container Security Consideration +------------------------------------------- + +Privileged docker container can interact with host system devices. This can cause harm to host operating system without proper care. In order to mitigate risk of allowing privileged container to run on Hadoop cluster, we implemented a controlled process to sandbox unauthorized privileged docker images. + +The default behavior is disallow any privileged docker containers. When `docker.privileged-containers.enabled` is set to enabled, docker image can run with root privileges in the docker container, but access to host level devices are disabled. This allows developer and tester to run docker images from internet without causing harm to host operating system. + +When docker images have been certified by developers and testers to be trustworthy. The trusted image can be promoted to trusted docker registry. System administrator can define `docker.trusted.registries`, and setup private docker registry server to promote trusted images. + +Trusted images are allowed to mount external devices such as HDFS via NFS gateway, or host level Hadoop configuration. If system administrators allow writing to external volumes using `docker.allow.rw-mounts directive`, privileged docker container can have full control of host level files in the predefined volumes. + Connecting to a Secure Docker Repository ----------------------------------------