commit 39f5a2fcae946393c651984ba0d00ac7eef30e85 Author: Eric Yang Date: Mon Dec 4 16:43:22 2017 -0500 YARN-7590. Added prefix directory validation check for container-executor. (Contributed by Eric Yang) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 3b04f88..fb25e1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -516,11 +516,28 @@ char *get_app_directory(const char * nm_root, const char *user, /** * Get the user directory of a particular user */ -char *get_user_directory(const char *nm_root, const char *user) { +char *get_user_directory(const char *nm_root, const char *user, int uid) { + int result = check_nm_local_dir(uid, nm_root); + if (result != 0) { + fprintf(LOGFILE, "Permission mismatch for %s for uid: %d.\n", nm_root, uid); + return NULL; + } return concatenate(USER_DIR_PATTERN, "user_dir_path", 2, nm_root, user); } /** + * Check node manager local dir permission. + */ +int check_nm_local_dir(int uid, const char *nm_root) { + struct stat info; + stat(nm_root, &info); + if (uid != info.st_uid) { + return 1; + } + return 0; +} + +/** * Get the container directory for the given container_id */ char *get_container_work_directory(const char *nm_root, const char *user, @@ -1006,13 +1023,13 @@ static int copy_file(int input, const char* in_filename, /** * Function to initialize the user directories of a user. */ -int initialize_user(const char *user, char* const* local_dirs) { +int initialize_user(const char *user, const int uid, char* const* local_dirs) { char *user_dir; char* const* local_dir_ptr; int failed = 0; for(local_dir_ptr = local_dirs; *local_dir_ptr != 0; ++local_dir_ptr) { - user_dir = get_user_directory(*local_dir_ptr, user); + user_dir = get_user_directory(*local_dir_ptr, user, uid); if (user_dir == NULL) { fprintf(LOGFILE, "Couldn't get userdir directory for %s.\n", user); failed = 1; @@ -1056,7 +1073,7 @@ int create_log_dirs(const char *app_id, char * const * log_dirs) { /** * Function to prepare the application directories for the container. */ -int initialize_app(const char *user, const char *app_id, +int initialize_app(const char *user, const int uid, const char *app_id, const char* nmPrivate_credentials_file, char* const* local_dirs, char* const* log_roots, char* const* args) { @@ -1066,7 +1083,7 @@ int initialize_app(const char *user, const char *app_id, } // create the user directory on all disks - int result = initialize_user(user, local_dirs); + int result = initialize_user(user, uid, local_dirs); if (result != 0) { return result; } @@ -1228,7 +1245,7 @@ int create_script_paths(const char *work_dir, return exit_code; } -int create_local_dirs(const char * user, const char *app_id, +int create_local_dirs(const char * user, const int uid, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, char* const* local_dirs, @@ -1237,7 +1254,7 @@ int create_local_dirs(const char * user, const char *app_id, int container_file_source, int cred_file_source) { int exit_code = -1; // create the user directory on all disks - int result = initialize_user(user, local_dirs); + int result = initialize_user(user, uid, local_dirs); if (result != 0) { fprintf(ERRORFILE, "Could not create user dir"); fflush(ERRORFILE); @@ -1304,7 +1321,7 @@ int create_local_dirs(const char * user, const char *app_id, return exit_code; } -int launch_docker_container_as_user(const char * user, const char *app_id, +int launch_docker_container_as_user(const char * user, const int uid, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, const char *pid_file, char* const* local_dirs, @@ -1350,7 +1367,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id, } fprintf(LOGFILE, "Creating local dirs...\n"); - exit_code = create_local_dirs(user, app_id, container_id, + exit_code = create_local_dirs(user, uid, app_id, container_id, work_dir, script_name, cred_file, local_dirs, log_dirs, 1, script_file_dest, cred_file_dest, container_file_source, cred_file_source); @@ -1530,7 +1547,7 @@ cleanup: } -int launch_container_as_user(const char *user, const char *app_id, +int launch_container_as_user(const char *user, const int uid, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, const char* pid_file, char* const* local_dirs, @@ -1601,7 +1618,7 @@ int launch_container_as_user(const char *user, const char *app_id, #endif fprintf(LOGFILE, "Creating local dirs...\n"); - exit_code = create_local_dirs(user, app_id, container_id, + exit_code = create_local_dirs(user, uid, app_id, container_id, work_dir, script_name, cred_file, local_dirs, log_dirs, 0, script_file_dest, cred_file_dest, container_file_source, cred_file_source); @@ -2213,4 +2230,4 @@ int traffic_control_read_stats(char *command_file) { */ struct configuration* get_cfg() { return &CFG; -} \ No newline at end of file +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 6d4220f..6447c51 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -90,11 +90,11 @@ void read_executor_config(const char* file_name); void free_executor_configurations(); // initialize the application directory -int initialize_app(const char *user, const char *app_id, +int initialize_app(const char *user, const int uid, const char *app_id, const char *credentials, char* const* local_dirs, char* const* log_dirs, char* const* args); -int launch_docker_container_as_user(const char * user, const char *app_id, +int launch_docker_container_as_user(const char * user, const int uid, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, const char *pid_file, char* const* local_dirs, @@ -123,7 +123,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id, * @param resources_value values needed to apply resource enforcement * @return -1 or errorcode enum value on error (should never return on success). */ -int launch_container_as_user(const char * user, const char *app_id, +int launch_container_as_user(const char * user, const int uid, const char *app_id, const char *container_id, const char *work_dir, const char *script_name, const char *cred_file, const char *pid_file, char* const* local_dirs, @@ -173,11 +173,16 @@ int set_user(const char *user); // methods to get the directories -char *get_user_directory(const char *nm_root, const char *user); +char *get_user_directory(const char *nm_root, const char *user, const int uid); char *get_app_directory(const char * nm_root, const char *user, const char *app_id); +/** + * Check node manager local dir permission. + */ +int check_nm_local_dir(int uid, const char *nm_root); + char *get_container_work_directory(const char *nm_root, const char *user, const char *app_id, const char *container_id); @@ -199,7 +204,7 @@ int mkdirs(const char* path, mode_t perm); /** * Function to initialize the user directories of a user. */ -int initialize_user(const char *user, char* const* local_dirs); +int initialize_user(const char *user, const int uid, char* const* local_dirs); /** * Create a top level directory for the user. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 7b8f63f..66cbd65 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -220,7 +220,7 @@ static struct { const char *docker_command_file; } cmd_input; -static int validate_run_as_user_commands(int argc, char **argv, int *operation); +static int validate_run_as_user_commands(const int uid, int argc, char **argv, int *operation); /* Validates that arguments used in the invocation are valid. In case of validation failure, an 'errorcode' is returned. In case of successful validation, a zero is @@ -230,7 +230,7 @@ line parsing mechanism (e.g getopt). For the time being, we'll use this manual validation mechanism so that we don't have to change the invocation interface. */ -static int validate_arguments(int argc, char **argv , int *operation) { +static int validate_arguments(const int uid, int argc, char **argv , int *operation) { if (argc < 2) { display_usage(stdout); return INVALID_ARGUMENT_NUMBER; @@ -328,11 +328,11 @@ static int validate_arguments(int argc, char **argv , int *operation) { a 'long option' - we should fix this at some point. The validation/argument parsing here is extensive enough that it done in a separate function */ - return validate_run_as_user_commands(argc, argv, operation); + return validate_run_as_user_commands(uid, argc, argv, operation); } /* Parse/validate 'run as user' commands */ -static int validate_run_as_user_commands(int argc, char **argv, int *operation) { +static int validate_run_as_user_commands(const int uid, int argc, char **argv, int *operation) { /* We need at least the following arguments in order to proceed further : , - i.e at argc should be at least 4 */ @@ -511,11 +511,12 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) } int main(int argc, char **argv) { + int uid = getuid(); open_log_files(); assert_valid_setup(argv[0]); int operation = -1; - int ret = validate_arguments(argc, argv, &operation); + int ret = validate_arguments(uid, argc, argv, &operation); if (ret != 0) { flush_and_close_log_files(); @@ -556,6 +557,7 @@ int main(int argc, char **argv) { } exit_code = initialize_app(cmd_input.yarn_user_name, + uid, cmd_input.app_id, cmd_input.cred_file, split(cmd_input.local_dirs), @@ -578,6 +580,7 @@ int main(int argc, char **argv) { } exit_code = launch_docker_container_as_user(cmd_input.yarn_user_name, + uid, cmd_input.app_id, cmd_input.container_id, cmd_input.current_dir, @@ -606,6 +609,7 @@ int main(int argc, char **argv) { } exit_code = launch_container_as_user(cmd_input.yarn_user_name, + uid, cmd_input.app_id, cmd_input.container_id, cmd_input.current_dir, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c index 4e8db6c..9584340 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c @@ -152,8 +152,9 @@ void check_pid_file(const char* pid_file, pid_t mypid) { } void test_get_user_directory() { - char *user_dir = get_user_directory(TMPDIR, "user"); - char *expected = TMPDIR "/usercache/user"; + int uid = getuid(); + char *user_dir = get_user_directory(TEST_ROOT, "user", uid); + char *expected = TEST_ROOT "/usercache/user"; if (strcmp(user_dir, expected) != 0) { printf("test_get_user_directory expected %s got %s\n", expected, user_dir); exit(1); @@ -161,9 +162,29 @@ void test_get_user_directory() { free(user_dir); } +void test_check_nm_local_dir() { + // check filesystem is same as running user. + int expected = 0; + int uid = getuid(); + char *local_path = "target"; + char *root_path = "/"; + int actual = check_nm_local_dir(uid, local_path); + if (expected != actual) { + printf("test_nm_local_dir expected %d got %d\n", expected, actual); + exit(1); + } + // check filesystem is different from running user. + expected = 1; + actual = check_nm_local_dir(uid, root_path); + if (expected != actual && uid != 0) { + printf("test_nm_local_dir expected %d got %d\n", expected, actual); + exit(1); + } +} + void test_get_app_directory() { - char *expected = TMPDIR "/usercache/user/appcache/app_200906101234_0001"; - char *app_dir = (char *) get_app_directory(TMPDIR, "user", + char *expected = TEST_ROOT "/usercache/user/appcache/app_200906101234_0001"; + char *app_dir = (char *) get_app_directory(TEST_ROOT, "user", "app_200906101234_0001"); if (strcmp(app_dir, expected) != 0) { printf("test_get_app_directory expected %s got %s\n", expected, app_dir); @@ -173,9 +194,9 @@ void test_get_app_directory() { } void test_get_container_directory() { - char *container_dir = get_container_work_directory(TMPDIR, "owen", "app_1", + char *container_dir = get_container_work_directory(TEST_ROOT, "owen", "app_1", "container_1"); - char *expected = TMPDIR"/usercache/owen/appcache/app_1/container_1"; + char *expected = TEST_ROOT "/usercache/owen/appcache/app_1/container_1"; if (strcmp(container_dir, expected) != 0) { printf("Fail get_container_work_directory got %s expected %s\n", container_dir, expected); @@ -185,9 +206,9 @@ void test_get_container_directory() { } void test_get_container_launcher_file() { - char *expected_file = (TMPDIR"/usercache/user/appcache/app_200906101234_0001" + char *expected_file = (TEST_ROOT "/usercache/user/appcache/app_200906101234_0001" "/launch_container.sh"); - char *app_dir = get_app_directory(TMPDIR, "user", + char *app_dir = get_app_directory(TEST_ROOT, "user", "app_200906101234_0001"); char *container_file = get_container_launcher_file(app_dir); if (strcmp(container_file, expected_file) != 0) { @@ -258,7 +279,8 @@ void test_check_configuration_permissions() { } void test_delete_container() { - if (initialize_user(yarn_username, local_dirs)) { + int uid = getuid(); + if (initialize_user(yarn_username, uid, local_dirs)) { printf("FAIL: failed to initialize user %s\n", yarn_username); exit(1); } @@ -730,6 +752,7 @@ void test_signal_container_group() { } void test_init_app() { + int uid = getuid(); printf("\nTesting init app\n"); if (seteuid(0) != 0) { printf("FAIL: seteuid to root failed - %s\n", strerror(errno)); @@ -774,7 +797,7 @@ void test_init_app() { exit(1); } else if (child == 0) { char *final_pgm[] = {"touch", "my-touch-file", 0}; - if (initialize_app(yarn_username, "app_4", TEST_ROOT "/creds.txt", + if (initialize_app(yarn_username, uid, "app_4", TEST_ROOT "/creds.txt", local_dirs, log_dirs, final_pgm) != 0) { printf("FAIL: failed in child\n"); exit(42); @@ -818,6 +841,7 @@ void test_init_app() { } void test_run_container() { + int uid = getuid(); printf("\nTesting run container\n"); if (seteuid(0) != 0) { printf("FAIL: seteuid to root failed - %s\n", strerror(errno)); @@ -873,7 +897,7 @@ void test_run_container() { strerror(errno)); exit(1); } else if (child == 0) { - if (launch_container_as_user(yarn_username, "app_4", "container_1", + if (launch_container_as_user(yarn_username, uid, "app_4", "container_1", container_dir, script_name, TEST_ROOT "/creds.txt", pid_file, local_dirs, log_dirs, "cgroups", cgroups_pids) != 0) { @@ -1044,7 +1068,8 @@ static void test_delete_race_internal() { } void test_delete_race() { - if (initialize_user(yarn_username, local_dirs)) { + int uid = getuid(); + if (initialize_user(yarn_username, uid, local_dirs)) { printf("FAIL: failed to initialize user %s\n", yarn_username); exit(1); } @@ -1232,6 +1257,9 @@ int main(int argc, char **argv) { printf("\nTesting get_user_directory()\n"); test_get_user_directory(); + printf("\nTesting check_nm_local_dir()\n"); + test_check_nm_local_dir(); + printf("\nTesting get_app_directory()\n"); test_get_app_directory();