diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt index 300bb65c322..f0f005d53b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt @@ -135,6 +135,7 @@ add_library(container main/native/container-executor/impl/modules/common/module-configs.c main/native/container-executor/impl/modules/gpu/gpu-module.c main/native/container-executor/impl/modules/fpga/fpga-module.c + main/native/container-executor/impl/modules/devices/devices-module.c main/native/container-executor/impl/utils/docker-util.c ) @@ -169,6 +170,7 @@ add_executable(cetest main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc main/native/container-executor/test/modules/gpu/test-gpu-module.cc main/native/container-executor/test/modules/fpga/test-fpga-module.cc + main/native/container-executor/test/modules/devices/test-devices-module.cc main/native/container-executor/test/test_util.cc main/native/container-executor/test/utils/test_docker_util.cc) target_link_libraries(cetest gtest container) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 7b13e7ca848..2f2be2f4876 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -24,6 +24,7 @@ #include "modules/gpu/gpu-module.h" #include "modules/fpga/fpga-module.h" #include "modules/cgroups/cgroups-operations.h" +#include "modules/devices/devices-module.h" #include "utils/string-utils.h" #include @@ -279,6 +280,11 @@ static int validate_arguments(int argc, char **argv , int *operation) { &argv[1]); } + if (strcmp("--module-devices", argv[1]) == 0) { + return handle_devices_request(&update_cgroups_parameters, "devices", argc - 1, + &argv[1]); + } + if (strcmp("--checksetup", argv[1]) == 0) { *operation = CHECK_SETUP; return 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c index ea1d36d5532..ab1eab53248 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c @@ -132,7 +132,7 @@ int update_cgroups_parameters( goto cleanup; } - fprintf(ERRORFILE, "CGroups: Updating cgroups, path=%s, value=%s", + fprintf(ERRORFILE, "CGroups: Updating cgroups, path=%s, value=%s\n", full_path, value); // Write values to file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c new file mode 100644 index 00000000000..d0caf35274b --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c @@ -0,0 +1,209 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "configuration.h" +#include "container-executor.h" +#include "utils/string-utils.h" +#include "modules/devices/devices-module.h" +#include "modules/cgroups/cgroups-operations.h" +#include "modules/common/module-configs.h" +#include "modules/common/constants.h" +#include "util.h" + +#include +#include +#include +#include +#include + +#define EXCLUDED_DEVICES_OPTION "excluded_devices" +#define CONTAINER_ID_OPTION "container_id" +#define MAX_CONTAINER_ID_LEN 128 + +static const struct section* cfg_section; + +// Search a string in a string list +static int search_in_list(char** list, char* token) { + int i = 0; + char** iterator = list; + // search token in list + while (iterator[i] != NULL) { + if (strstr(token, iterator[i]) != NULL) { + // Found deny device in allowed list + return 1; + } + i++; + } + return 0; +} + +static int internal_handle_devices_request( + update_cgroups_parameters_function update_cgroups_parameters_func_p, + char** devices_number_tokens, + const char* container_id) { + int return_code = 0; + + char** allowed_numbers = NULL; + char* allowed_str = get_section_value(DEVICES_ALLOWED_NUMBERS, + cfg_section); + // Get denied "major:minor" device numbers from cfg, if not set, means all + // devices can be used by YARN + if (allowed_str != NULL) { + allowed_numbers = split_delimiter(allowed_str, ","); + if (NULL == allowed_numbers) { + fprintf(ERRORFILE, + "Invalid value set for %s, value=%s\n", + DEVICES_ALLOWED_NUMBERS, + allowed_str); + return_code = -1; + goto cleanup; + } + } + + char** iterator = devices_number_tokens; + int count = 0; + char* value = NULL; + int index = 0; + while (iterator[count] != NULL) { + // Replace like "c-242:0-rwm" to "c 242:0 rwm" + value = iterator[count]; + index = 0; + while (value[index] != '\0') { + if (value[index] == '-') { + value[index] = ' '; + } + index++; + } + + // Check if excluded device number is in allowed list + if (allowed_numbers != NULL) { + fprintf(LOGFILE, "Checking if in allowed list:%s\n", iterator[count]); + int found = search_in_list(allowed_numbers, iterator[count]); + if (!found) { + fprintf(ERRORFILE, + "Try to deny this but its device number is not in configured allowed list: %s\n", + iterator[count]); + return_code = -1; + goto cleanup; + } + } + + // Update device cgroups value + int rc = update_cgroups_parameters_func_p("devices", "deny", + container_id, iterator[count]); + + if (0 != rc) { + fprintf(ERRORFILE, "CGroups: Failed to update cgroups\n"); + return_code = -1; + goto cleanup; + } + count++; + } + +cleanup: + if (allowed_numbers != NULL) { + free_values(allowed_numbers); + } + + return return_code; +} + +void reload_devices_configuration() { + cfg_section = get_configuration_section(DEVICES_MODULE_SECTION_NAME, get_cfg()); +} + +/* + * Format of devices request commandline: + * The excluded_devices is comma separated device cgroups values with device type. + * The "-" will be replaced with " " to match the cgrooups parameter + * c-e --module-devices --excluded_devices b-8:16,c-244:0,c-244:1 --container_id container_x_y + */ +int handle_devices_request(update_cgroups_parameters_function func, + const char* module_name, int module_argc, char** module_argv) { + if (!cfg_section) { + reload_devices_configuration(); + } + + if (!module_enabled(cfg_section, DEVICES_MODULE_SECTION_NAME)) { + fprintf(ERRORFILE, + "Please make sure devices module is enabled before using it.\n"); + return -1; + } + + static struct option long_options[] = { + {EXCLUDED_DEVICES_OPTION, required_argument, 0, 'e' }, + {CONTAINER_ID_OPTION, required_argument, 0, 'c' }, + {0, 0, 0, 0} + }; + + int c = 0; + int option_index = 0; + + char** device_value_tokens = NULL; + char container_id[MAX_CONTAINER_ID_LEN]; + memset(container_id, 0, sizeof(container_id)); + int failed = 0; + + optind = 1; + while((c = getopt_long(module_argc, module_argv, "e:c:", + long_options, &option_index)) != -1) { + switch(c) { + case 'e': + device_value_tokens = split_delimiter(optarg, ","); + break; + case 'c': + if (!validate_container_id(optarg)) { + fprintf(ERRORFILE, + "Specified container_id=%s is invalid\n", optarg); + failed = 1; + goto cleanup; + } + strncpy(container_id, optarg, MAX_CONTAINER_ID_LEN); + break; + default: + fprintf(ERRORFILE, + "Unknown option in devices command character %d %c, optionindex = %d\n", + c, c, optind); + failed = 1; + goto cleanup; + } + } + + if (0 == container_id[0]) { + fprintf(ERRORFILE, + "[%s] --container_id must be specified.\n", __func__); + failed = 1; + goto cleanup; + } + + if (NULL == device_value_tokens) { + // Devices number is null, skip following call. + fprintf(ERRORFILE, "--excluded_devices is not specified, skip cgroups call.\n"); + goto cleanup; + } + + failed = internal_handle_devices_request(func, + device_value_tokens, + container_id); + +cleanup: + if (device_value_tokens) { + free_values(device_value_tokens); + } + return failed; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h new file mode 100644 index 00000000000..83ec8375285 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef __FreeBSD__ +#define _WITH_GETLINE +#endif + +#ifndef _MODULES_DEVICES_MUDULE_H_ +#define _MODULES_DEVICES_MUDULE_H_ + +// Allowed device list. value format is "major1:minor1,major2:minor2" +#define DEVICES_ALLOWED_NUMBERS "devices.allowed-numbers" +#define DEVICES_MODULE_SECTION_NAME "devices" + +// For unit test stubbing +typedef int (*update_cgroups_parameters_function)(const char*, const char*, + const char*, const char*); + +/** + * Handle devices requests + */ +int handle_devices_request(update_cgroups_parameters_function func, + const char* module_name, int module_argc, char** module_argv); + +/** + * Reload config from filesystem, visible for testing. + */ +void reload_devices_configuration(); + +#endif \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c index eea3e108ea6..1753954cb2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c @@ -44,6 +44,9 @@ char** split_delimiter(char *value, const char *delim) { memset(return_values, 0, sizeof(char *) * return_values_size); temp_tok = strtok_r(value, delim, &tempstr); + if (NULL == temp_tok) { + return_values[size++] = strdup(value); + } while (temp_tok != NULL) { temp_tok = strdup(temp_tok); if (NULL == temp_tok) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/devices/test-devices-module.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/devices/test-devices-module.cc new file mode 100644 index 00000000000..d6fbbd79a07 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/devices/test-devices-module.cc @@ -0,0 +1,240 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +extern "C" { +#include "configuration.h" +#include "container-executor.h" +#include "modules/cgroups/cgroups-operations.h" +#include "modules/devices/devices-module.h" +#include "test/test-container-executor-common.h" +#include "util.h" +} + +namespace ContainerExecutor { + +class TestDevicesModule : public ::testing::Test { +protected: + virtual void SetUp() { + if (mkdirs(TEST_ROOT, 0755) != 0) { + fprintf(ERRORFILE, "Failed to mkdir TEST_ROOT: %s\n", TEST_ROOT); + exit(1); + } + LOGFILE = stdout; + ERRORFILE = stderr; + } + + virtual void TearDown() { + + } +}; + +static std::vector cgroups_parameters_invoked; + +static int mock_update_cgroups_parameters( + const char* controller_name, + const char* param_name, + const char* group_id, + const char* value) { + char* buf = (char*) malloc(128); + strcpy(buf, controller_name); + cgroups_parameters_invoked.push_back(buf); + + buf = (char*) malloc(128); + strcpy(buf, param_name); + cgroups_parameters_invoked.push_back(buf); + + buf = (char*) malloc(128); + strcpy(buf, group_id); + cgroups_parameters_invoked.push_back(buf); + + buf = (char*) malloc(128); + strcpy(buf, value); + cgroups_parameters_invoked.push_back(buf); + return 0; +} + +static void clear_cgroups_parameters_invoked() { + for (std::vector::size_type i = 0; i < cgroups_parameters_invoked.size(); i++) { + free((void *) cgroups_parameters_invoked[i]); + } + cgroups_parameters_invoked.clear(); +} + +static void verify_param_updated_to_cgroups( + int argc, const char** argv) { + ASSERT_EQ(argc, cgroups_parameters_invoked.size()); + + int offset = 0; + while (offset < argc) { + ASSERT_STREQ(argv[offset], cgroups_parameters_invoked[offset]); + offset++; + } +} + +static void write_and_load_devices_module_to_cfg(const char* cfg_filepath, int enabled) { + FILE *file = fopen(cfg_filepath, "w"); + if (file == NULL) { + printf("FAIL: Could not open configuration file: %s\n", cfg_filepath); + exit(1); + } + fprintf(file, "[devices]\n"); + if (enabled) { + fprintf(file, "module.enabled=true\n"); + } else { + fprintf(file, "module.enabled=false\n"); + } + fclose(file); + + // Read config file + read_executor_config(cfg_filepath); + reload_devices_configuration(); +} + +static void test_devices_module_enabled_disabled(int enabled) { + // Write config file. + const char *filename = TEST_ROOT "/test_cgroups_module_enabled_disabled.cfg"; + write_and_load_devices_module_to_cfg(filename, enabled); + + char* argv[] = { (char*) "--module-devices", (char*) "--excluded_devices", + (char*) "c-243:0-rwm,c-243:1-rwm", + (char*) "--container_id", + (char*) "container_1498064906505_0001_01_000001" }; + + int rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 5, argv); + + int EXPECTED_RC; + if (enabled) { + EXPECTED_RC = 0; + } else { + EXPECTED_RC = -1; + } + ASSERT_EQ(EXPECTED_RC, rc); + + clear_cgroups_parameters_invoked(); + free_executor_configurations(); +} + +TEST_F(TestDevicesModule, test_verify_device_module_calls_cgroup_parameter) { + // Write config file. + const char *filename = TEST_ROOT "/test_verify_devices_module_calls_cgroup_parameter.cfg"; + write_and_load_devices_module_to_cfg(filename, 1); + + char* container_id = (char*) "container_1498064906505_0001_01_000001"; + char* argv[] = { (char*) "--module-devices", (char*) "--excluded_devices", + (char*) "c-243:0-rwm,c-243:1-rwm", + (char*) "--container_id", + container_id }; + + /* Test case 1: block 2 devices */ + clear_cgroups_parameters_invoked(); + int rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 5, argv); + ASSERT_EQ(0, rc) << "Should success.\n"; + + // Verify cgroups parameters + const char* expected_cgroups_argv[] = { "devices", "deny", container_id, "c 243:0 rwm", + "devices", "deny", container_id, "c 243:1 rwm"}; + verify_param_updated_to_cgroups(8, expected_cgroups_argv); + + /* Test case 2: block 0 devices */ + clear_cgroups_parameters_invoked(); + char* argv_1[] = { (char*) "--module-devices", (char*) "--container_id", container_id }; + rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 3, argv_1); + ASSERT_EQ(0, rc) << "Should success.\n"; + + // Verify cgroups parameters + verify_param_updated_to_cgroups(0, NULL); + + /* Test case 3: block 2 non-sequential devices */ + clear_cgroups_parameters_invoked(); + char* argv_2[] = { (char*) "--module-devices", (char*) "--excluded_devices", + (char*) "c-243:0-rwm,c-243:2-rwm", + (char*) "--container_id", container_id }; + rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 5, argv_2); + ASSERT_EQ(0, rc) << "Should success.\n"; + + // Verify cgroups parameters + const char* expected_cgroups_argv_2[] = { "devices", "deny", container_id, "c 243:0 rwm", + "devices", "deny", container_id, "c 243:2 rwm"}; + verify_param_updated_to_cgroups(8, expected_cgroups_argv_2); + + clear_cgroups_parameters_invoked(); + free_executor_configurations(); +} + +TEST_F(TestDevicesModule, test_illegal_cli_parameters) { + // Write config file. + const char *filename = TEST_ROOT "/test_illegal_cli_parameters.cfg"; + write_and_load_devices_module_to_cfg(filename, 1); + + // Illegal container id - 1 + char* argv[] = { (char*) "--module-devices", (char*) "--excluded_devices", + (char*) "c-243:0-rwm,c-243:1-rwm", + (char*) "--container_id", (char*) "xxxx" }; + int rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 5, argv); + ASSERT_NE(0, rc) << "Should fail.\n"; + + // Illegal container id - 2 + clear_cgroups_parameters_invoked(); + char* argv_1[] = { (char*) "--module-devices", (char*) "--excluded_devices", + (char*) "c-243:0-rwm,c-243:1-rwm", + (char*) "--container_id", (char*) "container_1" }; + rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 5, argv_1); + ASSERT_NE(0, rc) << "Should fail.\n"; + + // Illegal container id - 3 + clear_cgroups_parameters_invoked(); + char* argv_2[] = { (char*) "--module-devices", (char*) "--excluded_devices", + (char*) "c-243:0-rwm,c-243:1-rwm" }; + rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 3, argv_2); + ASSERT_NE(0, rc) << "Should fail.\n"; + + clear_cgroups_parameters_invoked(); + free_executor_configurations(); +} + +TEST_F(TestDevicesModule, test_devices_module_disabled) { + test_devices_module_enabled_disabled(0); +} + +TEST_F(TestDevicesModule, test_devices_module_enabled) { + test_devices_module_enabled_disabled(1); +} +} // namespace ContainerExecutor \ No newline at end of file