diff --git a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg index d19874f2421..aa5b7412388 100644 --- a/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg +++ b/hadoop-yarn-project/hadoop-yarn/conf/container-executor.cfg @@ -21,4 +21,10 @@ feature.tc.enabled=false #[fpga] # module.enabled=## Enable/Disable the FPGA resource handler module. set to "true" to enable, disabled by default # fpga.major-device-number=## Major device number of FPGA, by default is 246. Strongly recommend setting this -# fpga.allowed-device-minor-numbers=## Comma separated allowed minor device numbers, empty means all FPGA devices managed by YARN. \ No newline at end of file +# fpga.allowed-device-minor-numbers=## Comma separated allowed minor device numbers, empty means all FPGA devices managed by YARN. + +# The configs below deal with settings for resource handled by pluggable device plugin framework +#[devices] +# module.enabled=## Enable/Disable the device resource handler module for isolation. Disabled by default. +# #for instance, "8:16,8:32" +# devices.denied-numbers=## Blacklisted devices not permitted to use. The format is comma separated "majorNumber:minorNumber". Leave it empty means default devices reported by device plugin are all allowed. \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt index 300bb65c322..f0f005d53b5 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt @@ -135,6 +135,7 @@ add_library(container main/native/container-executor/impl/modules/common/module-configs.c main/native/container-executor/impl/modules/gpu/gpu-module.c main/native/container-executor/impl/modules/fpga/fpga-module.c + main/native/container-executor/impl/modules/devices/devices-module.c main/native/container-executor/impl/utils/docker-util.c ) @@ -169,6 +170,7 @@ add_executable(cetest main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc main/native/container-executor/test/modules/gpu/test-gpu-module.cc main/native/container-executor/test/modules/fpga/test-fpga-module.cc + main/native/container-executor/test/modules/devices/test-devices-module.cc main/native/container-executor/test/test_util.cc main/native/container-executor/test/utils/test_docker_util.cc) target_link_libraries(cetest gtest container) diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 7b13e7ca848..2f2be2f4876 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -24,6 +24,7 @@ #include "modules/gpu/gpu-module.h" #include "modules/fpga/fpga-module.h" #include "modules/cgroups/cgroups-operations.h" +#include "modules/devices/devices-module.h" #include "utils/string-utils.h" #include @@ -279,6 +280,11 @@ static int validate_arguments(int argc, char **argv , int *operation) { &argv[1]); } + if (strcmp("--module-devices", argv[1]) == 0) { + return handle_devices_request(&update_cgroups_parameters, "devices", argc - 1, + &argv[1]); + } + if (strcmp("--checksetup", argv[1]) == 0) { *operation = CHECK_SETUP; return 0; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c index ea1d36d5532..ab1eab53248 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c @@ -132,7 +132,7 @@ int update_cgroups_parameters( goto cleanup; } - fprintf(ERRORFILE, "CGroups: Updating cgroups, path=%s, value=%s", + fprintf(ERRORFILE, "CGroups: Updating cgroups, path=%s, value=%s\n", full_path, value); // Write values to file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c new file mode 100644 index 00000000000..18c5d685783 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.c @@ -0,0 +1,275 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "configuration.h" +#include "container-executor.h" +#include "utils/string-utils.h" +#include "modules/devices/devices-module.h" +#include "modules/cgroups/cgroups-operations.h" +#include "modules/common/module-configs.h" +#include "modules/common/constants.h" +#include "util.h" + +#include +#include +#include +#include +#include +#include + +#define EXCLUDED_DEVICES_OPTION "excluded_devices" +#define ALLOWED_DEVICES_OPTION "allowed_devices" +#define CONTAINER_ID_OPTION "container_id" +#define MAX_CONTAINER_ID_LEN 128 + +static const struct section* cfg_section; + +// Search a string in a string list, return 1 when found +static int search_in_list(char** list, char* token) { + int i = 0; + char** iterator = list; + // search token in list + while (iterator[i] != NULL) { + if (strstr(token, iterator[i]) != NULL || + strstr(iterator[i], token) != NULL) { + // Found deny device in allowed list + return 1; + } + i++; + } + return 0; +} + +static int is_block_device(const char* value) { + int is_block = 0; + char* block_path = malloc(512); + if (snprintf(block_path, 512, "/sys/dev/block/%s", + value) < 0) { + fprintf(ERRORFILE, "Failed to construct system block device path.\n"); + goto cleanup; + } + struct stat sb; + // file exists, is block device + if (stat(block_path, &sb) == 0) { + is_block = 1; + } +cleanup: + if (block_path) { + free(block_path); + } + return is_block; +} + +static int internal_handle_devices_request( + update_cgroups_parameters_function update_cgroups_parameters_func_p, + char** deny_devices_number_tokens, + char** allow_devices_number_tokens, + const char* container_id) { + int return_code = 0; + + char** ce_denied_numbers = NULL; + char* ce_denied_str = get_section_value(DEVICES_DENIED_NUMBERS, + cfg_section); + // Get denied "major:minor" device numbers from cfg, if not set, means all + // devices can be used by YARN. + if (ce_denied_str != NULL) { + ce_denied_numbers = split_delimiter(ce_denied_str, ","); + if (NULL == ce_denied_numbers) { + fprintf(ERRORFILE, + "Invalid value set for %s, value=%s\n", + DEVICES_DENIED_NUMBERS, + ce_denied_str); + return_code = -1; + goto cleanup; + } + // Check allowed devices passed in + char** allow_iterator = allow_devices_number_tokens; + int allow_count = 0; + while (allow_iterator[allow_count] != NULL) { + if (search_in_list(ce_denied_numbers, allow_iterator[allow_count])) { + fprintf(ERRORFILE, + "Trying to allow device with device number=%s which is not permitted in container-executor.cfg. %s\n", + allow_iterator[allow_count], + "It could be caused by a mismatch of devices reported by device plugin"); + return_code = -1; + goto cleanup; + } + allow_count++; + } + + // Deny devices configured in c-e.cfg + char** ce_iterator = ce_denied_numbers; + int ce_count = 0; + while (ce_iterator[ce_count] != NULL) { + // skip if duplicate with denied numbers passed in + if (search_in_list(deny_devices_number_tokens, ce_iterator[ce_count])) { + ce_count++; + continue; + } + char param_value[128]; + char type = 'c'; + memset(param_value, 0, sizeof(param_value)); + if (is_block_device(ce_iterator[ce_count])) { + type = 'b'; + } + snprintf(param_value, sizeof(param_value), "%c %s rwm", + type, + ce_iterator[ce_count]); + // Update device cgroups value + int rc = update_cgroups_parameters_func_p("devices", "deny", + container_id, param_value); + + if (0 != rc) { + fprintf(ERRORFILE, "CGroups: Failed to update cgroups. %s\n", param_value); + return_code = -1; + goto cleanup; + } + ce_count++; + } + } + + // Deny devices passed from java side + char** iterator = deny_devices_number_tokens; + int count = 0; + char* value = NULL; + int index = 0; + while (iterator[count] != NULL) { + // Replace like "c-242:0-rwm" to "c 242:0 rwm" + value = iterator[count]; + index = 0; + while (value[index] != '\0') { + if (value[index] == '-') { + value[index] = ' '; + } + index++; + } + // Update device cgroups value + int rc = update_cgroups_parameters_func_p("devices", "deny", + container_id, iterator[count]); + + if (0 != rc) { + fprintf(ERRORFILE, "CGroups: Failed to update cgroups\n"); + return_code = -1; + goto cleanup; + } + count++; + } + +cleanup: + if (ce_denied_numbers != NULL) { + free_values(ce_denied_numbers); + } + return return_code; +} + +void reload_devices_configuration() { + cfg_section = get_configuration_section(DEVICES_MODULE_SECTION_NAME, get_cfg()); +} + +/* + * Format of devices request commandline: + * The excluded_devices is comma separated device cgroups values with device type. + * The "-" will be replaced with " " to match the cgrooups parameter + * c-e --module-devices \ + * --excluded_devices b-8:16,c-244:0,c-244:1 \ + * --allowed_devices 8:32,8:48,243:2 \ + * --container_id container_x_y + */ +int handle_devices_request(update_cgroups_parameters_function func, + const char* module_name, int module_argc, char** module_argv) { + if (!cfg_section) { + reload_devices_configuration(); + } + + if (!module_enabled(cfg_section, DEVICES_MODULE_SECTION_NAME)) { + fprintf(ERRORFILE, + "Please make sure devices module is enabled before using it.\n"); + return -1; + } + + static struct option long_options[] = { + {EXCLUDED_DEVICES_OPTION, required_argument, 0, 'e' }, + {ALLOWED_DEVICES_OPTION, required_argument, 0, 'a' }, + {CONTAINER_ID_OPTION, required_argument, 0, 'c' }, + {0, 0, 0, 0} + }; + + int c = 0; + int option_index = 0; + + char** deny_device_value_tokens = NULL; + char** allow_device_value_tokens = NULL; + char container_id[MAX_CONTAINER_ID_LEN]; + memset(container_id, 0, sizeof(container_id)); + int failed = 0; + + optind = 1; + while((c = getopt_long(module_argc, module_argv, "e:a:c:", + long_options, &option_index)) != -1) { + switch(c) { + case 'e': + deny_device_value_tokens = split_delimiter(optarg, ","); + break; + case 'a': + allow_device_value_tokens = split_delimiter(optarg, ","); + break; + case 'c': + if (!validate_container_id(optarg)) { + fprintf(ERRORFILE, + "Specified container_id=%s is invalid\n", optarg); + failed = 1; + goto cleanup; + } + strncpy(container_id, optarg, MAX_CONTAINER_ID_LEN); + break; + default: + fprintf(ERRORFILE, + "Unknown option in devices command character %d %c, optionindex = %d\n", + c, c, optind); + failed = 1; + goto cleanup; + } + } + + if (0 == container_id[0]) { + fprintf(ERRORFILE, + "[%s] --container_id must be specified.\n", __func__); + failed = 1; + goto cleanup; + } + + if (NULL == deny_device_value_tokens) { + // Devices number is null, skip following call. + fprintf(ERRORFILE, "--excluded_devices is not specified, skip cgroups call.\n"); + goto cleanup; + } + + failed = internal_handle_devices_request(func, + deny_device_value_tokens, + allow_device_value_tokens, + container_id); + +cleanup: + if (deny_device_value_tokens) { + free_values(deny_device_value_tokens); + } + if (allow_device_value_tokens) { + free_values(allow_device_value_tokens); + } + return failed; +} diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h new file mode 100644 index 00000000000..c5d67851b83 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/devices/devices-module.h @@ -0,0 +1,45 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifdef __FreeBSD__ +#define _WITH_GETLINE +#endif + +#ifndef _MODULES_DEVICES_MUDULE_H_ +#define _MODULES_DEVICES_MUDULE_H_ + +// Denied device list. value format is "major1:minor1,major2:minor2" +#define DEVICES_DENIED_NUMBERS "devices.denied-numbers" +#define DEVICES_MODULE_SECTION_NAME "devices" + +// For unit test stubbing +typedef int (*update_cgroups_parameters_function)(const char*, const char*, + const char*, const char*); + +/** + * Handle devices requests + */ +int handle_devices_request(update_cgroups_parameters_function func, + const char* module_name, int module_argc, char** module_argv); + +/** + * Reload config from filesystem, visible for testing. + */ +void reload_devices_configuration(); + +#endif \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c index eea3e108ea6..1753954cb2e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.c @@ -44,6 +44,9 @@ char** split_delimiter(char *value, const char *delim) { memset(return_values, 0, sizeof(char *) * return_values_size); temp_tok = strtok_r(value, delim, &tempstr); + if (NULL == temp_tok) { + return_values[size++] = strdup(value); + } while (temp_tok != NULL) { temp_tok = strdup(temp_tok); if (NULL == temp_tok) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/devices/test-devices-module.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/devices/test-devices-module.cc new file mode 100644 index 00000000000..e80590a2e55 --- /dev/null +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/devices/test-devices-module.cc @@ -0,0 +1,298 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +extern "C" { +#include "configuration.h" +#include "container-executor.h" +#include "modules/cgroups/cgroups-operations.h" +#include "modules/devices/devices-module.h" +#include "test/test-container-executor-common.h" +#include "util.h" +} + +namespace ContainerExecutor { + +class TestDevicesModule : public ::testing::Test { +protected: + virtual void SetUp() { + if (mkdirs(TEST_ROOT, 0755) != 0) { + fprintf(ERRORFILE, "Failed to mkdir TEST_ROOT: %s\n", TEST_ROOT); + exit(1); + } + LOGFILE = stdout; + ERRORFILE = stderr; + } + + virtual void TearDown() { + + } +}; + +static std::vector cgroups_parameters_invoked; + +static int mock_update_cgroups_parameters( + const char* controller_name, + const char* param_name, + const char* group_id, + const char* value) { + char* buf = (char*) malloc(128); + strcpy(buf, controller_name); + cgroups_parameters_invoked.push_back(buf); + + buf = (char*) malloc(128); + strcpy(buf, param_name); + cgroups_parameters_invoked.push_back(buf); + + buf = (char*) malloc(128); + strcpy(buf, group_id); + cgroups_parameters_invoked.push_back(buf); + + buf = (char*) malloc(128); + strcpy(buf, value); + cgroups_parameters_invoked.push_back(buf); + return 0; +} + +static void clear_cgroups_parameters_invoked() { + for (std::vector::size_type i = 0; i < cgroups_parameters_invoked.size(); i++) { + free((void *) cgroups_parameters_invoked[i]); + } + cgroups_parameters_invoked.clear(); +} + +static void verify_param_updated_to_cgroups( + int argc, const char** argv) { + ASSERT_EQ(argc, cgroups_parameters_invoked.size()); + + int offset = 0; + while (offset < argc) { + ASSERT_STREQ(argv[offset], cgroups_parameters_invoked[offset]); + offset++; + } +} + +static void write_and_load_devices_module_to_cfg(const char* cfg_filepath, int enabled) { + FILE *file = fopen(cfg_filepath, "w"); + if (file == NULL) { + printf("FAIL: Could not open configuration file: %s\n", cfg_filepath); + exit(1); + } + fprintf(file, "[devices]\n"); + if (enabled) { + fprintf(file, "module.enabled=true\n"); + } else { + fprintf(file, "module.enabled=false\n"); + } + fclose(file); + + // Read config file + read_executor_config(cfg_filepath); + reload_devices_configuration(); +} + +static void append_config(const char* cfg_filepath, char[] values) { + FILE *file = fopen(cfg_filepath, "a"); + if (file == NULL) { + printf("FAIL: Could not open configuration file: %s\n", cfg_filepath); + exit(1); + } + fprintf(file, values); + fclose(file); + + // Read config file + read_executor_config(cfg_filepath); + reload_devices_configuration(); +} + +static void test_devices_module_enabled_disabled(int enabled) { + // Write config file. + const char *filename = TEST_ROOT "/test_cgroups_module_enabled_disabled.cfg"; + write_and_load_devices_module_to_cfg(filename, enabled); + char excluded_devices[] = "c-243:0-rwm,c-243:1-rwm"; + char allowed_devices[] = "243:2"; + char* argv[] = { (char*) "--module-devices", (char*) "--excluded_devices", + excluded_devices, + (char*) "--allowed_devices", + allowed_devices, + (char*) "--container_id", + (char*) "container_1498064906505_0001_01_000001" }; + + int rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 7, argv); + + int EXPECTED_RC; + if (enabled) { + EXPECTED_RC = 0; + } else { + EXPECTED_RC = -1; + } + ASSERT_EQ(EXPECTED_RC, rc); + + clear_cgroups_parameters_invoked(); + free_executor_configurations(); +} + +TEST_F(TestDevicesModule, test_verify_device_module_calls_cgroup_parameter) { + // Write config file. + const char *filename = TEST_ROOT "/test_verify_devices_module_calls_cgroup_parameter.cfg"; + write_and_load_devices_module_to_cfg(filename, 1); + + char* container_id = (char*) "container_1498064906505_0001_01_000001"; + char excluded_devices[] = "c-243:0-rwm,c-243:1-rwm"; + char allowed_devices[] = "243:2"; + char* argv[] = { (char*) "--module-devices", (char*) "--excluded_devices", + excluded_devices, + (char*) "--allowed_devices", + allowed_devices, + (char*) "--container_id", + container_id }; + /* Test case 1: block 2 devices */ + clear_cgroups_parameters_invoked(); + int rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 7, argv); + ASSERT_EQ(0, rc) << "Should success.\n"; + // Verify cgroups parameters + const char* expected_cgroups_argv[] = { "devices", "deny", container_id, "c 243:0 rwm", + "devices", "deny", container_id, "c 243:1 rwm"}; + verify_param_updated_to_cgroups(8, expected_cgroups_argv); + + /* Test case 2: block 0 devices */ + clear_cgroups_parameters_invoked(); + char* argv_1[] = { (char*) "--module-devices", (char*) "--container_id", container_id }; + rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 3, argv_1); + ASSERT_EQ(0, rc) << "Should success.\n"; + + // Verify cgroups parameters + verify_param_updated_to_cgroups(0, NULL); + + clear_cgroups_parameters_invoked(); + free_executor_configurations(); +} + +TEST_F(TestDevicesModule, test_update_cgroup_parameter_with_config) { + // Write config file. + const char *filename = TEST_ROOT "/test_update_cgroup_parameter_with_config.cfg"; + write_and_load_devices_module_to_cfg(filename, 1); + // Add denied numbers + char tokens[] = "devices.denied-numbers=243:1\n"; + append_config(filename, tokens); + + char* container_id = (char*) "container_1498064906505_0001_01_000001"; + char excluded_devices[] = "c-243:0-rwm,c-243:1-rwm"; + char allowed_devices[] = "243:2"; + char* argv[] = { (char*) "--module-devices", (char*) "--excluded_devices", + excluded_devices, + (char*) "--allowed_devices", + allowed_devices, + (char*) "--container_id", + container_id }; + /* Test case 1: block 2 devices */ + clear_cgroups_parameters_invoked(); + int rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 7, argv); + ASSERT_EQ(0, rc) << "Should success.\n"; + // Verify cgroups parameters + const char* expected_cgroups_argv[] = { "devices", "deny", container_id, "c 243:0 rwm", + "devices", "deny", container_id, "c 243:1 rwm"}; + verify_param_updated_to_cgroups(8, expected_cgroups_argv); + + /* Test case 2: block 2 devices but try allow devices not permitted by config*/ + clear_cgroups_parameters_invoked(); + // device plugin reported 0,1,2,3 totally. Allocated 1,2 + // But c-e.cfg has device 1 denied. + char excluded_devices2[] = "c-243:0-rwm,c-243:3-rwm"; + char allowed_devices2[] = "243:1,243:2"; + char* argv1[] = { (char*) "--module-devices", (char*) "--excluded_devices", + excluded_devices2, + (char*) "--allowed_devices", + allowed_devices2, + (char*) "--container_id", + container_id }; + rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 7, argv1); + ASSERT_NE(0, rc) << "Should fail.\n"; + + clear_cgroups_parameters_invoked(); + free_executor_configurations(); +} + +TEST_F(TestDevicesModule, test_illegal_cli_parameters) { + // Write config file. + const char *filename = TEST_ROOT "/test_illegal_cli_parameters.cfg"; + write_and_load_devices_module_to_cfg(filename, 1); + char excluded_devices[] = "c-243:0-rwm,c-243:1-rwm"; + char allowed_devices[] = "243:2"; + // Illegal container id - 1 + char* argv[] = { (char*) "--module-devices", (char*) "--excluded_devices", + excluded_devices, + (char*) "--allowed_devices", + allowed_devices, + (char*) "--container_id", (char*) "xxxx" }; + int rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 7, argv); + ASSERT_NE(0, rc) << "Should fail.\n"; + + // Illegal container id - 2 + clear_cgroups_parameters_invoked(); + char* argv_1[] = { (char*) "--module-devices", (char*) "--excluded_devices", + excluded_devices, + (char*) "--allowed_devices", + allowed_devices, + (char*) "--container_id", (char*) "container_1" }; + rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 7, argv_1); + ASSERT_NE(0, rc) << "Should fail.\n"; + + // Illegal container id - 3 + clear_cgroups_parameters_invoked(); + char* argv_2[] = { (char*) "--module-devices", + (char*) "--excluded_devices", + excluded_devices }; + rc = handle_devices_request(&mock_update_cgroups_parameters, + "devices", 3, argv_2); + ASSERT_NE(0, rc) << "Should fail.\n"; + + clear_cgroups_parameters_invoked(); + free_executor_configurations(); +} + +TEST_F(TestDevicesModule, test_devices_module_disabled) { + test_devices_module_enabled_disabled(0); +} + +TEST_F(TestDevicesModule, test_devices_module_enabled) { + test_devices_module_enabled_disabled(1); +} +} // namespace ContainerExecutor \ No newline at end of file