commit 7969d230e93d802bbee04536238808a1001bc082 Author: Eric Yang Date: Wed Apr 18 14:08:15 2018 -0400 YARN-8097. Added Docker env-file flag support. Contributed by Eric Yang diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 15c578a..42478e2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -234,7 +234,8 @@ @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_DELAYED_REMOVAL = "YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL"; - + public static final String ENV_DOCKER_COTAINER_ENV_FILE = + "YARN_CONTAINER_RUNTIME_DOCKER_ENV_FILE"; private Configuration conf; private Context nmContext; private DockerClient dockerClient; @@ -883,6 +884,10 @@ public void launchContainer(ContainerRuntimeContext ctx) .getCommands()); runCommand.disableDetach(); runCommand.setLogDir(container.getLogDir()); + if(environment.containsKey(ENV_DOCKER_COTAINER_ENV_FILE)) { + String envfile = environment.get(ENV_DOCKER_COTAINER_ENV_FILE); + runCommand.setEnv(envfile); + } } else { List overrideCommands = new ArrayList<>(); Path launchDst = diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index 2530f97..99d0a38 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -1060,22 +1060,38 @@ static int set_devices(const struct configuration *command_config, const struct static int set_env(const struct configuration *command_config, char **out, int *index) { int ret = 0; int count = 0; - const struct section *section = get_configuration_section(DOCKER_COMMAND_ENV_SECTION, command_config); - if (section == NULL) { - return ret; - } - for (count = 0; count < section->size; count++) { - ret = add_to_buffer(out, index, "-e"); + // Use envfile method. + char *envfile = get_configuration_value("environ", DOCKER_COMMAND_FILE_SECTION, command_config); + if (envfile != NULL) { + ret = add_to_buffer(out, index, "--env-file"); if (ret != 0) { ret = BUFFER_TOO_SMALL; - break; } - char *tmp_buffer = make_string("%s=%s", section->kv_pairs[count]->key, section->kv_pairs[count]->value); - ret = add_to_buffer(out, index, tmp_buffer); - free(tmp_buffer); + ret = add_to_buffer(out, index, envfile); if (ret != 0) { ret = BUFFER_TOO_SMALL; - break; + } + free(envfile); + } + // Use command arguements method. + if (ret == 0) { + const struct section *section = get_configuration_section(DOCKER_COMMAND_ENV_SECTION, command_config); + if (section == NULL) { + return ret; + } + for (count = 0; count < section->size; count++) { + ret = add_to_buffer(out, index, "-e"); + if (ret != 0) { + ret = BUFFER_TOO_SMALL; + break; + } + char *tmp_buffer = make_string("%s=%s", section->kv_pairs[count]->key, section->kv_pairs[count]->value); + ret = add_to_buffer(out, index, tmp_buffer); + free(tmp_buffer); + if (ret != 0) { + ret = BUFFER_TOO_SMALL; + break; + } } } return ret; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc index 82872d6..630b9af 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc @@ -1347,11 +1347,12 @@ namespace ContainerExecutor { " image=hadoop/docker-image\n" " user=nobody\n" " use-entry-point=true\n" + " environ=/tmp/test.env\n" "[docker-command-environment]\n" " HADOOP_CONF_DIR=/etc/hadoop/conf\n" " LANG=en_US.UTF-8\n", "/usr/bin/docker run --name=container_e1_12312_11111_02_000001 --user=nobody --cap-drop=ALL " - "-e HADOOP_CONF_DIR=/etc/hadoop/conf -e LANG=en_US.UTF-8 hadoop/docker-image")); + "--env-file /tmp/test.env -e HADOOP_CONF_DIR=/etc/hadoop/conf -e LANG=en_US.UTF-8 hadoop/docker-image")); std::vector > bad_file_cmd_vec;