commit e942df887a62cf817baeca3ac4daeb032e2a677e Author: Eric Yang Date: Wed Apr 25 21:42:28 2018 -0400 YARN-7654. Added support for docker ENTRY_PONT. Contributed by Eric Yang Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java Conflicts: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java index 38ad596..b63fe61 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java @@ -244,7 +244,14 @@ * Comma separate list of directories that the container should use for * logging. */ - LOG_DIRS("LOG_DIRS"); + LOG_DIRS("LOG_DIRS"), + + /** + * $YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE + * Final, Docker run support ENTRY_POINT. + */ + YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE( + "YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE"); private final String variable; private Environment(String variable) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java index dc51b25..30bfe8f 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/containerlaunch/AbstractLauncher.java @@ -101,6 +101,11 @@ public void addCommand(String cmd) { commands.add(cmd); } + public void setCommand(String cmd) { + commands.clear(); + commands.add(cmd); + } + /** * Complete the launch context (copy in env vars, etc). * @return the container to launch diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java index c3e2619..1351937 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/provider/docker/DockerProviderService.java @@ -17,13 +17,23 @@ */ package org.apache.hadoop.yarn.service.provider.docker; +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.yarn.service.component.instance.ComponentInstance; import org.apache.hadoop.yarn.service.provider.AbstractProviderService; +import org.apache.hadoop.yarn.service.provider.ProviderUtils; +import org.apache.hadoop.yarn.api.records.Container; +import org.apache.hadoop.yarn.service.api.records.Component; import org.apache.hadoop.yarn.service.api.records.Service; import org.apache.hadoop.yarn.service.utils.SliderFileSystem; import org.apache.hadoop.yarn.service.containerlaunch.AbstractLauncher; +import org.apache.hadoop.yarn.service.containerlaunch.CommandLineBuilder; +import org.apache.hadoop.yarn.service.containerlaunch.ContainerLaunchService; +import org.apache.hadoop.yarn.service.exceptions.SliderException; +import org.apache.hadoop.yarn.api.ApplicationConstants.Environment; import java.io.IOException; +import java.util.Map; public class DockerProviderService extends AbstractProviderService implements DockerKeys { @@ -39,4 +49,40 @@ public void processArtifact(AbstractLauncher launcher, launcher.setRunPrivilegedContainer( compInstance.getCompSpec().getRunPrivilegedContainer()); } + + @Override + public void buildContainerLaunchContext(AbstractLauncher launcher, + Service service, ComponentInstance instance, + SliderFileSystem fileSystem, Configuration yarnConf, Container container, + ContainerLaunchService.ComponentLaunchContext compLaunchContext) + throws IOException, SliderException { + Component component = instance.getComponent().getComponentSpec(); + processArtifact(launcher, instance, fileSystem, service); + super.buildContainerLaunchContext(launcher, service, instance, fileSystem, + yarnConf, container, compLaunchContext); + Map globalTokens = + instance.getComponent().getScheduler().globalTokens; + Map tokensForSubstitution = ProviderUtils + .initCompTokensForSubstitute(instance, container, compLaunchContext); + tokensForSubstitution.putAll(globalTokens); + + // substitute launch command + String launchCommand = component.getLaunchCommand(); + // docker container may have empty commands + if (!StringUtils.isEmpty(launchCommand)) { + launchCommand = ProviderUtils + .substituteStrWithTokens(launchCommand, tokensForSubstitution); + CommandLineBuilder operation = new CommandLineBuilder(); + operation.add(launchCommand); + boolean overrideDisable = Boolean.parseBoolean(component + .getConfiguration().getEnv(Environment + .YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE.name())); + if (!overrideDisable) { + operation.addOutAndErrFiles(OUT_FILE, ERR_FILE); + } + launcher.setCommand(operation.build()); + } + } + + } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index fa77899..16aec8e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -1677,6 +1677,20 @@ public void sanitizeEnv(Map environment, Path pwd, containerLogDirs, Map> resources, Path nmPrivateClasspathJarDir, Set nmVars) throws IOException { + // Base on discussion in YARN-7654, for ENTRY_POINT enabled + // docker container, we forward user defined environment variables + // without node manager environment variables. This is the reason + // that we skip sanitizeEnv method. + boolean overrideDisable = Boolean.parseBoolean( + environment.get( + Environment. + YARN_CONTAINER_RUNTIME_DOCKER_RUN_OVERRIDE_DISABLE. + name())); + if (overrideDisable) { + environment.remove("WORK_DIR"); + return; + } + /** * Non-modifiable environment variables */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java index 0bacd03..d025e5e 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java @@ -81,6 +81,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE; import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.LinuxContainerRuntimeConstants.*; /** @@ -235,7 +236,8 @@ @InterfaceAudience.Private public static final String ENV_DOCKER_CONTAINER_DELAYED_REMOVAL = "YARN_CONTAINER_RUNTIME_DOCKER_DELAYED_REMOVAL"; - + public static final String ENV_DOCKER_COTAINER_ENV_FILE = + "YARN_CONTAINER_RUNTIME_DOCKER_ENV_FILE"; private Configuration conf; private Context nmContext; private DockerClient dockerClient; @@ -741,6 +743,10 @@ public void launchContainer(ContainerRuntimeContext ctx) String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE); String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK); String hostname = environment.get(ENV_DOCKER_CONTAINER_HOSTNAME); + boolean dockerOverride = ((environment + .get(ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE) == null) ? false : + Boolean.parseBoolean(environment + .get(ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE))); if(network == null || network.isEmpty()) { network = defaultNetwork; @@ -802,9 +808,10 @@ public void launchContainer(ContainerRuntimeContext ctx) @SuppressWarnings("unchecked") DockerRunCommand runCommand = new DockerRunCommand(containerIdStr, dockerRunAsUser, imageName) - .detachOnRun() - .setContainerWorkDir(containerWorkDir.toString()) .setNetworkType(network); + if (!dockerOverride) { + runCommand.setContainerWorkDir(containerWorkDir.toString()); + } // Only add hostname if network is not host or if Registry DNS is enabled. if (!network.equalsIgnoreCase("host") || conf.getBoolean(RegistryConstants.KEY_DNS_ENABLED, @@ -875,11 +882,16 @@ public void launchContainer(ContainerRuntimeContext ctx) addCGroupParentIfRequired(resourcesOpts, containerIdStr, runCommand); - String disableOverride = environment.get( - ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE); - - if (disableOverride != null && disableOverride.equals("true")) { + if (dockerOverride) { LOG.info("command override disabled"); + runCommand.setOverrideDisabled(true); + for (Map.Entry kv: environment.entrySet()) { + runCommand.addEnv(kv.getKey(), kv.getValue()); + } + runCommand.setOverrideCommandWithArgs(container.getLaunchContext() + .getCommands()); + runCommand.disableDetach(); + runCommand.setLogDir(container.getLogDir()); } else { List overrideCommands = new ArrayList<>(); Path launchDst = @@ -888,6 +900,7 @@ public void launchContainer(ContainerRuntimeContext ctx) overrideCommands.add("bash"); overrideCommands.add(launchDst.toUri().getPath()); runCommand.setOverrideCommandWithArgs(overrideCommands); + runCommand.detachOnRun(); } if(enableUserReMapping) { diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java index 7bd4546..0f71497 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java @@ -49,6 +49,7 @@ LoggerFactory.getLogger(DockerClient.class); private static final String TMP_FILE_PREFIX = "docker."; private static final String TMP_FILE_SUFFIX = ".cmd"; + private static final String TMP_ENV_FILE_SUFFIX = ".env"; private final String tmpDirPath; public DockerClient(Configuration conf) throws ContainerExecutionException { @@ -95,7 +96,6 @@ public String writeCommandToTempFile(DockerCommand cmd, String filePrefix) .join(",", entry.getValue())); } printWriter.close(); - return dockerCommandFile.getAbsolutePath(); } catch (IOException e) { LOG.warn("Unable to write docker command to temporary file!"); @@ -103,6 +103,22 @@ public String writeCommandToTempFile(DockerCommand cmd, String filePrefix) } } + private String writeEnvFile(DockerCommand cmd, String filePrefix, + String cmdDir) throws IOException { + File dockerEnvFile = File.createTempFile(TMP_FILE_PREFIX + filePrefix, + TMP_ENV_FILE_SUFFIX, new + File(cmdDir)); + Writer envWriter = new OutputStreamWriter( + new FileOutputStream(dockerEnvFile), "UTF-8"); + PrintWriter envPrintWriter = new PrintWriter(envWriter); + for (Map.Entry entry : + ((DockerRunCommand) cmd).getEnv().entrySet()) { + envPrintWriter.println(entry.getKey() + "=" + entry.getValue()); + } + envPrintWriter.close(); + return dockerEnvFile.getAbsolutePath(); + } + public String writeCommandToTempFile(DockerCommand cmd, ContainerId containerId, Context nmContext) throws ContainerExecutionException { @@ -132,11 +148,13 @@ public String writeCommandToTempFile(DockerCommand cmd, for (Map.Entry> entry : cmd.getDockerCommandWithArguments().entrySet()) { if (entry.getKey().contains("=")) { + printWriter.close(); throw new ContainerExecutionException( "'=' found in entry for docker command file, key = " + entry .getKey() + "; value = " + entry.getValue()); } if (entry.getValue().contains("\n")) { + printWriter.close(); throw new ContainerExecutionException( "'\\n' found in entry for docker command file, key = " + entry .getKey() + "; value = " + entry.getValue()); @@ -144,6 +162,12 @@ public String writeCommandToTempFile(DockerCommand cmd, printWriter.println(" " + entry.getKey() + "=" + StringUtils .join(",", entry.getValue())); } + if (cmd instanceof DockerRunCommand) { + if (((DockerRunCommand) cmd).containsEnv()) { + String path = writeEnvFile(cmd, filePrefix, cmdDir); + printWriter.println(" environ=" + path); + } + } printWriter.close(); return dockerCommandFile.toString(); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java index bfeeaf5..1408808 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerRunCommand.java @@ -24,9 +24,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeMap; public class DockerRunCommand extends DockerCommand { private static final String RUN_COMMAND = "run"; + private final Map userEnv; /** The following are mandatory: */ public DockerRunCommand(String containerId, String user, String image) { @@ -34,6 +36,7 @@ public DockerRunCommand(String containerId, String user, String image) { super.addCommandArguments("name", containerId); super.addCommandArguments("user", user); super.addCommandArguments("image", image); + this.userEnv = new TreeMap(); } public DockerRunCommand removeContainerOnExit() { @@ -174,4 +177,49 @@ public DockerRunCommand setOverrideCommandWithArgs( public Map> getDockerCommandWithArguments() { return super.getDockerCommandWithArguments(); } + + public DockerRunCommand setOverrideDisabled(boolean toggle) { + String value = "false"; + if (toggle) { + value = "true"; + } + super.addCommandArguments("use-entry-point", value); + return this; + } + + public DockerRunCommand setLogDir(String logDir) { + super.addCommandArguments("log-dir", logDir); + return this; + } + + /** + * Check if user defined environment variables are empty. + * + * @return true if user defined environment variables are not empty. + */ + public boolean containsEnv() { + if (userEnv.size() > 0) { + return true; + } + return false; + } + + /** + * Get user defined environment variables. + * + * @return a map of user defined environment variables + */ + public Map getEnv() { + return userEnv; + } + + /** + * Add user defined environment variables. + * + * @param key Captalized environment variable key name + * @param value Environment variable value + */ + public final void addEnv(String key, String value) { + userEnv.put(key, value); + } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index c5adbe4..79fa9f2 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -68,6 +68,7 @@ #endif static const int DEFAULT_MIN_USERID = 1000; +static const int MAX_RETRIES = 10; static const char* DEFAULT_BANNED_USERS[] = {"yarn", "mapred", "hdfs", "bin", 0}; @@ -94,6 +95,8 @@ static gid_t nm_gid = -1; struct configuration CFG = {.size=0, .sections=NULL}; struct section executor_cfg = {.size=0, .kv_pairs=NULL}; +static char *chosen_container_log_dir = NULL; + char *concatenate(char *concat_pattern, char *return_path_name, int numArgs, ...); @@ -755,8 +758,9 @@ static int create_container_directories(const char* user, const char *app_id, } else if (mkdirs(container_log_dir, perms) != 0) { free(container_log_dir); } else { - free(container_log_dir); result = 0; + chosen_container_log_dir = strdup(container_log_dir); + free(container_log_dir); } } free(combined_name); @@ -1129,6 +1133,31 @@ char* get_container_log_directory(const char *log_root, const char* app_id, container_id); } +char *init_log_path(const char *container_log_dir, const char *logfile) { + char *tmp_buffer = NULL; + tmp_buffer = make_string("%s/%s", container_log_dir, logfile); + + mode_t permissions = S_IRUSR | S_IWUSR | S_IRGRP; + int fd = open(tmp_buffer, O_CREAT | O_WRONLY, permissions); + if (fd >= 0) { + close(fd); + if (change_owner(tmp_buffer, user_detail->pw_uid, user_detail->pw_gid) != 0) { + fprintf(ERRORFILE, "Failed to chown %s to %d:%d: %s\n", tmp_buffer, user_detail->pw_uid, user_detail->pw_gid, + strerror(errno)); + tmp_buffer = NULL; + } else if (chmod(tmp_buffer, permissions) != 0) { + fprintf(ERRORFILE, "Can't chmod %s - %s\n", + tmp_buffer, strerror(errno)); + tmp_buffer = NULL; + } + } else { + fprintf(ERRORFILE, "Failed to create file %s - %s\n", tmp_buffer, + strerror(errno)); + tmp_buffer = NULL; + } + return tmp_buffer; +} + int create_container_log_dirs(const char *container_id, const char *app_id, char * const * log_dirs) { char* const* log_root; @@ -1506,6 +1535,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id, char *docker_inspect_exitcode_command = NULL; int container_file_source =-1; int cred_file_source = -1; + int docker_override = -1; gid_t user_gid = getegid(); uid_t prev_uid = geteuid(); @@ -1560,6 +1590,18 @@ int launch_docker_container_as_user(const char * user, const char *app_id, goto cleanup; } + docker_override = use_entry_point(); + char *so = init_log_path(chosen_container_log_dir, "stdout.txt"); + if (so == NULL) { + exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + goto cleanup; + } + char *se = init_log_path(chosen_container_log_dir, "stderr.txt"); + if (se == NULL) { + exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + goto cleanup; + } + docker_command_with_binary = flatten(docker_command); // Launch container @@ -1573,14 +1615,75 @@ int launch_docker_container_as_user(const char * user, const char *app_id, } if (child_pid == 0) { + FILE* so_fd = fopen(so, "a+"); + if (so_fd == NULL) { + fprintf(ERRORFILE, "Could not append to %s\n", so); + exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + goto cleanup; + } + FILE* se_fd = fopen(se, "a+"); + if (se_fd == NULL) { + fprintf(ERRORFILE, "Could not append to %s\n", se); + exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + fclose(so_fd); + goto cleanup; + } + fprintf(so_fd, "Launching docker container...\n"); + fprintf(so_fd, "Docker run command: %s\n", docker_command_with_binary); + // if entry point is enabled, clone docker command output + // to stdout.txt and stderr.txt for yarn. + if (docker_override == 0) { + if (dup2(fileno(so_fd), fileno(stdout)) == -1) { + fprintf(ERRORFILE, "Could not append to stdout.txt\n"); + fclose(so_fd); + return UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + } + if (dup2(fileno(se_fd), fileno(stderr)) == -1) { + fprintf(ERRORFILE, "Could not append to stderr.txt\n"); + fclose(se_fd); + return UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + } + } + fclose(so_fd); + fclose(se_fd); execvp(docker_binary, docker_command); fprintf(ERRORFILE, "failed to execute docker command! error: %s\n", strerror(errno)); return UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; } else { - exit_code = wait_and_get_exit_code(child_pid); - if (exit_code != 0) { - exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; - goto cleanup; + if (docker_override == 0) { + int pid = 0; + int res = 0; + int count = 0; + docker_inspect_command = make_string( + "%s inspect --format {{.State.Pid}} %s", + docker_binary, container_id); + // check for docker container pid + while (count < MAX_RETRIES) { + fprintf(LOGFILE, "Inspecting docker container...\n"); + fprintf(LOGFILE, "Docker inspect command: %s\n", docker_inspect_command); + fflush(LOGFILE); + FILE* inspect_docker = popen(docker_inspect_command, "r"); + res = fscanf (inspect_docker, "%d", &pid); + fprintf(LOGFILE, "pid from docker inspect: %d\n", pid); + if (pclose (inspect_docker) != 0 || res <= 0) { + fprintf (ERRORFILE, + "Could not inspect docker to get pid %s.\n", docker_inspect_command); + fflush(ERRORFILE); + exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + } else { + if (pid != 0) { + break; + } + } + sleep(3); + count++; + } + } else { + exit_code = wait_and_get_exit_code(child_pid); + if (exit_code != 0) { + exit_code = UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + goto cleanup; + } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c index 8cd59f7..8e68b47 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.c @@ -32,6 +32,8 @@ #include #include +int entry_point = -1; + static int read_and_verify_command_file(const char *command_file, const char *docker_command, struct configuration *command_config) { int ret = 0; @@ -348,6 +350,10 @@ char *get_docker_binary(const struct configuration *conf) { return docker_binary; } +int use_entry_point() { + return entry_point; +} + int docker_module_enabled(const struct configuration *conf) { struct section *section = get_configuration_section(CONTAINER_EXECUTOR_CFG_DOCKER_SECTION, conf); if (section != NULL) { @@ -365,6 +371,12 @@ int get_docker_command(const char *command_file, const struct configuration *con return INVALID_COMMAND_FILE; } + char *value = get_configuration_value("use-entry-point", DOCKER_COMMAND_FILE_SECTION, &command_config); + if (value != NULL && strcasecmp(value, "true") == 0) { + entry_point = 0; + } + free(value); + char *command = get_configuration_value("docker-command", DOCKER_COMMAND_FILE_SECTION, &command_config); if (strcmp(DOCKER_INSPECT_COMMAND, command) == 0) { return get_docker_inspect_command(command_file, conf, args); @@ -999,6 +1011,24 @@ static int set_devices(const struct configuration *command_config, const struct return ret; } +static int set_env(const struct configuration *command_config, struct args *args) { + int ret = 0; + // Use envfile method. + char *envfile = get_configuration_value("environ", DOCKER_COMMAND_FILE_SECTION, command_config); + if (envfile != NULL) { + ret = add_to_args(args, "--env-file"); + if (ret != 0) { + ret = BUFFER_TOO_SMALL; + } + ret = add_to_args(args, envfile); + if (ret != 0) { + ret = BUFFER_TOO_SMALL; + } + free(envfile); + } + return ret; +} + /** * Helper function to help normalize mounts for checking if mounts are * permitted. The function does the following - @@ -1510,6 +1540,11 @@ int get_docker_run_command(const char *command_file, const struct configuration return ret; } + ret = set_env(&command_config, args); + if (ret != 0) { + return BUFFER_TOO_SMALL; + } + ret = add_to_args(args, image); if (ret != 0) { reset_args(args); diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h index 330d722..b07c055 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/docker-util.h @@ -86,6 +86,12 @@ char *get_docker_binary(const struct configuration *conf); int get_docker_command(const char* command_file, const struct configuration* conf, args *args); /** + * Check if use-entry-point flag is set. + * @return 0 when use-entry-point flag is set. + */ +int use_entry_point(); + +/** * Get the Docker inspect command line string. The function will verify that the params file is meant for the * inspect command. * @param command_file File containing the params for the Docker inspect command diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc index 1096935..a69ea0d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test_docker_util.cc @@ -1312,6 +1312,48 @@ namespace ContainerExecutor { run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command); } + TEST_F(TestDockerUtil, test_docker_run_entry_point) { + + std::string container_executor_contents = "[docker]\n" + " docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n" + " docker.allowed.rw-mounts=/tmp\n docker.allowed.networks=bridge\n " + " docker.privileged-containers.enabled=1\n docker.allowed.capabilities=CHOWN,SETUID\n" + " docker.allowed.devices=/dev/test\n docker.privileged-containers.registries=hadoop\n"; + write_file(container_executor_cfg_file, container_executor_contents); + int ret = read_config(container_executor_cfg_file.c_str(), &container_executor_cfg); + if (ret != 0) { + FAIL(); + } + ret = create_ce_file(); + if (ret != 0) { + std::cerr << "Could not create ce file, skipping test" << std::endl; + return; + } + + std::vector > file_cmd_vec; + file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n" + " docker-command=run\n" + " name=container_e1_12312_11111_02_000001\n" + " image=hadoop/docker-image\n" + " user=nobody\n" + " use-entry-point=true\n" + " environ=/tmp/test.env\n", + "/usr/bin/docker run --name=container_e1_12312_11111_02_000001 --user=nobody --cap-drop=ALL " + "--env-file /tmp/test.env hadoop/docker-image")); + + std::vector > bad_file_cmd_vec; + + bad_file_cmd_vec.push_back(std::make_pair( + "[docker-command-execution]\n" + " docker-command=run\n" + " image=hadoop/docker-image\n" + " user=nobody", + static_cast(INVALID_DOCKER_CONTAINER_NAME))); + + run_docker_command_test(file_cmd_vec, bad_file_cmd_vec, get_docker_run_command); + } + TEST_F(TestDockerUtil, test_docker_run_no_privileged) { std::string container_executor_contents[] = {"[docker]\n docker.allowed.ro-mounts=/var,/etc,/usr/bin/cut\n"