diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java index 918c30a..f6c28f4 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java @@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerLivenessContext; import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisitionContext; @@ -183,12 +184,14 @@ public abstract void deleteAsUser(DeletionAsUserContext ctx) /** * Create a symlink file which points to the target. - * @param target The target for symlink + * @param src The source file for symlink * @param symlink the symlink file + * @param container the container for which to create the symlink * @throws IOException Error when creating symlinks + * @throws PrivilegedOperationException Error when creating symlinks */ - public abstract void symLink(String target, String symlink) - throws IOException; + public abstract void symLink(String src, String symlink, Container container) + throws IOException, PrivilegedOperationException; /** * Check if a container is alive. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java index 9a0549d..1f6507c 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java @@ -513,8 +513,9 @@ public void deleteAsUser(DeletionAsUserContext ctx) } @Override - public void symLink(String target, String symlink) throws IOException { - FileUtil.symLink(target, symlink); + public void symLink(String src, String symlink, Container container) + throws IOException { + FileUtil.symLink(src, symlink); } /** Permissions for user dir. diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java index 1390214..7b8235a 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java @@ -491,7 +491,7 @@ public void deleteAsUser(DeletionAsUserContext ctx) } @Override - public void symLink(String target, String symlink) + public void symLink(String src, String symlink, Container container) throws IOException { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java index cc12b20..c194361 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java @@ -23,6 +23,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.util.ReflectionUtils; @@ -32,6 +33,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerDiagnosticsUpdateEvent; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor; @@ -55,7 +57,11 @@ import org.apache.hadoop.yarn.server.nodemanager.util.DefaultLCEResourcesHandler; import org.apache.hadoop.yarn.server.nodemanager.util.LCEResourcesHandler; import java.io.File; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.OutputStreamWriter; +import java.io.PrintWriter; +import java.io.Writer; import java.net.InetSocketAddress; import java.util.ArrayList; import java.util.Arrays; @@ -683,8 +689,50 @@ public void deleteAsUser(DeletionAsUserContext ctx) { } @Override - public void symLink(String target, String symlink) { + public void symLink(String src, String symlink, Container container) + throws IOException, PrivilegedOperationException { + ContainerLaunch.ShellScriptBuilder scriptBuilder = + ContainerLaunch.ShellScriptBuilder.create(); + scriptBuilder.symlink(new Path(src), new Path(symlink)); + ContainerId containerId = container.getContainerId(); + String nmPrivateDir = container.getNMContext().getLocalDirsHandler() + .getLocalPathForWrite(ContainerLaunch.getContainerPrivateDir( + containerId.getApplicationAttemptId().getApplicationId().toString(), + containerId.toString())).toString(); + File privateScriptFile = + writeScriptToNMPrivateDir(nmPrivateDir, scriptBuilder.toString()); + String userScriptFile = + container.getWorkDir() + Path.SEPARATOR + privateScriptFile.getName(); + runScriptAsUser(container.getUser(), privateScriptFile, userScriptFile); + FileUtil.fullyDelete(privateScriptFile); + } + protected void runScriptAsUser(String user, File privateScriptFile, + String userScriptFile) + throws PrivilegedOperationException { + String runAsUser = getRunAsUser(user); + PrivilegedOperation op = + new PrivilegedOperation(PrivilegedOperation.OperationType.RUN_SCRIPT); + op.appendArgs(runAsUser); + op.appendArgs(privateScriptFile.getAbsolutePath()); + op.appendArgs(userScriptFile); + + if (LOG.isDebugEnabled()) { + LOG.debug("Copy " + privateScriptFile + " to " + userScriptFile); + } + PrivilegedOperationExecutor executor = + PrivilegedOperationExecutor.getInstance(getConf()); + executor.executePrivilegedOperation(null, op, null, null, false, true); + } + + private File writeScriptToNMPrivateDir(String nmPrivateDir, String command) + throws IOException { + File tmp = File.createTempFile("cmd", ".tmp", new File(nmPrivateDir)); + Writer writer = new OutputStreamWriter(new FileOutputStream(tmp), "UTF-8"); + PrintWriter printWriter = new PrintWriter(writer); + printWriter.print(command); + printWriter.close(); + return tmp; } @Override diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java index 79eeaf1..b348894 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java @@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.event.EventHandler; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet; import java.util.List; @@ -70,7 +71,7 @@ void setIpAndHost(String[] ipAndHost); - String toString(); + Context getNMContext(); ResourceSet getResourceSet(); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java index 7f6d5b6..4ab3229 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java @@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerFinishedEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType; +import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet; import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationCleanupEvent; @@ -583,6 +584,11 @@ public ResourceSet getResourceSet() { return this.resourceSet; } + @Override + public Context getNMContext() { + return context; + } + @SuppressWarnings("unchecked") private void sendFinishedEvents() { // Inform the application @@ -822,25 +828,35 @@ public ContainerState transition(ContainerImpl container, public void transition(ContainerImpl container, ContainerEvent event) { ContainerResourceLocalizedEvent rsrcEvent = (ContainerResourceLocalizedEvent) event; - List links = container.resourceSet - .resourceLocalized(rsrcEvent.getResource(), rsrcEvent.getLocation()); + List links = container.resourceSet.getPendingResources() + .get(rsrcEvent.getResource()); + if (links == null || links.isEmpty()) { + LOG.warn("No symlinks for " + rsrcEvent.getLocation()); + return; + } // creating symlinks. for (String link : links) { - try { - String linkFile = new Path(container.workDir, link).toString(); - if (new File(linkFile).exists()) { - LOG.info("Symlink file already exists: " + linkFile); - } else { + String linkFile = new Path(container.workDir, link).toString(); + if (new File(linkFile).exists()) { + LOG.info("Symlink file already exists: " + linkFile); + } else { + String srcLocation = rsrcEvent.getLocation().toString(); + try { container.context.getContainerExecutor() - .symLink(rsrcEvent.getLocation().toString(), linkFile); - LOG.info("Created symlink: " + linkFile + " -> " + rsrcEvent - .getLocation()); + .symLink(srcLocation, linkFile, container); + LOG.info("Created symlink: [" + linkFile + " -> " + rsrcEvent + .getLocation() + "] for " + container.getContainerId()); + container.resourceSet.resourceLocalized(rsrcEvent.getResource(), + rsrcEvent.getLocation()); + } catch (IOException | PrivilegedOperationException e) { + String message = + "Error when creating symlink for " + container.getContainerId() + + ": " + srcLocation + " -> " + linkFile; + LOG.error(message, e); + container.resourceSet + .resourceLocalizationFailed(rsrcEvent.getResource()); + container.addDiagnostics(message + ": " + e.getMessage()); } - } catch (IOException e) { - String message = String - .format("Error when creating symlink %s -> %s", link, - rsrcEvent.getLocation()); - LOG.error(message, e); } } } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java index d8239ef..bc50caf 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java @@ -776,13 +776,13 @@ public static String getRelativeContainerLogDir(String appIdStr, return appIdStr + Path.SEPARATOR + containerIdStr; } - protected String getContainerPrivateDir(String appIdStr, + public static String getContainerPrivateDir(String appIdStr, String containerIdStr) { return getAppPrivateDir(appIdStr) + Path.SEPARATOR + containerIdStr + Path.SEPARATOR; } - private String getAppPrivateDir(String appIdStr) { + private static String getAppPrivateDir(String appIdStr) { return ResourceLocalizationService.NM_PRIVATE_DIR + Path.SEPARATOR + appIdStr; } @@ -811,9 +811,6 @@ public final void symlink(Path src, Path dst) throws IOException { if (!src.isAbsolute()) { throw new IOException("Source must be absolute"); } - if (dst.isAbsolute()) { - throw new IOException("Destination must be relative"); - } if (dst.toUri().getPath().indexOf('/') != -1) { mkdir(dst.getParent()); } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java index 8402a16..23a22e8 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperation.java @@ -51,7 +51,8 @@ TC_READ_STATS("--tc-read-stats"), ADD_PID_TO_CGROUP(""), //no CLI switch supported yet. RUN_DOCKER_CMD("--run-docker"), - LIST_AS_USER(""); //no CLI switch supported yet. + LIST_AS_USER(""), //no CLI switch supported yet. + RUN_SCRIPT("--run-script"); private final String option; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationException.java index 3622489..7860b11 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationException.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationException.java @@ -68,4 +68,21 @@ public String getOutput() { } public String getErrorOutput() { return errorOutput; } + + @Override + public String getMessage() { + StringBuilder message = new StringBuilder(); + if (exitCode != null) { + message.append("Exitcode: " + exitCode).append(System.lineSeparator()); + } + if (output != null) { + message.append("Output: ").append(output).append(System.lineSeparator()); + } + if (errorOutput != null) { + message.append("Error output: ").append(errorOutput) + .append(System.lineSeparator()); + } + message.append(super.getMessage()); + return message.toString(); + } } \ No newline at end of file diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c index 99e42ca..8106ddb 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c @@ -878,6 +878,8 @@ static int open_file_as_nm(const char* filename) { uid_t user = geteuid(); gid_t group = getegid(); if (change_effective_user(nm_uid, nm_gid) != 0) { + fprintf(ERRORFILE, "Failed to change effective user as NM. \n"); + fflush(ERRORFILE); return -1; } int result = open(filename, O_RDONLY); @@ -886,6 +888,8 @@ static int open_file_as_nm(const char* filename) { strerror(errno)); } if (change_effective_user(user, group)) { + fprintf(ERRORFILE, "Failed to change user to %i - %i\n", user, group); + fflush(ERRORFILE); result = -1; } return result; @@ -1132,6 +1136,55 @@ int run_docker(const char *command_file) { return exit_code; } +int run_script_as_user(const char *src_script_file, const char *dst_script_file) { + int exit_code = -1; + int src_file_fd = open_file_as_nm(src_script_file); + if (src_file_fd == -1) { + exit_code = INVALID_NM_ROOT_DIRS; + fprintf(ERRORFILE, "Could not open file %s\n", src_script_file); + fflush(ERRORFILE); + unlink(src_script_file); + return exit_code; + } + + // give up root privs + if (change_user(user_detail->pw_uid, user_detail->pw_gid) != 0) { + unlink(src_script_file); + return SETUID_OPER_FAILED; + } + + // 700 + if (copy_file(src_file_fd, src_script_file, dst_script_file, S_IRWXU) != 0) { + fprintf(ERRORFILE, "Could not copy file: %s to %s\n", src_script_file, + dst_script_file); + fflush(ERRORFILE); + exit_code = INVALID_COMMAND_PROVIDED; + return exit_code; + } + + pid_t child_pid = fork(); + if (child_pid !=0) { + int exit_code = wait_and_get_exit_code(child_pid); + unlink(dst_script_file); + if (exit_code != 0) { + fprintf(ERRORFILE, "Couldn't execute the script file: %s - %s", + dst_script_file, strerror(errno)); + fflush(ERRORFILE); + return UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + } + return 0; + } else { + if (execl(dst_script_file, dst_script_file, NULL) != 0) { + fprintf(ERRORFILE, "Couldn't execute the script file: %s - %s", + dst_script_file, strerror(errno)); + fflush(ERRORFILE); + unlink(dst_script_file); + return UNABLE_TO_EXECUTE_CONTAINER_SCRIPT; + } + } + return 0; +} + int create_script_paths(const char *work_dir, const char *script_name, const char *cred_file, char** script_file_dest, char** cred_file_dest, diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h index 1fc6fc2..0ba769b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h @@ -75,7 +75,8 @@ enum operations { RUN_AS_USER_DELETE = 9, RUN_AS_USER_LAUNCH_DOCKER_CONTAINER = 10, RUN_DOCKER = 11, - RUN_AS_USER_LIST = 12 + RUN_AS_USER_LIST = 12, + RUN_SCRIPT = 13 }; #define NM_GROUP_KEY "yarn.nodemanager.linux-container-executor.group" @@ -281,6 +282,11 @@ int traffic_control_read_stats(char *command_file); int run_docker(const char *command_file); /** + * Run a linux command. + */ +int run_script_as_user(const char *src_script_file, const char *dst_script_file); + +/** * Function to prepare the container directories. * It creates the container work and log directories. */ diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c index 3cc3244..d237393 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c @@ -50,6 +50,7 @@ static void display_usage(FILE *stream) { " container-executor --tc-read-state \n" \ " container-executor --tc-read-stats \n" \ " container-executor --run-docker \n" \ + " container-executor --run-script \n" \ " container-executor \n" \ " where command and command-args: \n" \ " initialize container: %2d appid tokens nm-local-dirs nm-log-dirs cmd app...\n" \ @@ -164,7 +165,7 @@ static struct { const char *target_dir; int container_pid; int signal; - const char *docker_command_file; + const char *command_file; } cmd_input; static int validate_run_as_user_commands(int argc, char **argv, int *operation); @@ -238,10 +239,23 @@ static int validate_arguments(int argc, char **argv , int *operation) { return INVALID_ARGUMENT_NUMBER; } optind++; - cmd_input.docker_command_file = argv[optind++]; + cmd_input.command_file = argv[optind++]; *operation = RUN_DOCKER; return 0; } + + if (strcmp("--run-script", argv[1]) == 0) { + if (argc != 5) { + display_usage(stdout); + return INVALID_ARGUMENT_NUMBER; + } + optind++; + cmd_input.run_as_user_name = argv[optind++]; + cmd_input.script_file = argv[optind++]; // script_file in nmPrivateDir + cmd_input.command_file = argv[optind++];// script_file in container workDir + *operation = RUN_SCRIPT; + return 0; + } /* Now we have to validate 'run as user' operations that don't use a 'long option' - we should fix this at some point. The validation/argument parsing here is extensive enough that it done in a separate function */ @@ -302,7 +316,7 @@ static int validate_run_as_user_commands(int argc, char **argv, int *operation) cmd_input.pid_file = argv[optind++]; cmd_input.local_dirs = argv[optind++];// good local dirs as a comma separated list cmd_input.log_dirs = argv[optind++];// good log dirs as a comma separated list - cmd_input.docker_command_file = argv[optind++]; + cmd_input.command_file = argv[optind++]; resources = argv[optind++];// key,value pair describing resources resources_key = malloc(strlen(resources)); resources_value = malloc(strlen(resources)); @@ -447,7 +461,14 @@ int main(int argc, char **argv) { exit_code = traffic_control_read_stats(cmd_input.traffic_control_command_file); break; case RUN_DOCKER: - exit_code = run_docker(cmd_input.docker_command_file); + exit_code = run_docker(cmd_input.command_file); + break; + case RUN_SCRIPT: + exit_code = set_user(cmd_input.run_as_user_name); + if (exit_code != 0) { + break; + } + exit_code = run_script_as_user(cmd_input.script_file, cmd_input.command_file); break; case RUN_AS_USER_INITIALIZE_CONTAINER: exit_code = set_user(cmd_input.run_as_user_name); @@ -486,7 +507,7 @@ int main(int argc, char **argv) { cmd_input.pid_file, extract_values(cmd_input.local_dirs), extract_values(cmd_input.log_dirs), - cmd_input.docker_command_file, + cmd_input.command_file, cmd_input.resources_key, cmd_input.resources_values); break; diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java index d24f89d..b58d6ee 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java @@ -91,7 +91,7 @@ public void deleteAsUser(DeletionAsUserContext ctx) } @Override - public void symLink(String target, String symlink) + public void symLink(String src, String symlink, Container container) throws IOException { } diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java index 7a54263..ac8920d 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java @@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider; import org.apache.hadoop.yarn.security.ContainerTokenIdentifier; import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus; +import org.apache.hadoop.yarn.server.nodemanager.Context; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState; @@ -184,4 +185,9 @@ public void setLogDir(String logDir) { public void setIpAndHost(String[] ipAndHost) { } + + @Override + public Context getNMContext() { + return null; + } }