diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java index 3644761..dd6c1ad 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java @@ -38,6 +38,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.logging.Log; @@ -200,7 +201,7 @@ // Shell command to be executed private String shellCommand = ""; // Args to be passed to the shell command - private String shellArgs = ""; + private String[] shellArgs = new String[] {}; // Env variables to be setup for the shell command private Map shellEnv = new HashMap(); @@ -306,7 +307,9 @@ public boolean init(String[] args) throws ParseException, IOException { "Shell command to be executed by the Application Master"); opts.addOption("shell_script", true, "Location of the shell script to be executed"); - opts.addOption("shell_args", true, "Command line args for the shell script"); + opts.addOption("shell_args", true, "Command line args for the shell script. " + + "Multiple args can be separated by empty space."); + opts.getOption("shell_args").setArgs(Option.UNLIMITED_VALUES); opts.addOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs"); opts.addOption("container_memory", true, @@ -381,7 +384,7 @@ public boolean init(String[] args) throws ParseException, IOException { shellCommand = cliParser.getOptionValue("shell_command"); if (cliParser.hasOption("shell_args")) { - shellArgs = cliParser.getOptionValue("shell_args"); + shellArgs = cliParser.getOptionValues("shell_args"); } if (cliParser.hasOption("shell_env")) { String shellEnvs[] = cliParser.getOptionValues("shell_env"); @@ -848,7 +851,15 @@ public void run() { } // Set args for the shell command if any - vargs.add(shellArgs); + if (shellArgs.length > 0) { + StringBuilder args = new StringBuilder(); + for (String arg : shellArgs) { + args.append(arg); + args.append(" "); + } + vargs.add(args); + } + // Add log redirect params vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout"); vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 199a16d..2e9e368 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -30,6 +30,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.GnuParser; import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; import org.apache.commons.logging.Log; @@ -134,7 +135,7 @@ // Location of shell script private String shellScriptPath = ""; // Args to be passed to the shell command - private String shellArgs = ""; + private String[] shellArgs = new String[] {}; // Env variables to be setup for the shell command private Map shellEnv = new HashMap(); // Shell Command Container priority @@ -216,7 +217,9 @@ public Client(Configuration conf) throws Exception { opts.addOption("jar", true, "Jar file containing the application master"); opts.addOption("shell_command", true, "Shell command to be executed by the Application Master"); opts.addOption("shell_script", true, "Location of the shell script to be executed"); - opts.addOption("shell_args", true, "Command line args for the shell script"); + opts.addOption("shell_args", true, "Command line args for the shell script. " + + "Multiple args can be separated by empty space."); + opts.getOption("shell_args").setArgs(Option.UNLIMITED_VALUES); opts.addOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs"); opts.addOption("shell_cmd_priority", true, "Priority for the shell command containers"); opts.addOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command"); @@ -295,7 +298,7 @@ public boolean init(String[] args) throws ParseException { shellScriptPath = cliParser.getOptionValue("shell_script"); } if (cliParser.hasOption("shell_args")) { - shellArgs = cliParser.getOptionValue("shell_args"); + shellArgs = cliParser.getOptionValues("shell_args"); } if (cliParser.hasOption("shell_env")) { String envs[] = cliParser.getOptionValues("shell_env"); @@ -544,8 +547,13 @@ public boolean run() throws IOException, YarnException { if (!shellCommand.isEmpty()) { vargs.add("--shell_command " + shellCommand + ""); } - if (!shellArgs.isEmpty()) { - vargs.add("--shell_args " + shellArgs + ""); + if (shellArgs.length > 0) { + StringBuilder args = new StringBuilder(); + for (String arg : shellArgs) { + args.append(arg); + args.append(" "); + } + vargs.add("--shell_args " + args + ""); } for (Map.Entry entry : shellEnv.entrySet()) { vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 2f311b5..e444b1b 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -18,12 +18,15 @@ package org.apache.hadoop.yarn.applications.distributedshell; +import java.io.BufferedReader; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.FileOutputStream; +import java.io.FileReader; import java.io.IOException; import java.io.OutputStream; import java.net.URL; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; @@ -171,6 +174,43 @@ public void run() { } @Test(timeout=90000) + public void testDSShellWithMultipleArgs() throws Exception { + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "3", + "--shell_command", + "echo", + "--shell_args", + "HADOOP YARN MAPREDUCE HDFS", + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1" + }; + + LOG.info("Initializing DS Client"); + final Client client = + new Client(new Configuration(yarnCluster.getConfig())); + boolean initSuccess = client.init(args); + Assert.assertTrue(initSuccess); + LOG.info("Running DS Client"); + boolean result = client.run(); + LOG.info("Client run completed. Result=" + result); + + List> containerLogs = readContainerLog(3); + for (List logs : containerLogs) { + Assert + .assertTrue(logs.get(0).trim().equals("HADOOP YARN MAPREDUCE HDFS")); + } + } + + @Test(timeout=90000) public void testDSShellWithInvalidArgs() throws Exception { Client client = new Client(new Configuration(yarnCluster.getConfig())); @@ -306,5 +346,56 @@ public void testContainerLaunchFailureHandling() throws Exception { } + private List> readContainerLog(int containerNum) { + File logFolder = + new File(yarnCluster.getNodeManager(0).getConfig() + .get(YarnConfiguration.NM_LOG_DIRS, + YarnConfiguration.DEFAULT_NM_LOG_DIRS)); + + File[] listOfFiles = logFolder.listFiles(); + int currentContainerLogFileIndex = -1; + for (int i = 0; i < listOfFiles.length; i++) { + if (listOfFiles[i].listFiles().length == containerNum + 1) { + currentContainerLogFileIndex = i; + break; + } + } + Assert.assertTrue(currentContainerLogFileIndex != -1); + File[] containerFiles = + listOfFiles[currentContainerLogFileIndex].listFiles(); + + List> outputList = new ArrayList>(); + for (int i = 1; i < containerFiles.length; i++) { + for (File output : containerFiles[i].listFiles()) { + if (output.getName().contains("stdout")) { + BufferedReader br = null; + List outStrings = new ArrayList(); + try { + + String sCurrentLine; + + br = new BufferedReader(new FileReader(output)); + + while ((sCurrentLine = br.readLine()) != null) { + outStrings.add(sCurrentLine); + System.out.println(sCurrentLine); + } + + } catch (IOException e) { + e.printStackTrace(); + } finally { + try { + if (br != null) + br.close(); + } catch (IOException ex) { + ex.printStackTrace(); + } + } + outputList.add(outStrings); + } + } + } + return outputList; + } }