diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 199a16d..0094ac8 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -214,8 +214,16 @@ public Client(Configuration conf) throws Exception { opts.addOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"); opts.addOption("master_vcores", true, "Amount of virtual cores to be requested to run the application master"); opts.addOption("jar", true, "Jar file containing the application master"); - opts.addOption("shell_command", true, "Shell command to be executed by the Application Master"); - opts.addOption("shell_script", true, "Location of the shell script to be executed"); + opts.addOption("shell_command", true, + "Shell command to be executed by the Application Master. " + + "Does not support multiple --shell_command options, " + + "multiple shell commands and command pipline. " + + "For multiple shell commands or command pipeline, " + + "please create a shell script and use --shell_script option"); + opts.addOption("shell_script", true, "Location of the shell script to " + + "be executed. Support only one --shell_script option. " + + "For multiple shell scripts, combine them " + + "into one shell script"); opts.addOption("shell_args", true, "Command line args for the shell script"); opts.addOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs"); opts.addOption("shell_cmd_priority", true, "Priority for the shell command containers"); @@ -249,6 +257,17 @@ private void printUsage() { */ public boolean init(String[] args) throws ParseException { + if (numOfTargetOpts(args, "--shell_command") > 1) { + throw new IllegalArgumentException("DistributedShell does not support " + + "multiple shell commands. Please create a shell script " + + "and use --shell_script option."); + } + + if (numOfTargetOpts(args, "--shell_script") > 1) { + throw new IllegalArgumentException("DistributedShell does not support " + + "multiple shell scripts. Please combine them into one shell script."); + } + CommandLine cliParser = new GnuParser().parse(opts, args); if (args.length == 0) { @@ -291,6 +310,12 @@ public boolean init(String[] args) throws ParseException { } shellCommand = cliParser.getOptionValue("shell_command"); + if (shellCommand.contains(";") || shellCommand.contains("|")) { + throw new IllegalArgumentException( + "DistributedShell does not support multiple commands " + + "or command pipeline. Please create a shell script for " + + "them and use --shell_script option"); + } if (cliParser.hasOption("shell_script")) { shellScriptPath = cliParser.getOptionValue("shell_script"); } @@ -715,4 +740,13 @@ private void forceKillApplication(ApplicationId appId) yarnClient.killApplication(appId); } + private int numOfTargetOpts(String[] args, String targetArgs) { + int num = 0; + for (String arg : args) { + if (arg.equals(targetArgs)) { + num ++; + } + } + return num; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 2f311b5..c2ab4d0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -266,6 +266,121 @@ public void testDSShellWithInvalidArgs() throws Exception { Assert.assertTrue("The throw exception is not expected", e.getMessage().contains("Invalid virtual cores specified")); } + + LOG.info("Initializing DS Client with multiple --shell_command options"); + try { + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "2", + "--shell_command", + Shell.WINDOWS ? "dir" : "ls", + "--shell_command", + "pwd", + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1" + }; + client.init(args); + Assert.fail("Exception is expected"); + } catch (IllegalArgumentException e) { + Assert.assertTrue("The throw exception is not expected", + e.getMessage().contains("DistributedShell does not support " + + "multiple shell commands. Please create a shell script " + + "and use --shell_script option.")); + } + + LOG.info("Initializing DS Client with multiple --shell_script options"); + try { + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "2", + "--shell_command", + Shell.WINDOWS ? "dir" : "ls", + "--shell_script", + "/tmp/script1.sh", + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1", + "--shell_script", + "/tmp/script2.sh" + }; + client.init(args); + Assert.fail("Exception is expected"); + } catch (IllegalArgumentException e) { + Assert.assertTrue("The throw exception is not expected", + e.getMessage().contains("DistributedShell does not support " + + "multiple shell scripts. Please combine them into " + + "one shell script.")); + } + + LOG.info("Initializing DS Client with shell commands separated by ';'"); + try { + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "2", + "--shell_command", + "ls;pwd", + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1" + }; + client.init(args); + Assert.fail("Exception is expected"); + } catch (IllegalArgumentException e) { + Assert.assertTrue("The throw exception is not expected", + e.getMessage().contains("DistributedShell does not support multiple " + + "commands or command pipeline. Please create a shell script for " + + "them and use --shell_script option")); + } + + LOG.info("Initializing DS Client with shell commands piped"); + try { + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "2", + "--shell_command", + "ls|wc", + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1" + }; + client.init(args); + Assert.fail("Exception is expected"); + } catch (IllegalArgumentException e) { + Assert.assertTrue("The throw exception is not expected", + e.getMessage().contains("DistributedShell does not support multiple " + + "commands or command pipeline. Please create a shell script for " + + "them and use --shell_script option")); + } + } protected static void waitForNMToRegister(NodeManager nm)