diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java index 199a16d..4e643c0 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java @@ -214,8 +214,14 @@ public Client(Configuration conf) throws Exception { opts.addOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"); opts.addOption("master_vcores", true, "Amount of virtual cores to be requested to run the application master"); opts.addOption("jar", true, "Jar file containing the application master"); - opts.addOption("shell_command", true, "Shell command to be executed by the Application Master"); - opts.addOption("shell_script", true, "Location of the shell script to be executed"); + opts.addOption("shell_command", true, + "Shell command to be executed by the Application Master. " + + "Does not support multiple --shell_command options, " + + "multiple shell commands and command pipeline. " + + "For multiple shell commands or command pipeline, " + + "please create a shell script and use --shell_script option"); + opts.addOption("shell_script", true, "Location of the shell script to " + + "be executed. Support only one --shell_script option."); opts.addOption("shell_args", true, "Command line args for the shell script"); opts.addOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs"); opts.addOption("shell_cmd_priority", true, "Priority for the shell command containers"); @@ -249,6 +255,17 @@ private void printUsage() { */ public boolean init(String[] args) throws ParseException { + if (numOfTargetOpts(args, "--shell_command") > 1) { + throw new IllegalArgumentException("DistributedShell does not support " + + "multiple shell commands. Please create a shell script " + + "and use --shell_script option."); + } + + if (numOfTargetOpts(args, "--shell_script") > 1) { + throw new IllegalArgumentException("DistributedShell does not support " + + "multiple shell scripts."); + } + CommandLine cliParser = new GnuParser().parse(opts, args); if (args.length == 0) { @@ -291,6 +308,19 @@ public boolean init(String[] args) throws ParseException { } shellCommand = cliParser.getOptionValue("shell_command"); + if (shellCommand.contains(";") || shellCommand.contains("|") + || shellCommand.contains("&")) { + throw new IllegalArgumentException( + "DistributedShell does not support multiple commands " + + "or command pipeline. Please create a shell script for " + + "them and use --shell_script option"); + } + if (shellCommand.contains(">")) { + throw new IllegalArgumentException( + "Please create a shell script for redirected output " + + "and use --shell_script option"); + } + if (cliParser.hasOption("shell_script")) { shellScriptPath = cliParser.getOptionValue("shell_script"); } @@ -715,4 +745,13 @@ private void forceKillApplication(ApplicationId appId) yarnClient.killApplication(appId); } + private int numOfTargetOpts(String[] args, String targetArgs) { + int num = 0; + for (String arg : args) { + if (arg.equals(targetArgs)) { + num ++; + } + } + return num; + } } diff --git hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java index 2f311b5..5ae356a 100644 --- hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java +++ hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java @@ -266,6 +266,146 @@ public void testDSShellWithInvalidArgs() throws Exception { Assert.assertTrue("The throw exception is not expected", e.getMessage().contains("Invalid virtual cores specified")); } + + LOG.info("Initializing DS Client with multiple --shell_command options"); + try { + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "2", + "--shell_command", + Shell.WINDOWS ? "dir" : "ls", + "--shell_command", + "pwd", + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1" + }; + client.init(args); + Assert.fail("Exception is expected"); + } catch (IllegalArgumentException e) { + Assert.assertTrue("The throw exception is not expected", + e.getMessage().contains("DistributedShell does not support " + + "multiple shell commands. Please create a shell script " + + "and use --shell_script option.")); + } + + LOG.info("Initializing DS Client with multiple --shell_script options"); + try { + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "2", + "--shell_command", + Shell.WINDOWS ? "dir" : "ls", + "--shell_script", + "/tmp/script1.sh", + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1", + "--shell_script", + "/tmp/script2.sh" + }; + client.init(args); + Assert.fail("Exception is expected"); + } catch (IllegalArgumentException e) { + Assert.assertTrue("The throw exception is not expected", + e.getMessage().contains("DistributedShell does not support " + + "multiple shell scripts.")); + } + + LOG.info("Initializing DS Client with multiple shell commands"); + try { + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "2", + "--shell_command", + Shell.WINDOWS ? "ls&&pwd" : "ls;pwd", + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1" + }; + client.init(args); + Assert.fail("Exception is expected"); + } catch (IllegalArgumentException e) { + Assert.assertTrue("The throw exception is not expected", + e.getMessage().contains("DistributedShell does not support multiple " + + "commands or command pipeline. Please create a shell script for " + + "them and use --shell_script option")); + } + + LOG.info("Initializing DS Client with shell commands piped"); + try { + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "2", + "--shell_command", + "ls|wc", + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1" + }; + client.init(args); + Assert.fail("Exception is expected"); + } catch (IllegalArgumentException e) { + Assert.assertTrue("The throw exception is not expected", + e.getMessage().contains("DistributedShell does not support multiple " + + "commands or command pipeline. Please create a shell script for " + + "them and use --shell_script option")); + } + + LOG.info("Initializing DS Client with redirected output"); + try { + String[] args = { + "--jar", + APPMASTER_JAR, + "--num_containers", + "2", + "--shell_command", + "pwd>tmp.out", + "--master_memory", + "512", + "--master_vcores", + "2", + "--container_memory", + "128", + "--container_vcores", + "1" + }; + client.init(args); + Assert.fail("Exception is expected"); + } catch (IllegalArgumentException e) { + Assert.assertTrue("The throw exception is not expected", + e.getMessage().contains("Please create a shell script for " + + "redirected output and use --shell_script option")); + } + } protected static void waitForNMToRegister(NodeManager nm)