diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JarDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JarDelegator.java index a2dc23e..fed49ae 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JarDelegator.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/JarDelegator.java @@ -71,20 +71,24 @@ public EnqueueBean run(String user, String jar, String mainClass, if (TempletonUtils.isset(mainClass)) args.add(mainClass); if (TempletonUtils.isset(libjars)) { + String libjarsListAsString = + TempletonUtils.hadoopFsListAsString(libjars, appConf, runAs); args.add("-libjars"); - args.add(TempletonUtils.hadoopFsListAsString(libjars, appConf, - runAs)); + args.add(TempletonUtils.quoteForWindowsCmdPrompt(libjarsListAsString)); } if (TempletonUtils.isset(files)) { + String filesListAsString = + TempletonUtils.hadoopFsListAsString(files, appConf, runAs); args.add("-files"); - args.add(TempletonUtils.hadoopFsListAsString(files, appConf, - runAs)); + args.add(TempletonUtils.quoteForWindowsCmdPrompt(filesListAsString)); } //the token file location comes after mainClass, as a -Dprop=val args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER); - for (String d : defines) - args.add("-D" + d); + for (String d : defines) { + args.add("-D"); + args.add(TempletonUtils.quoteForWindowsCmdPrompt(d)); + } args.addAll(jarArgs); } catch (FileNotFoundException e) { diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java index 29ac4b3..f89bd8a 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/Server.java @@ -579,7 +579,9 @@ public EnqueueBean mapReduceStreaming(@FormParam("input") List inputs, @FormParam("output") String output, @FormParam("mapper") String mapper, @FormParam("reducer") String reducer, - @FormParam("file") List files, + @FormParam("combiner") String combiner, + @FormParam("file") List fileList, + @FormParam("files") String files, @FormParam("define") List defines, @FormParam("cmdenv") List cmdenvs, @FormParam("arg") List args, @@ -593,8 +595,8 @@ public EnqueueBean mapReduceStreaming(@FormParam("input") List inputs, verifyParam(reducer, "reducer"); StreamingDelegator d = new StreamingDelegator(appConf); - return d.run(getUser(), inputs, output, mapper, reducer, - files, defines, cmdenvs, args, + return d.run(getUser(), inputs, output, mapper, reducer, combiner, + fileList, files, defines, cmdenvs, args, statusdir, callback, getCompletedUrl()); } diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StreamingDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StreamingDelegator.java index 85557ba..d15ee9a 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StreamingDelegator.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/StreamingDelegator.java @@ -23,6 +23,7 @@ import java.util.List; import org.apache.commons.exec.ExecuteException; +import org.apache.hcatalog.templeton.tool.TempletonUtils; /** * Submit a streaming job to the MapReduce queue. Really just a front @@ -37,8 +38,10 @@ public StreamingDelegator(AppConfig appConf) { public EnqueueBean run(String user, List inputs, String output, - String mapper, String reducer, - List files, List defines, + String mapper, String reducer, String combiner, + List fileList, + String files, + List defines, List cmdenvs, List jarArgs, String statusdir, @@ -46,13 +49,13 @@ public EnqueueBean run(String user, String completedUrl) throws NotAuthorizedException, BadParam, BusyException, QueueException, ExecuteException, IOException, InterruptedException { - List args = makeArgs(inputs, output, mapper, reducer, - files, defines, cmdenvs, jarArgs); + List args = makeArgs(inputs, output, mapper, reducer, combiner, + fileList, cmdenvs, jarArgs); JarDelegator d = new JarDelegator(appConf); return d.run(user, appConf.streamingJar(), null, - null, null, args, defines, + null, files, args, defines, statusdir, callback, completedUrl); } @@ -60,10 +63,10 @@ public EnqueueBean run(String user, String output, String mapper, String reducer, - List files, - List defines, + String combiner, + List fileList, List cmdenvs, - List jarArgs) { + List jarArgs) throws BadParam { ArrayList args = new ArrayList(); for (String input : inputs) { args.add("-input"); @@ -76,12 +79,20 @@ public EnqueueBean run(String user, args.add("-reducer"); args.add(reducer); - for (String f : files) - args.add("-file" + f); - for (String d : defines) - args.add("-D" + d); - for (String e : cmdenvs) - args.add("-cmdenv" + e); + if (TempletonUtils.isset(combiner)) { + args.add("-combiner"); + args.add(combiner); + } + + for (String f : fileList) { + args.add("-file"); + args.add(f); + } + + for (String e : cmdenvs) { + args.add("-cmdenv"); + args.add(TempletonUtils.quoteForWindowsCmdPrompt(e)); + } args.addAll(jarArgs); return args; diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java index af4e1cf..b6fdac0 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hcatalog/templeton/tool/TempletonUtils.java @@ -37,7 +37,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; +import org.apache.hcatalog.templeton.BadParam; /** * General utility methods. @@ -274,4 +276,38 @@ public static int fetchUrl(URL url) return env; } + + // Add double quotes around the given input parameter if it is not already + // quoted. Quotes are not allowed in the middle of the parameter, and + // BadParam exception is thrown if this is the case. + // + // This method should be used to escape parameters before they get passed to + // Windows cmd scripts (specifically, special characters like a comma or an + // equal sign might be lost as part of the cmd script processing if not + // under quotes). + public static String quoteForWindowsCmdPrompt(String param) throws BadParam { + if (Shell.WINDOWS) { + if (param != null && param.length() > 0) { + String nonQuotedPart = param; + boolean addQuotes = true; + if (param.charAt(0) == '\"' && param.charAt(param.length() - 1) == '\"') { + if (param.length() < 2) + throw new BadParam("Passed in parameter is incorrectly quoted: " + param); + + addQuotes = false; + nonQuotedPart = param.substring(1, param.length() - 1); + } + + // If we have any quotes other then the outside quotes, throw + if (nonQuotedPart.contains("\"")) { + throw new BadParam("Passed in parameter is incorrectly quoted: " + param); + } + + if (addQuotes) { + param = '\"' + param + '\"'; + } + } + } + return param; + } } diff --git hcatalog/webhcat/svr/src/test/java/org/apache/hcatalog/templeton/tool/TestTempletonUtils.java hcatalog/webhcat/svr/src/test/java/org/apache/hcatalog/templeton/tool/TestTempletonUtils.java index 4f23e0a..95b5066 100644 --- hcatalog/webhcat/svr/src/test/java/org/apache/hcatalog/templeton/tool/TestTempletonUtils.java +++ hcatalog/webhcat/svr/src/test/java/org/apache/hcatalog/templeton/tool/TestTempletonUtils.java @@ -24,6 +24,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.util.StringUtils; +import org.apache.hcatalog.templeton.BadParam; import org.junit.Test; public class TestTempletonUtils { @@ -204,4 +205,33 @@ public void testHadoopFsListAsString() { } } + @Test + public void testQuoteForWindows() throws Exception, BadParam { + Assert.assertEquals(null, TempletonUtils.quoteForWindowsCmdPrompt(null)); + Assert.assertEquals("", TempletonUtils.quoteForWindowsCmdPrompt("")); + Assert.assertEquals("\"test\"", TempletonUtils.quoteForWindowsCmdPrompt("test")); + Assert.assertEquals("\"test with spaces\"", TempletonUtils.quoteForWindowsCmdPrompt("test with spaces")); + Assert.assertEquals("\"already quoted\"", TempletonUtils.quoteForWindowsCmdPrompt("\"already quoted\"")); + Assert.assertEquals("\"\"", TempletonUtils.quoteForWindowsCmdPrompt("\"\"")); + + try { + Assert.assertEquals(null, TempletonUtils.quoteForWindowsCmdPrompt("\"")); + Assert.assertTrue("Should have thrown", false); + } catch (BadParam e) {} + + try { + Assert.assertEquals(null, TempletonUtils.quoteForWindowsCmdPrompt("\"test")); + Assert.assertTrue("Should have thrown", false); + } catch (BadParam e) {} + + try { + Assert.assertEquals(null, TempletonUtils.quoteForWindowsCmdPrompt("test\"")); + Assert.assertTrue("Should have thrown", false); + } catch (BadParam e) {} + + try { + Assert.assertEquals(null, TempletonUtils.quoteForWindowsCmdPrompt("middle\"quote")); + Assert.assertTrue("Should have thrown", false); + } catch (BadParam e) {} + } }