diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java index a232c21..2e0d5bb 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java @@ -75,16 +75,18 @@ public EnqueueBean run(String user, Map userArgs, for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) { args.add("--hiveconf"); - args.add(prop); + args.add(TempletonUtils.quoteForWindows(prop)); } for (String prop : defines) { args.add("--hiveconf"); - args.add(prop); + args.add(TempletonUtils.quoteForWindows(prop)); + } + for (String hiveArg : hiveArgs) { + args.add(TempletonUtils.quoteForWindows(hiveArg)); } - args.addAll(hiveArgs); if (TempletonUtils.isset(execute)) { args.add("-e"); - args.add(execute); + args.add(TempletonUtils.quoteForWindows(execute)); } else if (TempletonUtils.isset(srcFile)) { args.add("-f"); args.add(TempletonUtils.hadoopFsPath(srcFile, appConf, runAs) diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java index b451f5e..82e16bc 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JarDelegator.java @@ -74,22 +74,27 @@ public EnqueueBean run(String user, Map userArgs, String jar, St if (TempletonUtils.isset(mainClass)) args.add(mainClass); if (TempletonUtils.isset(libjars)) { + String libjarsListAsString = + TempletonUtils.hadoopFsListAsString(libjars, appConf, runAs); args.add("-libjars"); - args.add(TempletonUtils.hadoopFsListAsString(libjars, appConf, - runAs)); + args.add(TempletonUtils.quoteForWindows(libjarsListAsString)); } if (TempletonUtils.isset(files)) { + String filesListAsString = + TempletonUtils.hadoopFsListAsString(files, appConf, runAs); args.add("-files"); - args.add(TempletonUtils.hadoopFsListAsString(files, appConf, - runAs)); + args.add(TempletonUtils.quoteForWindows(filesListAsString)); } //the token file location comes after mainClass, as a -Dprop=val args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER); - for (String d : defines) - args.add("-D" + d); - - args.addAll(jarArgs); + for (String d : defines) { + args.add("-D"); + TempletonUtils.quoteForWindows(d); + } + for (String arg : jarArgs) { + args.add(TempletonUtils.quoteForWindows(arg)); + } } catch (FileNotFoundException e) { throw new BadParam(e.getMessage()); } catch (URISyntaxException e) { diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java index ff1e2c0..2e011a4 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/PigDelegator.java @@ -78,10 +78,12 @@ public EnqueueBean run(String user, Map userArgs, //the token file location should be first argument of pig args.add("-D" + TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER); - args.addAll(pigArgs); + for (String pigArg : pigArgs) { + args.add(TempletonUtils.quoteForWindows(pigArg)); + } if (TempletonUtils.isset(execute)) { args.add("-execute"); - args.add(execute); + args.add(TempletonUtils.quoteForWindows(execute)); } else if (TempletonUtils.isset(srcFile)) { args.add("-file"); args.add(TempletonUtils.hadoopFsPath(srcFile, appConf, runAs) diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java index d58334f..024f62b 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/Server.java @@ -587,7 +587,9 @@ public EnqueueBean mapReduceStreaming(@FormParam("input") List inputs, @FormParam("output") String output, @FormParam("mapper") String mapper, @FormParam("reducer") String reducer, - @FormParam("file") List files, + @FormParam("combiner") String combiner, + @FormParam("file") List fileList, + @FormParam("files") String files, @FormParam("define") List defines, @FormParam("cmdenv") List cmdenvs, @FormParam("arg") List args, @@ -607,6 +609,8 @@ public EnqueueBean mapReduceStreaming(@FormParam("input") List inputs, userArgs.put("output", output); userArgs.put("mapper", mapper); userArgs.put("reducer", reducer); + userArgs.put("combiner", combiner); + userArgs.put("file", fileList); userArgs.put("files", files); userArgs.put("define", defines); userArgs.put("cmdenv", cmdenvs); @@ -619,8 +623,8 @@ public EnqueueBean mapReduceStreaming(@FormParam("input") List inputs, throw new BadParam("enablelog is only applicable when statusdir is set"); StreamingDelegator d = new StreamingDelegator(appConf); - return d.run(getDoAsUser(), userArgs, inputs, output, mapper, reducer, - files, defines, cmdenvs, args, + return d.run(getDoAsUser(), userArgs, inputs, output, mapper, reducer, combiner, + fileList, files, defines, cmdenvs, args, statusdir, callback, getCompletedUrl(), enablelog, JobType.STREAMING); } diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java index 5cfbbc1..65f3ec8 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/StreamingDelegator.java @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.commons.exec.ExecuteException; +import org.apache.hive.hcatalog.templeton.tool.TempletonUtils; /** * Submit a streaming job to the MapReduce queue. Really just a front @@ -38,8 +39,9 @@ public StreamingDelegator(AppConfig appConf) { public EnqueueBean run(String user, Map userArgs, List inputs, String output, - String mapper, String reducer, - List files, List defines, + String mapper, String reducer, String combiner, + List fileList, + String files, List defines, List cmdenvs, List jarArgs, String statusdir, @@ -49,13 +51,13 @@ public EnqueueBean run(String user, Map userArgs, JobType jobType) throws NotAuthorizedException, BadParam, BusyException, QueueException, ExecuteException, IOException, InterruptedException { - List args = makeArgs(inputs, output, mapper, reducer, - files, defines, cmdenvs, jarArgs); + List args = makeArgs(inputs, output, mapper, reducer, combiner, + fileList, cmdenvs, jarArgs); JarDelegator d = new JarDelegator(appConf); return d.run(user, userArgs, appConf.streamingJar(), null, - null, null, args, defines, + null, files, args, defines, statusdir, callback, completedUrl, enableLog, jobType); } @@ -63,10 +65,12 @@ public EnqueueBean run(String user, Map userArgs, String output, String mapper, String reducer, - List files, - List defines, + String combiner, + List fileList, List cmdenvs, - List jarArgs) { + List jarArgs) + throws BadParam + { ArrayList args = new ArrayList(); for (String input : inputs) { args.add("-input"); @@ -79,13 +83,24 @@ public EnqueueBean run(String user, Map userArgs, args.add("-reducer"); args.add(reducer); - for (String f : files) - args.add("-file" + f); - for (String d : defines) - args.add("-D" + d); - for (String e : cmdenvs) - args.add("-cmdenv" + e); - args.addAll(jarArgs); + if (TempletonUtils.isset(combiner)) { + args.add("-combiner"); + args.add(combiner); + } + + for (String f : fileList) { + args.add("-file"); + args.add(f); + } + + for (String e : cmdenvs) { + args.add("-cmdenv"); + args.add(TempletonUtils.quoteForWindows(e)); + } + + for (String arg : jarArgs) { + args.add(TempletonUtils.quoteForWindows(arg)); + } return args; } diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java index be43277..036a2d9 100644 --- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java +++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/TempletonUtils.java @@ -26,6 +26,7 @@ import java.net.URL; import java.net.URLConnection; import java.security.PrivilegedExceptionAction; +import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; @@ -41,8 +42,10 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.security.UserGroupInformation; +import org.apache.hadoop.util.Shell; import org.apache.hadoop.util.StringUtils; import org.apache.hive.hcatalog.templeton.UgiFactory; +import org.apache.hive.hcatalog.templeton.BadParam; /** * General utility methods. @@ -298,4 +301,46 @@ public static int fetchUrl(URL url) return env; } + + // Add double quotes around the given input parameter if it is not already + // quoted. Quotes are not allowed in the middle of the parameter, and + // BadParam exception is thrown if this is the case. + // + // This method should be used to escape parameters before they get passed to + // Windows cmd scripts (specifically, special characters like a comma or an + // equal sign might be lost as part of the cmd script processing if not + // under quotes). + public static String quoteForWindows(String param) throws BadParam { + if (Shell.WINDOWS) { + if (param != null && param.length() > 0) { + String nonQuotedPart = param; + boolean addQuotes = true; + if (param.charAt(0) == '\"' && param.charAt(param.length() - 1) == '\"') { + if (param.length() < 2) + throw new BadParam("Passed in parameter is incorrectly quoted: " + param); + + addQuotes = false; + nonQuotedPart = param.substring(1, param.length() - 1); + } + + // If we have any quotes other then the outside quotes, throw + if (nonQuotedPart.contains("\"")) { + throw new BadParam("Passed in parameter is incorrectly quoted: " + param); + } + + if (addQuotes) { + param = '\"' + param + '\"'; + } + } + } + return param; + } + + public static void addCmdForWindows(ArrayList args) { + if(Shell.WINDOWS){ + args.add("cmd"); + args.add("/c"); + args.add("call"); + } + } }