diff --git hcatalog/webhcat/svr/src/main/config/webhcat-default.xml hcatalog/webhcat/svr/src/main/config/webhcat-default.xml index ee39667..2de1ad1 100644 --- hcatalog/webhcat/svr/src/main/config/webhcat-default.xml +++ hcatalog/webhcat/svr/src/main/config/webhcat-default.xml @@ -143,7 +143,8 @@ templeton.hive.properties hive.metastore.uris=thrift://localhost:9933,hive.metastore.sasl.enabled=false - Properties to set when running hive. + Properties to set when running hive. This is expected to be a comma-separated prop=value list. + If some value is itself a comma-separated list the escape character is '\' diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java index e28c81a..b58b1c8 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/AppConfig.java @@ -21,6 +21,8 @@ import java.io.File; import java.net.URL; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Comparator; import java.util.Iterator; @@ -31,6 +33,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.util.StringUtils; import org.apache.hadoop.util.VersionInfo; import org.apache.hive.hcatalog.templeton.tool.JobState; import org.apache.hive.hcatalog.templeton.tool.ZooKeeperCleanup; @@ -104,6 +107,11 @@ * see webhcat-default.xml */ public static final String HCAT_HOME_PATH = "templeton.hcat.home"; + /** + * is a comma separated list of name=value pairs; + * In case some value is itself a comma-separated list, the comma needs to + * be escaped with {@link org.apache.hadoop.util.StringUtils#ESCAPE_CHAR} + */ public static final String HIVE_PROPS_NAME = "templeton.hive.properties"; public static final String SQOOP_ARCHIVE_NAME = "templeton.sqoop.archive"; public static final String SQOOP_PATH_NAME = "templeton.sqoop.path"; @@ -251,8 +259,17 @@ private boolean loadOneClasspathConfig(String fname) { public String controllerMRChildOpts() { return get(TEMPLETON_CONTROLLER_MR_CHILD_OPTS); } - - + /** + * @see #HIVE_PROPS_NAME + */ + public Collection hiveProps() { + String[] props= StringUtils.split(get(HIVE_PROPS_NAME)); + final String escStr = String .valueOf(StringUtils.ESCAPE_CHAR); + for(int i = 0; i < props.length; i++) { + props[i] = props[i].replace(escStr, ""); + } + return Arrays.asList(props); + } public String[] overrideJars() { if (getBoolean(OVERRIDE_JARS_ENABLED, true)) diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java index 5190c54..23b1c4f 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java @@ -82,7 +82,7 @@ public EnqueueBean run(String user, Map userArgs, args.add("--hiveconf"); args.add(JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ); - for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) { + for (String prop : appConf.hiveProps()) { args.add("--hiveconf"); args.add(TempletonUtils.quoteForWindows(prop)); } diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java index 1a0aaac..5f6c18c 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java @@ -45,6 +45,7 @@ "__TEZ_CREDENTIALS_OPTION=WEBHCAT_TOKEN_FILE_LOCATION_TEZ__"; /** * constants needed for Pig job submission + * The string values here are what Pig expects to see in it's environment */ public static interface PigConstants { public static final String HIVE_HOME = "HIVE_HOME"; diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java index 1616306..3366caa 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java @@ -29,6 +29,7 @@ import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.util.Shell; +import org.apache.hadoop.util.StringUtils; import org.apache.hive.hcatalog.templeton.BadParam; import org.apache.hive.hcatalog.templeton.LauncherDelegator; @@ -79,8 +80,9 @@ private static void handlePigEnvVars(Configuration conf, Map env } if(conf.get(PigConstants.PIG_OPTS) != null) { StringBuilder pigOpts = new StringBuilder(); - for(String prop : conf.get(PigConstants.PIG_OPTS).split(",")) { - pigOpts.append("-D").append(prop).append(" "); + final String escStr = String.valueOf(StringUtils.ESCAPE_CHAR); + for(String prop : StringUtils.split(conf.get(PigConstants.PIG_OPTS))) { + pigOpts.append("-D").append(prop.replace(escStr, "")).append(" "); } env.put(PigConstants.PIG_OPTS, pigOpts.toString()); }