diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java index a9e13c6..5190c54 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/HiveDelegator.java @@ -27,6 +27,7 @@ import java.util.Map; import org.apache.commons.exec.ExecuteException; +import org.apache.hive.hcatalog.templeton.tool.JobSubmissionConstants; import org.apache.hive.hcatalog.templeton.tool.TempletonControllerJob; import org.apache.hive.hcatalog.templeton.tool.TempletonUtils; @@ -76,6 +77,11 @@ public EnqueueBean run(String user, Map userArgs, args.add("--hiveconf"); args.add(TempletonControllerJob.TOKEN_FILE_ARG_PLACEHOLDER); + //this is needed specifcally for Hive on Tez (in addition to + //JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER) + args.add("--hiveconf"); + args.add(JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ); + for (String prop : appConf.getStrings(AppConfig.HIVE_PROPS_NAME)) { args.add("--hiveconf"); args.add(TempletonUtils.quoteForWindows(prop)); diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java index 482e993..d83894f 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/JobSubmissionConstants.java @@ -34,6 +34,7 @@ public static final int WATCHER_TIMEOUT_SECS = 10; public static final int KEEP_ALIVE_MSEC = 60 * 1000; public static final String TOKEN_FILE_ARG_PLACEHOLDER = "__WEBHCAT_TOKEN_FILE_LOCATION__"; + public static final String TOKEN_FILE_ARG_PLACEHOLDER_TEZ = "__WEBHCAT_TOKEN_FILE_LOCATION_TEZ__"; /** * constants needed for Pig job submission */ diff --git hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java index 100a504..1616306 100644 --- hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java +++ hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/tool/LaunchMapper.java @@ -100,41 +100,47 @@ protected Process startJob(Context context, String user, String overrideClasspat Map env = TempletonUtils.hadoopUserEnv(user, overrideClasspath); handlePigEnvVars(conf, env); List jarArgsList = new LinkedList(Arrays.asList(jarArgs)); - String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION"); - + handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER, "mapreduce.job.credentials.binary"); + handleTokenFile(jarArgsList, JobSubmissionConstants.TOKEN_FILE_ARG_PLACEHOLDER_TEZ, "tez.credentials.path"); + boolean overrideLog4jProps = conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS) == null ? + false : Boolean.valueOf(conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS)); + return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env, overrideLog4jProps); + } + /** + * Replace placeholder with actual "prop=file". This is done multiple times (possibly) since + * Tez and MR use different property names + */ + private static void handleTokenFile(List jarArgsList, String tokenPlaceHolder, String tokenProperty) throws IOException { + String tokenFile = System.getenv("HADOOP_TOKEN_FILE_LOCATION"); if (tokenFile != null) { //Token is available, so replace the placeholder tokenFile = tokenFile.replaceAll("\"", ""); - String tokenArg = "mapreduce.job.credentials.binary=" + tokenFile; + String tokenArg = tokenProperty + "=" + tokenFile; if (Shell.WINDOWS) { try { tokenArg = TempletonUtils.quoteForWindows(tokenArg); } catch (BadParam e) { - String msg = "cannot pass " + tokenFile + " to mapreduce.job.credentials.binary"; + String msg = "cannot pass " + tokenFile + " to " + tokenProperty; LOG.error(msg, e); throw new IOException(msg, e); } } for(int i=0; i it = jarArgsList.iterator(); while(it.hasNext()){ String arg = it.next(); - if(arg.contains(TOKEN_FILE_ARG_PLACEHOLDER)){ + if(arg.contains(tokenPlaceHolder)){ it.remove(); } } } - boolean overrideLog4jProps = conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS) == null ? - false : Boolean.valueOf(conf.get(OVERRIDE_CONTAINER_LOG4J_PROPS)); - return TrivialExecService.getInstance().run(jarArgsList, removeEnv, env, overrideLog4jProps); } private void copyLocal(String var, Configuration conf) throws IOException {