diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 02fc5e9..e34cc09 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -29,6 +29,7 @@ import javax.security.auth.login.LoginException; import org.apache.commons.lang.StringUtils; +import org.apache.commons.io.FilenameUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -203,11 +204,11 @@ private static Vertex createVertex(JobConf conf, MapWork mapWork, int seqNo, if (localWorkLr != null) { localResources.put(hashTableArchive.getName(), localWorkLr); } - localResources.put(appJarLr.getResource().getFile(), appJarLr); + localResources.put(getBaseName(appJarLr), appJarLr); for (LocalResource lr: additionalLr) { - localResources.put(lr.getResource().getFile(), lr); + localResources.put(getBaseName(lr), lr); } - localResources.put(planPath.getName(), planLr); + localResources.put(FilenameUtils.getName(planPath.getName()), planLr); MRHelpers.updateLocalResourcesForInputSplits(FileSystem.get(conf), inputSplitInfo, localResources); @@ -272,11 +273,11 @@ private static Vertex createVertex(JobConf conf, ReduceWork reduceWork, int seqN reducer.setJavaOpts(MRHelpers.getReduceJavaOpts(conf)); Map localResources = new HashMap(); - localResources.put(appJarLr.getResource().getFile(), appJarLr); + localResources.put(getBaseName(appJarLr), appJarLr); for (LocalResource lr: additionalLr) { - localResources.put(lr.getResource().getFile(), lr); + localResources.put(getBaseName(lr), lr); } - localResources.put(planPath.getName(), planLr); + localResources.put(FilenameUtils.getName(planPath.getName()), planLr); reducer.setTaskLocalResources(localResources); return reducer; @@ -380,7 +381,13 @@ private static String getExecJarPathLocal () { return null; } - + + /* + * Helper function to retrieve the basename of a local resource + */ + public static String getBaseName(LocalResource lr) { + return FilenameUtils.getName(lr.getResource().getFile()); + } /** * @param pathStr - the string from which we try to determine the version @@ -485,7 +492,6 @@ public static JobConf createConfiguration(HiveConf hiveConf) throws IOException hiveConf.setBoolean("mapred.mapper.new-api", false); JobConf conf = (JobConf) MRHelpers.getBaseMRConfiguration(); - MRHelpers.doJobClientMagic(conf); for (Map.Entry entry: hiveConf) { if (conf.get(entry.getKey()) == null) { diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java index c0dd791..1a36b4b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java @@ -153,7 +153,7 @@ private DAGClient submit(JobConf conf, DAG dag, Path scratchDir, LocalResource a // setup local resources used by application master Map amLrs = new HashMap(); - amLrs.put(appJarLr.getResource().getFile(), appJarLr); + amLrs.put(DagUtils.getBaseName(appJarLr), appJarLr); // ready to start execution on the cluster DAGClient dagClient = tezClient.submitDAGApplication(dag, scratchDir,