diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java index cf81424..c10205d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/RemoteHiveSparkClient.java @@ -297,9 +297,10 @@ public Serializable call(JobContext jc) throws Exception { // may need to load classes from this jar in other threads. Map addedJars = jc.getAddedJars(); if (addedJars != null && !addedJars.isEmpty()) { - SparkClientUtilities.addToClassPath(addedJars, localJobConf, jc.getLocalTmpDir()); + List localAddedJars = SparkClientUtilities.addToClassPath(addedJars, + localJobConf, jc.getLocalTmpDir()); KryoSerializer.setClassLoader(Thread.currentThread().getContextClassLoader()); - localJobConf.set(Utilities.HIVE_ADDED_JARS, StringUtils.join(addedJars.keySet(), ";")); + localJobConf.set(Utilities.HIVE_ADDED_JARS, StringUtils.join(localAddedJars, ";")); } Path localScratchDir = KryoSerializer.deserialize(scratchDirBytes, Path.class); diff --git a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java index bbbd97b..ef5d1b3 100644 --- a/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java +++ b/spark-client/src/main/java/org/apache/hive/spark/client/SparkClientUtilities.java @@ -23,6 +23,7 @@ import java.io.File; import java.net.URL; import java.net.URLClassLoader; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -42,15 +43,18 @@ * Add new elements to the classpath. * * @param newPaths Map of classpath elements and corresponding timestamp + * @return locally accessible files corresponding to the newPaths */ - public static void addToClassPath(Map newPaths, Configuration conf, File localTmpDir) - throws Exception { + public static List addToClassPath(Map newPaths, Configuration conf, + File localTmpDir) throws Exception { URLClassLoader loader = (URLClassLoader) Thread.currentThread().getContextClassLoader(); List curPath = Lists.newArrayList(loader.getURLs()); + List localNewPaths = new ArrayList<>(); boolean newPathAdded = false; for (Map.Entry entry : newPaths.entrySet()) { URL newUrl = urlFromPathString(entry.getKey(), entry.getValue(), conf, localTmpDir); + localNewPaths.add(newUrl.toString()); if (newUrl != null && !curPath.contains(newUrl)) { curPath.add(newUrl); LOG.info("Added jar[" + newUrl + "] to classpath."); @@ -63,6 +67,7 @@ public static void addToClassPath(Map newPaths, Configuration conf new URLClassLoader(curPath.toArray(new URL[curPath.size()]), loader); Thread.currentThread().setContextClassLoader(newLoader); } + return localNewPaths; } /**