diff --git cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java index 27b8504..de62a93 100644 --- cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java +++ cli/src/java/org/apache/hadoop/hive/cli/CliDriver.java @@ -724,7 +724,8 @@ private int executeDriver(CliSessionState ss, HiveConf conf, OptionsProcessor o ClassLoader loader = conf.getClassLoader(); String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS); if (StringUtils.isNotBlank(auxJars)) { - loader = Utilities.addToClassPath(loader, StringUtils.split(auxJars, ",")); + // allow non-local paths in aux jars to be copied locally. + loader = Utilities.addToClassPath(loader, StringUtils.split(auxJars, ","), true); } conf.setClassLoader(loader); Thread.currentThread().setContextClassLoader(loader); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index cc840be..f7e9b26 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -1909,6 +1909,11 @@ public static String getResourceFiles(Configuration conf, SessionState.ResourceT * Array of classpath elements */ public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) throws Exception { + return addToClassPath(cloader, newPaths, false); + } + + public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths, + boolean downloadFiles) throws Exception { URLClassLoader loader = (URLClassLoader) cloader; List curPath = Arrays.asList(loader.getURLs()); ArrayList newPath = new ArrayList(); @@ -1920,6 +1925,15 @@ public static ClassLoader addToClassPath(ClassLoader cloader, String[] newPaths) curPath = newPath; for (String onestr : newPaths) { + if (downloadFiles && SessionState.canDownloadResource(onestr)) { + // Download the file and add the local copy to class loader + try { + onestr = SessionState.get().downloadResource(onestr, false); + } catch (Exception e) { + LOG.warn("Unable to download " + onestr + ":" + e); + } + } + // special processing for hadoop-17. file:// needs to be removed if (StringUtils.indexOf(onestr, "file://") == 0) { onestr = StringUtils.substring(onestr, 7); diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java index 288da8e..b1577bf 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/mr/ExecDriver.java @@ -711,7 +711,8 @@ public static void main(String[] args) throws IOException, HiveException { // see also - code in CliDriver.java ClassLoader loader = conf.getClassLoader(); if (StringUtils.isNotBlank(auxJars)) { - loader = Utilities.addToClassPath(loader, StringUtils.split(auxJars, ",")); + // allow non-local paths in aux jars to be copied locally. + loader = Utilities.addToClassPath(loader, StringUtils.split(auxJars, ","), true); } if (StringUtils.isNotBlank(addedJars)) { loader = Utilities.addToClassPath(loader, StringUtils.split(addedJars, ",")); diff --git ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java index 99b6d77..2a97469 100644 --- ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java +++ ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java @@ -751,7 +751,7 @@ public static boolean canDownloadResource(String value) { return (scheme != null) && !scheme.equalsIgnoreCase("file"); } - private String downloadResource(String value, boolean convertToUnix) { + public String downloadResource(String value, boolean convertToUnix) { if (canDownloadResource(value)) { getConsole().printInfo("converting to local " + value); File resourceDir = new File(getConf().getVar(HiveConf.ConfVars.DOWNLOADED_RESOURCES_DIR));