diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 2655464..3e3b779 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -787,7 +787,7 @@ HIVE_OPTIMIZE_TEZ("hive.optimize.tez", false), HIVE_JAR_DIRECTORY("hive.jar.directory", "hdfs:///apps/hive/install/"), - HIVE_INSTALL_DIR("hive.user.install.directory", "hdfs:///users/"), + HIVE_INSTALL_DIR("hive.user.install.directory", "hdfs:///user/"), ; public final String varname; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 805af36..e70d05c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.exec.tez; +import java.io.FileNotFoundException; import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; @@ -314,9 +315,10 @@ private static LocalResource createLocalResource(FileSystem remoteFs, Path file, * * @param conf * @return List local resources to add to execution - * @throws IOException + * @throws IOException when hdfs operation fails + * @throws LoginException when user cannot be determined */ - public static List localizeTempFiles(Configuration conf) throws IOException { + public static List localizeTempFiles(Configuration conf) throws IOException, LoginException { List tmpResources = new ArrayList(); String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS); @@ -334,12 +336,39 @@ private static LocalResource createLocalResource(FileSystem remoteFs, Path file, throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(hdfsDirPathStr)); } + FileStatus fstatus = null; + try { + fstatus = fs.getFileStatus(hdfsDirPath); + } catch (FileNotFoundException fe) { + // do nothing + } + + System.out.println("jar path str: " + hdfsDirPathStr); + if ((fstatus == null) || (!fstatus.isDir())) { + System.out.println("Fstatus is null or is not a dir"); + UserGroupInformation ugi = ShimLoader.getHadoopShims().getUGIForConf(conf); + String userName = ShimLoader.getHadoopShims().getShortUserName(ugi); + String userPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_INSTALL_DIR); + Path userPath = new Path(userPathStr); + fs = userPath.getFileSystem(conf); + if (!(fs instanceof DistributedFileSystem)) { + throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(userPathStr)); + } + + String jarPathStr = userPathStr + "/" + userName; + System.out.println("hdfsDirPath: " + jarPathStr); + hdfsDirPathStr = jarPathStr; + hdfsDirPath = new Path(hdfsDirPathStr); + } + String allFiles = auxJars + "," + addedJars + "," + addedFiles + "," + addedArchives; String[] allFilesArr = allFiles.split(","); + System.out.println("All files: " + allFiles + " number of files: " + allFilesArr.length); for (String file : allFilesArr) { fs.copyFromLocalFile(new Path(file), hdfsDirPath); String hdfsFilePathStr = hdfsDirPathStr + "/" + getClientVersion(file); - LocalResource localResource = createLocalResource(fs, new Path(hdfsFilePathStr), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); + LocalResource localResource = createLocalResource(fs, new Path(hdfsFilePathStr), + LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); tmpResources.add(localResource); } @@ -392,9 +421,16 @@ public static LocalResource createHiveExecLocalResource(HiveConf conf) throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(hiveJarDir)); } - dirStatus = fs.getFileStatus(hiveJarDirPath); - if (dirStatus.isDir()) { + try { + dirStatus = fs.getFileStatus(hiveJarDirPath); + } catch (FileNotFoundException fe) { + // do nothing + } + System.out.println("Continuing after not finding dir"); + if ((dirStatus != null) && (dirStatus.isDir())) { + System.out.println("Dir status is dir!"); FileStatus[] listFileStatus = fs.listStatus(hiveJarDirPath); + System.out.println("list status got is of length: " + listFileStatus.length); for (FileStatus fstatus : listFileStatus) { String jarVersion = getClientVersion(fstatus.getPath().toString()); if (jarVersion.equals(currentJar)) { @@ -416,7 +452,9 @@ public static LocalResource createHiveExecLocalResource(HiveConf conf) * config variable HIVE_INSTALL_DIR. Path will be * HIVE_INSTALL_DIR/{username}/hive_install/ */ - if ((hiveJarDir == null) || ((dirStatus != null) && (!dirStatus.isDir()))) { + if ((hiveJarDir == null) || (dirStatus == null) || + ((dirStatus != null) && (!dirStatus.isDir()))) { + System.out.println("About to ship jar to /user location"); UserGroupInformation ugi = ShimLoader.getHadoopShims().getUGIForConf(conf); String userName = ShimLoader.getHadoopShims().getShortUserName(ugi); String userPathStr = conf.getVar(HiveConf.ConfVars.HIVE_INSTALL_DIR);