diff --git common/src/java/org/apache/hadoop/hive/conf/HiveConf.java common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 2655464..3e3b779 100644 --- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -787,7 +787,7 @@ HIVE_OPTIMIZE_TEZ("hive.optimize.tez", false), HIVE_JAR_DIRECTORY("hive.jar.directory", "hdfs:///apps/hive/install/"), - HIVE_INSTALL_DIR("hive.user.install.directory", "hdfs:///users/"), + HIVE_INSTALL_DIR("hive.user.install.directory", "hdfs:///user/"), ; public final String varname; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index 805af36..02fc5e9 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.ql.exec.tez; +import java.io.FileNotFoundException; import java.io.IOException; import java.net.URISyntaxException; import java.util.ArrayList; @@ -314,9 +315,10 @@ private static LocalResource createLocalResource(FileSystem remoteFs, Path file, * * @param conf * @return List local resources to add to execution - * @throws IOException + * @throws IOException when hdfs operation fails + * @throws LoginException when user cannot be determined */ - public static List localizeTempFiles(Configuration conf) throws IOException { + public static List localizeTempFiles(Configuration conf) throws IOException, LoginException { List tmpResources = new ArrayList(); String auxJars = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEAUXJARS); @@ -334,12 +336,35 @@ private static LocalResource createLocalResource(FileSystem remoteFs, Path file, throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(hdfsDirPathStr)); } + FileStatus fstatus = null; + try { + fstatus = fs.getFileStatus(hdfsDirPath); + } catch (FileNotFoundException fe) { + // do nothing + } + + if ((fstatus == null) || (!fstatus.isDir())) { + UserGroupInformation ugi = ShimLoader.getHadoopShims().getUGIForConf(conf); + String userName = ShimLoader.getHadoopShims().getShortUserName(ugi); + String userPathStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVE_INSTALL_DIR); + Path userPath = new Path(userPathStr); + fs = userPath.getFileSystem(conf); + if (!(fs instanceof DistributedFileSystem)) { + throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(userPathStr)); + } + + String jarPathStr = userPathStr + "/" + userName; + hdfsDirPathStr = jarPathStr; + hdfsDirPath = new Path(hdfsDirPathStr); + } + String allFiles = auxJars + "," + addedJars + "," + addedFiles + "," + addedArchives; String[] allFilesArr = allFiles.split(","); for (String file : allFilesArr) { fs.copyFromLocalFile(new Path(file), hdfsDirPath); String hdfsFilePathStr = hdfsDirPathStr + "/" + getClientVersion(file); - LocalResource localResource = createLocalResource(fs, new Path(hdfsFilePathStr), LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); + LocalResource localResource = createLocalResource(fs, new Path(hdfsFilePathStr), + LocalResourceType.FILE, LocalResourceVisibility.APPLICATION); tmpResources.add(localResource); } @@ -392,8 +417,12 @@ public static LocalResource createHiveExecLocalResource(HiveConf conf) throw new IOException(ErrorMsg.INVALID_HDFS_URI.format(hiveJarDir)); } - dirStatus = fs.getFileStatus(hiveJarDirPath); - if (dirStatus.isDir()) { + try { + dirStatus = fs.getFileStatus(hiveJarDirPath); + } catch (FileNotFoundException fe) { + // do nothing + } + if ((dirStatus != null) && (dirStatus.isDir())) { FileStatus[] listFileStatus = fs.listStatus(hiveJarDirPath); for (FileStatus fstatus : listFileStatus) { String jarVersion = getClientVersion(fstatus.getPath().toString()); @@ -416,7 +445,8 @@ public static LocalResource createHiveExecLocalResource(HiveConf conf) * config variable HIVE_INSTALL_DIR. Path will be * HIVE_INSTALL_DIR/{username}/hive_install/ */ - if ((hiveJarDir == null) || ((dirStatus != null) && (!dirStatus.isDir()))) { + if ((hiveJarDir == null) || (dirStatus == null) || + ((dirStatus != null) && (!dirStatus.isDir()))) { UserGroupInformation ugi = ShimLoader.getHadoopShims().getUGIForConf(conf); String userName = ShimLoader.getHadoopShims().getShortUserName(ugi); String userPathStr = conf.getVar(HiveConf.ConfVars.HIVE_INSTALL_DIR);