diff --git common/src/java/org/apache/hadoop/hive/common/FileUtils.java common/src/java/org/apache/hadoop/hive/common/FileUtils.java index 91f67b7..c2c54bc 100644 --- common/src/java/org/apache/hadoop/hive/common/FileUtils.java +++ common/src/java/org/apache/hadoop/hive/common/FileUtils.java @@ -335,16 +335,16 @@ public static void listStatusRecursively(FileSystem fs, FileStatus fileStatus, * @param fs * file system * @param path - * @return the argument path if it exists or a parent path exists. Returns - * NULL root is only parent that exists + * @return FileStatus for argument path if it exists or the first ancestor in the path that exists * @throws IOException */ - public static Path getPathOrParentThatExists(FileSystem fs, Path path) throws IOException { - if (!fs.exists(path)) { - Path parentPath = path.getParent(); - return getPathOrParentThatExists(fs, parentPath); + public static FileStatus getPathOrParentThatExists(FileSystem fs, Path path) throws IOException { + FileStatus stat = FileUtils.getFileStatusOrNull(fs, path); + if (stat != null) { + return stat; } - return path; + Path parentPath = path.getParent(); + return getPathOrParentThatExists(fs, parentPath); } /** @@ -743,4 +743,20 @@ public static void checkDeletePermission(Path path, Configuration conf, String u } + /** + * Attempts to get file status. This method differs from the FileSystem API in that it returns + * null instead of throwing FileNotFoundException if the path does not exist. + * + * @param fs file system to check + * @param path file system path to check + * @return FileStatus for path or null if path does not exist + * @throws IOException if there is an I/O error + */ + public static FileStatus getFileStatusOrNull(FileSystem fs, Path path) throws IOException { + try { + return fs.getFileStatus(path); + } catch (FileNotFoundException e) { + return null; + } + } } diff --git hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java index bf2ba5a..ad5122d 100644 --- hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java +++ hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/FileOutputCommitterContainer.java @@ -510,8 +510,9 @@ private void moveTaskOutputs(FileSystem fs, Path file, Path srcDir, } final Path finalOutputPath = getFinalPath(fs, file, srcDir, destDir, immutable); + FileStatus fileStatus = fs.getFileStatus(file); - if (fs.isFile(file)) { + if (fileStatus.isFile()) { if (dryRun){ if (immutable){ // Dryrun checks are meaningless for mutable table - we should always succeed @@ -541,7 +542,7 @@ private void moveTaskOutputs(FileSystem fs, Path file, Path srcDir, } } } - } else if(fs.getFileStatus(file).isDir()) { + } else if (fileStatus.isDirectory()) { FileStatus[] children = fs.listStatus(file); FileStatus firstChild = null; diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java index c9029f2..204ad6e 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java @@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; import org.apache.hadoop.hive.ql.Context; @@ -926,8 +927,9 @@ private boolean checkPreExisting(Path src, Path dest, Configuration conf) throws IOException { FileSystem destFS = dest.getFileSystem(conf); FileSystem sourceFS = src.getFileSystem(conf); - if (destFS.exists(dest)) { - return (sourceFS.getFileStatus(src).getLen() == destFS.getFileStatus(dest).getLen()); + FileStatus destStatus = FileUtils.getFileStatusOrNull(destFS, dest); + if (destStatus != null) { + return (sourceFS.getFileStatus(src).getLen() == destStatus.getLen()); } return false; } diff --git ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java index 0e119e2..728118a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java +++ ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcRawRecordMerger.java @@ -460,7 +460,7 @@ private void discoverKeyBounds(Reader reader, Path deltaFile = AcidUtils.createBucketFile(delta, bucket); FileSystem fs = deltaFile.getFileSystem(conf); long length = getLastFlushLength(fs, deltaFile); - if (fs.exists(deltaFile) && length != -1) { + if (length != -1 && fs.exists(deltaFile)) { Reader deltaReader = OrcFile.createReader(deltaFile, OrcFile.readerOptions(conf).maxLength(length)); ReaderPair deltaPair = new ReaderPair(key, deltaReader, bucket, minKey, diff --git ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index d3d3bac..69acd3b 100644 --- ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -2416,7 +2416,7 @@ public PrincipalPrivilegeSet get_privilege_set(HiveObjectType objectType, List> result = new ArrayList>(); try { - FileStatus destStatus = !replace && fs.exists(destf) ? fs.getFileStatus(destf) : null; + FileStatus destStatus = !replace ? FileUtils.getFileStatusOrNull(fs, destf) : null; if (destStatus != null && !destStatus.isDir()) { throw new HiveException("checkPaths: destination " + destf + " should be a directory"); diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java index 5786277..671257a 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/StorageBasedAuthorizationProvider.java @@ -361,19 +361,22 @@ protected void checkPermissions(final Configuration conf, final Path path, final FileSystem fs = path.getFileSystem(conf); - if (fs.exists(path)) { - checkPermissions(fs, path, actions, authenticator.getUserName()); + FileStatus pathStatus = FileUtils.getFileStatusOrNull(fs, path); + if (pathStatus != null) { + checkPermissions(fs, pathStatus, actions, authenticator.getUserName()); } else if (path.getParent() != null) { // find the ancestor which exists to check its permissions Path par = path.getParent(); + FileStatus parStatus = null; while (par != null) { - if (fs.exists(par)) { + parStatus = FileUtils.getFileStatusOrNull(fs, par); + if (parStatus != null) { break; } par = par.getParent(); } - checkPermissions(fs, par, actions, authenticator.getUserName()); + checkPermissions(fs, parStatus, actions, authenticator.getUserName()); } } @@ -382,18 +385,20 @@ protected void checkPermissions(final Configuration conf, final Path path, * does not exists, it returns. */ @SuppressWarnings("deprecation") - protected static void checkPermissions(final FileSystem fs, final Path path, + protected static void checkPermissions(final FileSystem fs, final FileStatus stat, final EnumSet actions, String user) throws IOException, AccessControlException, HiveException { - try { - FileStatus stat = fs.getFileStatus(path); - for (FsAction action : actions) { - FileUtils.checkFileAccessWithImpersonation(fs, stat, action, user); - } - } catch (FileNotFoundException fnfe) { + if (stat == null) { // File named by path doesn't exist; nothing to validate. return; + } + FsAction checkActions = FsAction.NONE; + for (FsAction action : actions) { + checkActions = checkActions.or(action); + } + try { + FileUtils.checkFileAccessWithImpersonation(fs, stat, checkActions, user); } catch (org.apache.hadoop.fs.permission.AccessControlException ace) { // Older hadoop version will throw this @deprecated Exception. throw accessControlException(ace); diff --git ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java index 1637162..329781c 100644 --- ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java +++ ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/SQLAuthorizationUtils.java @@ -385,8 +385,7 @@ public static RequiredPrivileges getPrivilegesFromFS(Path filePath, HiveConf con FileSystem fs; try { fs = FileSystem.get(filePath.toUri(), conf); - Path path = FileUtils.getPathOrParentThatExists(fs, filePath); - FileStatus fileStatus = fs.getFileStatus(path); + FileStatus fileStatus = FileUtils.getPathOrParentThatExists(fs, filePath); if (FileUtils.isOwnerOfFileHierarchy(fs, fileStatus, userName)) { availPrivs.addPrivilege(SQLPrivTypeGrant.OWNER_PRIV); }