commit 13f40fe65e44b6780c76310b7d9f7dd8b8d59b63 Author: Vihang Karajgaonkar Date: Mon Dec 17 17:13:56 2018 -0800 HIVE-21040 : msck does unnecessary file listing at last level of directory tree diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java index 2df45f6286d3fcf58ffecd89eb30fb153bd8b37f..33f54f07e079c26f8afaf73853677c86dde352cb 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HiveMetaStoreChecker.java @@ -474,10 +474,13 @@ private Path processPathDepthInfo(final PathDepthInfo pd) throws IOException, MetastoreException { final Path currentPath = pd.p; final int currentDepth = pd.depth; + if (currentDepth == partColNames.size()) { + return currentPath; + } FileStatus[] fileStatuses = fs.listStatus(currentPath, FileUtils.HIDDEN_FILES_PATH_FILTER); // found no files under a sub-directory under table base path; it is possible that the table // is empty and hence there are no partition sub-directories created under base path - if (fileStatuses.length == 0 && currentDepth > 0 && currentDepth < partColNames.size()) { + if (fileStatuses.length == 0 && currentDepth > 0) { // since maxDepth is not yet reached, we are missing partition // columns in currentPath logOrThrowExceptionWithMsg( @@ -485,12 +488,12 @@ private Path processPathDepthInfo(final PathDepthInfo pd) } else { // found files under currentPath add them to the queue if it is a directory for (FileStatus fileStatus : fileStatuses) { - if (!fileStatus.isDirectory() && currentDepth < partColNames.size()) { + if (!fileStatus.isDirectory()) { // found a file at depth which is less than number of partition keys logOrThrowExceptionWithMsg( "MSCK finds a file rather than a directory when it searches for " + fileStatus.getPath().toString()); - } else if (fileStatus.isDirectory() && currentDepth < partColNames.size()) { + } else if (fileStatus.isDirectory()) { // found a sub-directory at a depth less than number of partition keys // validate if the partition directory name matches with the corresponding // partition colName at currentDepth @@ -507,9 +510,6 @@ private Path processPathDepthInfo(final PathDepthInfo pd) } } } - if (currentDepth == partColNames.size()) { - return currentPath; - } } return null; }