commit 834d491b906fd1ec1629ae15752f1cef20f1b71e Author: Vihang Karajgaonkar Date: Thu Feb 23 10:26:38 2017 -0800 HIVE-16014 : HiveMetastoreChecker should use hive.metastore.fshandler.threads instead of hive.mv.files.thread for pool size diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java index 5d2ae2ca28b78280434437614518b13a0f3a95cd..08590225db93d7d661d581122c2977d6f89def53 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/HiveMetaStoreChecker.java @@ -404,7 +404,12 @@ static String getPartitionName(Path tablePath, Path partitionPath, */ private void checkPartitionDirs(Path basePath, Set allDirs, int maxDepth) throws IOException, HiveException { - int poolSize = conf.getInt(ConfVars.HIVE_MOVE_FILES_THREAD_COUNT.varname, 15); + // Here we just reuse the THREAD_COUNT configuration for + // METASTORE_FS_HANDLER_THREADS_COUNT since this results in better performance + // The number of missing partitions discovered are later added by metastore using a + // threadpool of size METASTORE_FS_HANDLER_THREADS_COUNT. If we have different sized + // pool here the smaller sized pool of the two becomes a bottleneck + int poolSize = conf.getInt(ConfVars.METASTORE_FS_HANDLER_THREADS_COUNT.varname, 15); // Check if too low config is provided for move files. 2x CPU is reasonable max count. poolSize = poolSize == 0 ? poolSize : Math.max(poolSize,