diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 15de10b..60f8ce4 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -3161,7 +3161,14 @@ private static void populateLlapDaemonVarsSet(Set llapDaemonVarsSetLocal "Comma-separated list of supported blobstore schemes."), HIVE_BLOBSTORE_USE_BLOBSTORE_AS_SCRATCHDIR("hive.blobstore.use.blobstore.as.scratchdir", false, - "Enable the use of scratch directories directly on blob storage systems (it may cause performance penalties)."); + "Enable the use of scratch directories directly on blob storage systems (it may cause performance penalties)."), + + HIVE_BLOBSTORE_MOVE_INDIVIDUAL_FILES("hive.blobstore.move.individual.files", true, + "When moving files within a blobstore, move files one at a time rather than at at directory level. Since " + + "moves may require copying the entire file, each move can take a long amount of time. Moving at a " + + "directory level may not be ideal if the blobstore connector cannot efficiently rename a directory " + + "(e.g. HADOOP-13600). By default, moves are done using a thread pool which allows each individual file " + + "to be moved in parallel."); public final String varname; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java index dab4c6a..d33edac 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java @@ -62,6 +62,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.PathFilter; +import org.apache.hadoop.hive.common.BlobStorageUtils; import org.apache.hadoop.hive.common.FileUtils; import org.apache.hadoop.hive.common.HiveStatsUtils; import org.apache.hadoop.hive.common.ObjectPair; @@ -3001,6 +3002,9 @@ public static boolean moveFile(final HiveConf conf, Path srcf, final Path destf, //needed for perm inheritance. final boolean inheritPerms = HiveConf.getBoolVar(conf, HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS); + final boolean moveIndividualFiles = HiveConf.getBoolVar(conf, + HiveConf.ConfVars.HIVE_BLOBSTORE_MOVE_INDIVIDUAL_FILES) && BlobStorageUtils.isBlobStorageFileSystem( + srcFs.getConf(), srcFs) && BlobStorageUtils.isBlobStorageFileSystem(destFs.getConf(), destFs); HdfsUtils.HadoopFileStatus destStatus = null; // If source path is a subdirectory of the destination path: @@ -3052,7 +3056,8 @@ public static boolean moveFile(final HiveConf conf, Path srcf, final Path destf, replace, // overwrite destination conf); } else { - if (destIsSubDir) { + if (destIsSubDir || moveIndividualFiles) { + FileStatus[] srcs = destFs.listStatus(srcf, FileUtils.HIDDEN_FILES_PATH_FILTER); List> futures = new LinkedList<>();