diff --git a/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java b/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java index 4149307..1aaafaf 100644 --- a/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java +++ b/core/src/main/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java @@ -67,6 +67,8 @@ import org.slf4j.LoggerFactory; */ class FileOutputCommitterContainer extends OutputCommitterContainer { + private static final String TEMP_DIR_NAME = "_temporary"; + private static final Logger LOG = LoggerFactory.getLogger(FileOutputCommitterContainer.class); private final boolean dynamicPartitioningUsed; private boolean partitionsDiscovered; @@ -431,45 +433,49 @@ class FileOutputCommitterContainer extends OutputCommitterContainer { } } else if(fs.getFileStatus(file).isDir()) { FileStatus[] children = fs.listStatus(file); + FileStatus firstChild = null; if (children != null && children.length > 0) { - FileStatus firstChild = children[0]; - if(firstChild.isDir()) { - // If the first child is directory, then rest would be directory too according to HCatalog dir structure - // recurse in that case - for (FileStatus child : children) { - moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun); - } - } else { + if (!children[0].toString().equals(TEMP_DIR_NAME)) { + firstChild = children[0]; + } else if (children.length > 1) { + firstChild = children[1]; + } + } + if(firstChild!=null && firstChild.isDir()) { + // If the first child is directory, then rest would be directory too according to HCatalog dir structure + // recurse in that case + for (FileStatus child : children) { + moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun); + } + } else { + if (!dryRun) { + if (dynamicPartitioningUsed) { + // Optimization: if the first child is file, we have reached the leaf directory, move the parent directory itself + // instead of moving each file under the directory. See HCATALOG-538 - if (!dryRun) { - if (dynamicPartitioningUsed) { - // Optimization: if the first child is file, we have reached the leaf directory, move the parent directory itself - // instead of moving each file under the directory. See HCATALOG-538 - - final Path parentDir = finalOutputPath.getParent(); - // Create the directory - fs.mkdirs(parentDir); - if (LOG.isDebugEnabled()) { - LOG.debug("Moving directory: " + file + " to " + parentDir); - } - if (!fs.rename(file, parentDir)) { - final String msg = "Failed to move file: " + file + " to " + parentDir; - LOG.error(msg); - throw new HCatException(ErrorType.ERROR_MOVE_FAILED, msg); - } - } else { - // In case of no partition we have to move each file - for (FileStatus child : children) { - moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun); - } + final Path parentDir = finalOutputPath.getParent(); + // Create the directory + fs.mkdirs(parentDir); + if (LOG.isDebugEnabled()) { + LOG.debug("Moving directory: " + file + " to " + parentDir); + } + if (!fs.rename(file, parentDir)) { + final String msg = "Failed to move file: " + file + " to " + parentDir; + LOG.error(msg); + throw new HCatException(ErrorType.ERROR_MOVE_FAILED, msg); } } else { - if(fs.exists(finalOutputPath)) { - throw new HCatException(ErrorType.ERROR_MOVE_FAILED, "Data already exists in " + finalOutputPath - + ", duplicate publish not possible."); + // In case of no partition we have to move each file + for (FileStatus child : children) { + moveTaskOutputs(fs, child.getPath(), srcDir, destDir, dryRun); } } + } else { + if(fs.exists(finalOutputPath)) { + throw new HCatException(ErrorType.ERROR_MOVE_FAILED, "Data already exists in " + finalOutputPath + + ", duplicate publish not possible."); + } } } } else {