Index: src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java =================================================================== --- src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java (revision 1389632) +++ src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java (working copy) @@ -144,8 +144,6 @@ if (baseOutputCommitter.needsTaskCommit(currContext)) { baseOutputCommitter.commitTask(currContext); } - org.apache.hadoop.mapred.JobContext currJobContext = HCatMapRedUtil.createJobContext(currContext); - baseOutputCommitter.commitJob(currJobContext); } } else { getBaseRecordWriter().close(reporter); Index: src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java =================================================================== --- src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java (revision 1389632) +++ src/java/org/apache/hcatalog/mapreduce/FileOutputCommitterContainer.java (working copy) @@ -38,6 +38,7 @@ import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobStatus.State; import org.apache.hadoop.mapreduce.TaskAttemptContext; +import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.security.AccessControlException; import org.apache.hcatalog.common.ErrorType; import org.apache.hcatalog.common.HCatConstants; @@ -207,6 +208,15 @@ public void commitJob(JobContext jobContext) throws IOException { if (dynamicPartitioningUsed) { discoverPartitions(jobContext); + // Commit each partition so it gets moved out of the job work dir + for (Entry entry : contextDiscoveredByPath.entrySet()) { + Path path = new Path(entry.getKey()); + JobContext context = entry.getValue(); + TaskAttemptContext taskAttemptContext = + HCatHadoopShims.Instance.get().createTaskAttemptContext(context.getConfiguration(), + HCatHadoopShims.Instance.get().createTaskAttemptID()); + new FileOutputCommitter(path, taskAttemptContext).commitJob(context); + } } if (getBaseOutputCommitter() != null && !dynamicPartitioningUsed) { getBaseOutputCommitter().commitJob(HCatMapRedUtil.createJobContext(jobContext));