Index: src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java =================================================================== --- src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java (revision 1389632) +++ src/java/org/apache/hcatalog/mapreduce/FileRecordWriterContainer.java (working copy) @@ -144,8 +144,11 @@ if (baseOutputCommitter.needsTaskCommit(currContext)) { baseOutputCommitter.commitTask(currContext); } - org.apache.hadoop.mapred.JobContext currJobContext = HCatMapRedUtil.createJobContext(currContext); - baseOutputCommitter.commitJob(currJobContext); + // HCATALOG-513: commitJob() removes the temporary job directory which causes the directory to be unavailable + // for other running tasks and they all fail. We need a way/place to do checkOutputSpecs() and commitJob() + // without interfering with other tasks + // org.apache.hadoop.mapred.JobContext currJobContext = HCatMapRedUtil.createJobContext(currContext); + // baseOutputCommitter.commitJob(currJobContext); } } else { getBaseRecordWriter().close(reporter);