Index: eclipse-templates/.classpath =================================================================== --- eclipse-templates/.classpath (revision 1066131) +++ eclipse-templates/.classpath (working copy) @@ -16,11 +16,11 @@ - - + + - + Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java =================================================================== --- ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java (revision 1066131) +++ ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java (working copy) @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.Serializable; import java.util.ArrayList; +import java.util.Iterator; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -177,13 +178,13 @@ // add the merge MR job setupMapRedWork(conf, work, trgtSize, totalSz); resTsks.add(mrTask); - + // add the move task for those partitions that do not need merging if (toMove.size() > 0) { // // modify the existing move task as it is already in the candidate running tasks MoveWork mvWork = (MoveWork) mvTask.getWork(); LoadFileDesc lfd = mvWork.getLoadFileWork(); - + String targetDir = lfd.getTargetDir(); List targetDirs = new ArrayList(toMove.size()); int numDPCols = dpCtx.getNumDPCols(); @@ -200,7 +201,7 @@ target = target + Path.SEPARATOR + moveStrSplits[dpIndex]; dpIndex ++; } - + targetDirs.add(target); } @@ -209,7 +210,21 @@ mvWork.setLoadFileWork(null); mvWork.setLoadTableWork(null); mvWork.setMultiFilesDesc(lmfd); - resTsks.add(mvTask); + + // running the MoveTask and MR task in parallel may + // cause the mvTask write to /ds=1 and MR task write + // to /ds=1_1 for the same partition. + // make the MoveTask as the child of the MR Task + List> cTasks = mrTask.getDependentTasks(); + if (cTasks != null) { + Iterator> itr = cTasks.iterator(); + while (itr.hasNext()) { + Task cld = itr.next(); + itr.remove(); + mvTask.addDependentTask(cld); + } + } + mrTask.addDependentTask(mvTask); } } else { // add the move task resTsks.add(mvTask); @@ -263,7 +278,7 @@ for (FileStatus fStat : fStats) { totalSz += fStat.getLen(); } - + if (totalSz < avgSize * fStats.length) { return totalSz; } else {