Index: eclipse-templates/.classpath
===================================================================
--- eclipse-templates/.classpath (revision 1066131)
+++ eclipse-templates/.classpath (working copy)
@@ -16,11 +16,11 @@
-
-
+
+
-
+
Index: ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java
===================================================================
--- ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java (revision 1066131)
+++ ql/src/java/org/apache/hadoop/hive/ql/plan/ConditionalResolverMergeFiles.java (working copy)
@@ -21,6 +21,7 @@
import java.io.IOException;
import java.io.Serializable;
import java.util.ArrayList;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
@@ -177,13 +178,13 @@
// add the merge MR job
setupMapRedWork(conf, work, trgtSize, totalSz);
resTsks.add(mrTask);
-
+
// add the move task for those partitions that do not need merging
if (toMove.size() > 0) { //
// modify the existing move task as it is already in the candidate running tasks
MoveWork mvWork = (MoveWork) mvTask.getWork();
LoadFileDesc lfd = mvWork.getLoadFileWork();
-
+
String targetDir = lfd.getTargetDir();
List targetDirs = new ArrayList(toMove.size());
int numDPCols = dpCtx.getNumDPCols();
@@ -200,7 +201,7 @@
target = target + Path.SEPARATOR + moveStrSplits[dpIndex];
dpIndex ++;
}
-
+
targetDirs.add(target);
}
@@ -209,7 +210,21 @@
mvWork.setLoadFileWork(null);
mvWork.setLoadTableWork(null);
mvWork.setMultiFilesDesc(lmfd);
- resTsks.add(mvTask);
+
+ // running the MoveTask and MR task in parallel may
+ // cause the mvTask write to /ds=1 and MR task write
+ // to /ds=1_1 for the same partition.
+ // make the MoveTask as the child of the MR Task
+ List> cTasks = mrTask.getDependentTasks();
+ if (cTasks != null) {
+ Iterator> itr = cTasks.iterator();
+ while (itr.hasNext()) {
+ Task extends Serializable> cld = itr.next();
+ itr.remove();
+ mvTask.addDependentTask(cld);
+ }
+ }
+ mrTask.addDependentTask(mvTask);
}
} else { // add the move task
resTsks.add(mvTask);
@@ -263,7 +278,7 @@
for (FileStatus fStat : fStats) {
totalSz += fStat.getLen();
}
-
+
if (totalSz < avgSize * fStats.length) {
return totalSz;
} else {