diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java index a575cdd..52a14a7 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java @@ -25,11 +25,14 @@ import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.Predicate; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -385,10 +388,19 @@ public boolean doNext(WritableComparable key, Writable value) throws IOException inputSplits = splitSampling(work.getSplitSample(), inputSplits); } if (inputSplits.length > 0) { + LinkedList originSplits = new LinkedList(Arrays.asList(inputSplits)); + CollectionUtils.filter(originSplits, new Predicate() { + @Override + public boolean evaluate(FetchInputFormatSplit split) { + return split.getLength() > 0; + } + }); + + FetchInputFormatSplit[] finalSplits = originSplits.toArray(new FetchInputFormatSplit[originSplits.size()]); if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_IN_TEST)) { - Arrays.sort(inputSplits, new FetchInputFormatSplitComparator()); + Arrays.sort(finalSplits, new FetchInputFormatSplitComparator()); } - return inputSplits; + return finalSplits; } } return null; @@ -705,7 +717,7 @@ public FetchInputFormatSplit(InputSplit split, InputFormat inputFormat) { return inputFormat.getRecordReader(getInputSplit(), job, Reporter.NULL); } } - + private static class FetchInputFormatSplitComparator implements Comparator { @Override public int compare(FetchInputFormatSplit a, FetchInputFormatSplit b) {