diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java index 13750cd..11b7787 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java @@ -25,11 +25,13 @@ import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -379,10 +381,14 @@ public boolean doNext(WritableComparable key, Writable value) throws IOException inputSplits = splitSampling(work.getSplitSample(), inputSplits); } if (inputSplits.length > 0) { + LinkedList originSplits = new LinkedList(Arrays.asList(inputSplits)); + CollectionUtils.filter(originSplits, split -> ((FetchInputFormatSplit)split).getLength() > 0); + + FetchInputFormatSplit[] finalSplits = originSplits.toArray(new FetchInputFormatSplit[originSplits.size()]); if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_IN_TEST)) { - Arrays.sort(inputSplits, new FetchInputFormatSplitComparator()); + Arrays.sort(finalSplits, new FetchInputFormatSplitComparator()); } - return inputSplits; + return finalSplits; } } return null; @@ -699,7 +705,7 @@ public FetchInputFormatSplit(InputSplit split, InputFormat inputFormat) { return inputFormat.getRecordReader(getInputSplit(), job, Reporter.NULL); } } - + private static class FetchInputFormatSplitComparator implements Comparator { @Override public int compare(FetchInputFormatSplit a, FetchInputFormatSplit b) {