diff --git ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java index 13750cd..00ed4e2 100644 --- ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java +++ ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java @@ -25,11 +25,13 @@ import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; +import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.Properties; import org.apache.commons.lang3.StringEscapeUtils; +import org.apache.commons.collections.CollectionUtils; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; @@ -38,6 +40,7 @@ import org.apache.hadoop.hive.common.ValidTxnList; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; +import org.apache.hadoop.hive.ql.io.AcidInputFormat; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader; import org.apache.hadoop.hive.ql.io.HiveInputFormat; @@ -379,10 +382,21 @@ public boolean doNext(WritableComparable key, Writable value) throws IOException inputSplits = splitSampling(work.getSplitSample(), inputSplits); } if (inputSplits.length > 0) { + LinkedList originSplits = new LinkedList(Arrays.asList(inputSplits)); + CollectionUtils.filter(originSplits, split -> { + FetchInputFormatSplit fetchSplit = (FetchInputFormatSplit)split; + // Ignore AcidInputFormat because of HIVE-10114 + return ((FetchInputFormatSplit)split).getLength() > 0 || + (fetchSplit.inputFormat instanceof AcidInputFormat); + }); + + FetchInputFormatSplit[] finalSplits = originSplits.toArray(new FetchInputFormatSplit[originSplits.size()]); if (HiveConf.getBoolVar(job, HiveConf.ConfVars.HIVE_IN_TEST)) { - Arrays.sort(inputSplits, new FetchInputFormatSplitComparator()); + Arrays.sort(finalSplits, new FetchInputFormatSplitComparator()); + } + if (finalSplits.length > 0) { + return finalSplits; } - return inputSplits; } } return null; @@ -699,7 +713,7 @@ public FetchInputFormatSplit(InputSplit split, InputFormat inputFormat) { return inputFormat.getRecordReader(getInputSplit(), job, Reporter.NULL); } } - + private static class FetchInputFormatSplitComparator implements Comparator { @Override public int compare(FetchInputFormatSplit a, FetchInputFormatSplit b) {