diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java index aa607cc875ceafe2f6c29592dd290a3ebffae874..b92f7ec9a1b597198ce1ea691c947db53b4c764e 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/CombineHiveRecordReader.java @@ -31,6 +31,7 @@ import org.apache.hadoop.mapred.InputSplit; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.Reporter; +import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.mapred.lib.CombineFileSplit; /** @@ -43,7 +44,7 @@ extends HiveContextAwareRecordReader { public CombineHiveRecordReader(InputSplit split, Configuration conf, - Reporter reporter, Integer partition) throws IOException { + Reporter reporter, Integer partition, RecordReader preReader) throws IOException { super((JobConf)conf); CombineHiveInputSplit hsplit = split instanceof CombineHiveInputSplit ? (CombineHiveInputSplit) split : @@ -67,6 +68,13 @@ public CombineHiveRecordReader(InputSplit split, Configuration conf, this.setRecordReader(inputFormat.getRecordReader(fsplit, jobConf, reporter)); this.initIOContext(fsplit, jobConf, inputFormatClass, this.recordReader); + + if (preReader != null && preReader instanceof CombineHiveRecordReader + && ((CombineHiveRecordReader)preReader).getFooterBuffer() != null) { + if (partition != 0 && hsplit.getPaths()[partition -1].equals(hsplit.getPaths()[partition])) //the split is from same file as previous split. + this.setFooterBuffer(((CombineHiveRecordReader)preReader).getFooterBuffer()); + } + } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java index d602c7601be9d0662834fe147b0b75b485d8475e..46f99702730aecf8f3ac888d7a78a7425120f60c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java @@ -244,6 +244,15 @@ public float getProgress() throws IOException { private int headerCount = 0; private int footerCount = 0; + protected FooterBuffer getFooterBuffer() { + return footerBuffer; + } + + protected void setFooterBuffer( FooterBuffer buf) { + footerBuffer = buf; + } + + public boolean doNext(K key, V value) throws IOException { if (this.isSorted) { if (this.getIOContext().shouldEndBinarySearch() || @@ -308,6 +317,7 @@ public boolean doNext(K key, V value) throws IOException { if (this.ioCxtRef.getCurrentBlockStart() == 0) { // Check if the table file has header to skip. + footerBuffer = null; Path filePath = this.ioCxtRef.getInputPath(); PartitionDesc part = null; try { diff --git a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java index 224ce3b0366ec1798a7173b7dd8e0229e3dd6914..018cb9fa8b479809e805784eaa7aee18d081b026 100644 --- a/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java +++ b/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java @@ -118,7 +118,8 @@ public void write(DataOutput out) throws IOException { InputSplit.class, Configuration.class, Reporter.class, - Integer.class + Integer.class, + RecordReader.class }; protected CombineFileSplit split; @@ -237,6 +238,7 @@ private boolean doNextWithExceptionHandler(K key, V value) throws IOException { */ protected boolean initNextRecordReader(K key) throws IOException { + RecordReader preReader = curReader; //it is OK, curReader is closed, for we only need footer buffer info from preReader. if (curReader != null) { curReader.close(); curReader = null; @@ -253,7 +255,7 @@ protected boolean initNextRecordReader(K key) throws IOException { // get a record reader for the idx-th chunk try { curReader = rrConstructor.newInstance(new Object[] - {split, jc, reporter, Integer.valueOf(idx)}); + {split, jc, reporter, Integer.valueOf(idx), preReader}); // change the key if need be if (key != null) {