From cf6f4fdaba04331130755ed60c3f018feac20672 Mon Sep 17 00:00:00 2001 From: Nishant Date: Wed, 8 Aug 2018 00:14:02 +0530 Subject: [PATCH] [HIVE-20279] Skip Header and footer check for druid review comment --- .../apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java | 5 ++++- ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java | 10 ++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java index 60833bf8ea..b90d2c1876 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java @@ -23,6 +23,7 @@ import java.util.List; import java.util.Map; +import org.apache.hadoop.mapred.TextInputFormat; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.fs.FileSystem; @@ -178,6 +179,8 @@ private void initIOContext(long startPos, boolean isBlockPointer, public void initIOContext(FileSplit split, JobConf job, Class inputFormatClass) throws IOException { this.initIOContext(split, job, inputFormatClass, null); + // only skip Header/Footer rows for TextInputFormat and its subclasses. + ioCxtRef.setHeaderOrFooterSkippable(TextInputFormat.class.isAssignableFrom(inputFormatClass)); } public void initIOContext(FileSplit split, JobConf job, @@ -314,7 +317,7 @@ public boolean doNext(K key, V value) throws IOException { * If file contains footer, used a FooterBuffer to remove footer lines * at the end of the table file. **/ - if (this.ioCxtRef.getCurrentBlockStart() == 0) { + if (this.ioCxtRef.isHeaderOrFooterSkippable() && this.ioCxtRef.getCurrentBlockStart() == 0) { // Check if the table file has header to skip. footerBuffer = null; diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java index e95b0456c0..e1fe42a76d 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/IOContext.java @@ -34,6 +34,7 @@ private long currentRow; private boolean isBlockPointer; private boolean ioExceptions; + private boolean isHeaderOrFooterSkippable; // Are we using the fact the input is sorted private boolean useSorted = false; @@ -65,6 +66,7 @@ public IOContext() { this.currentRow = 0; this.isBlockPointer = true; this.ioExceptions = false; + this.isHeaderOrFooterSkippable = true; } public long getCurrentBlockStart() { @@ -139,6 +141,14 @@ public void setEndBinarySearch(boolean endBinarySearch) { this.endBinarySearch = endBinarySearch; } + public boolean isHeaderOrFooterSkippable() { + return isHeaderOrFooterSkippable; + } + + public void setHeaderOrFooterSkippable(boolean headerOrFooterSkippable) { + isHeaderOrFooterSkippable = headerOrFooterSkippable; + } + public Comparison getComparison() { return comparison; } -- 2.15.2 (Apple Git-101.1)