diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java index 91c94efaf5..65933681a0 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/api/impl/LlapRecordReader.java @@ -501,7 +501,7 @@ public VectorizedRowBatch createValue() { @Override public long getPos() throws IOException { - return -1; // Position doesn't make sense for async reader, chunk order is arbitrary. + return isFirst ? 0 : -1; // Position doesn't make sense for async reader, chunk order is arbitrary. } @Override diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java index c8fa1b97dd..d8764d2d3c 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java @@ -40,6 +40,7 @@ import org.apache.hadoop.hive.common.ValidReaderWriteIdList; import org.apache.hadoop.hive.common.ValidWriteIdList; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.ql.exec.Utilities.RecordReaderStatus; import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.HiveContextAwareRecordReader; @@ -557,6 +558,7 @@ public InspectableObject getNextRow() throws IOException { if (context != null) { context.resetRow(); } + RecordReaderStatus rrStatus = RecordReaderStatus.NEXT; if (currRecReader == null) { currRecReader = getRecordReader(); if (currRecReader == null) { @@ -573,21 +575,24 @@ public InspectableObject getNextRow() throws IOException { footerCount = Utilities.getFooterCount(currDesc.getTableDesc(), job); // Skip header lines. - opNotEOF = Utilities.skipHeader(currRecReader, headerCount, key, value); + rrStatus = Utilities.skipHeader(currRecReader, headerCount, key, value); + opNotEOF = (rrStatus != RecordReaderStatus.EOF); // Initialize footer buffer. if (opNotEOF && footerCount > 0) { footerBuffer = new FooterBuffer(); - opNotEOF = footerBuffer.initializeBuffer(job, currRecReader, footerCount, key, value); + opNotEOF = footerBuffer.initializeBuffer(job, currRecReader, rrStatus, footerCount, key, value); } } if (opNotEOF && footerBuffer == null) { - /** - * When file doesn't end after skipping header line - * and there is no footer lines, read normally. - */ - opNotEOF = currRecReader.next(key, value); + if (rrStatus == RecordReaderStatus.NEXT) { + /** + * When file doesn't end after skipping header line + * and there is no footer lines, read normally. + */ + opNotEOF = currRecReader.next(key, value); + } } if (opNotEOF && footerBuffer != null) { opNotEOF = footerBuffer.updateBuffer(job, currRecReader, key, value); diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java index 8ead7975e5..83a471100b 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FooterBuffer.java @@ -22,15 +22,24 @@ import java.util.ArrayList; import org.apache.hadoop.hive.common.ObjectPair; +import org.apache.hadoop.hive.ql.exec.Utilities.RecordReaderStatus; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.io.Writable; import org.apache.hadoop.io.WritableComparable; import org.apache.hadoop.mapred.JobConf; import org.apache.hadoop.mapred.RecordReader; import org.apache.hadoop.util.ReflectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + public class FooterBuffer { private ArrayList> buffer; private int cur; + private int footerRowCount; + private int bufferRowCount; + + private static final Logger LOG = LoggerFactory.getLogger(FooterBuffer.class.getName()); public FooterBuffer() { } @@ -48,6 +57,10 @@ public void setCursor(int cur) { * @param recordreader * Record reader. * + * @param rrStatus + * Record reader status. CURRENT means the given key/value is not utilized yet. NEXT means + * need to fetch next key/value pair. + * * @param footerCount * Footer line number of the table files. * @@ -60,19 +73,36 @@ public void setCursor(int cur) { * @return Return true if there are 0 or more records left in the file * after initializing the footer buffer, otherwise return false. */ - public boolean initializeBuffer(JobConf job, RecordReader recordreader, - int footerCount, WritableComparable key, Writable value) throws IOException { + public boolean initializeBuffer(JobConf job, RecordReader recordreader, RecordReaderStatus rrStatus, + int footerCount, WritableComparable key, Writable value) throws IOException { + this.footerRowCount = footerCount; // Fill the buffer with key value pairs. this.buffer = new ArrayList<>(); - while (buffer.size() < footerCount) { - boolean notEOF = recordreader.next(key, value); - if (!notEOF) { - return false; + this.bufferRowCount = 0; + LOG.debug("FooterCount: {}", footerCount); + while (bufferRowCount < footerCount) { + if (rrStatus == RecordReaderStatus.CURRENT) { + LOG.debug("RR Status: CURRENT"); + rrStatus = RecordReaderStatus.NEXT; + } else { + boolean notEOF = recordreader.next(key, value); + if (!notEOF) { + return false; + } } ObjectPair tem = new ObjectPair<>(); tem.setFirst(ReflectionUtils.copy(job, key, tem.getFirst())); - tem.setSecond(ReflectionUtils.copy(job, value, tem.getSecond())); + if (value instanceof VectorizedRowBatch) { + VectorizedRowBatch vrb = (VectorizedRowBatch)value; + tem.setSecond(copyVrb(vrb, (VectorizedRowBatch)recordreader.createValue())); + bufferRowCount += vrb.size; + + LOG.debug("[initializeBuffer] Added Batch-{}: Size={}, BufferRowCount={}", buffer.size(), vrb.size, bufferRowCount); + } else { + tem.setSecond(ReflectionUtils.copy(job, value, tem.getSecond())); + bufferRowCount++; + } buffer.add(tem); } this.cur = 0; @@ -98,13 +128,77 @@ public boolean initializeBuffer(JobConf job, RecordReader recordreader, */ public boolean updateBuffer(JobConf job, RecordReader recordreader, WritableComparable key, Writable value) throws IOException { - key = ReflectionUtils.copy(job, buffer.get(cur).getFirst(), key); - value = ReflectionUtils.copy(job, buffer.get(cur).getSecond(), value); - boolean notEOF = recordreader.next(buffer.get(cur).getFirst(), buffer.get(cur).getSecond()); - if (notEOF) { - cur = (++cur) % buffer.size(); + if (buffer.size() == 0) { + LOG.debug("[updateBuffer] EOF - Fetched all rows."); + return false; + } + + key = ReflectionUtils.copy(job, (WritableComparable)buffer.get(cur).getFirst(), key); + boolean notEOF = true; + if (value instanceof VectorizedRowBatch) { + value = copyVrb((VectorizedRowBatch)buffer.get(cur).getSecond(), (VectorizedRowBatch)value); + VectorizedRowBatch vrb = (VectorizedRowBatch)value; + bufferRowCount -= vrb.size; + + LOG.debug("[updateBuffer] Fetched Batch-{}: Size={}, BufferRowCount={}", cur, vrb.size, bufferRowCount); + ObjectPair kvPair = buffer.remove(cur); + WritableComparable nextKey = (WritableComparable)kvPair.getFirst(); + Writable nextValue = (Writable)kvPair.getSecond(); + while (bufferRowCount < footerRowCount) { + notEOF = recordreader.next(nextKey, nextValue); + if (!notEOF) { + break; + } + VectorizedRowBatch nextVrb = (VectorizedRowBatch)nextValue; + ObjectPair tem = new ObjectPair<>(); + tem.setFirst(ReflectionUtils.copy(job, nextKey, tem.getFirst())); + tem.setSecond(copyVrb(nextVrb, (VectorizedRowBatch)recordreader.createValue())); + buffer.add(tem); + bufferRowCount += nextVrb.size; + LOG.debug("[updateBuffer] Added Batch-{}: Size={}, BufferRowCount={}", buffer.size(), nextVrb.size, bufferRowCount); + } + if (!notEOF) { + int balanceFooterCnt = footerRowCount - bufferRowCount; + if (vrb.size > balanceFooterCnt) { + int size = vrb.size - balanceFooterCnt; + int[] selected = new int[size]; + for (int i = 0; i < size; i++) { + if (vrb.selectedInUse) { + selected[i] = vrb.selected[i]; + } else { + selected[i] = i; + } + } + vrb.selectedInUse = true; + vrb.selected = selected; + vrb.size = size; + notEOF = true; + LOG.debug("[updateBuffer] Trimmed Batch-{}: Size={}, balanceFooterCnt={}", cur, size, balanceFooterCnt); + } + } + } else { + LOG.debug("[updateBuffer] Non-VectorizedRowBatch flow."); + value = ReflectionUtils.copy(job, (Writable)buffer.get(cur).getSecond(), value); + notEOF = recordreader.next(buffer.get(cur).getFirst(), buffer.get(cur).getSecond()); + if (notEOF) { + cur = (++cur) % buffer.size(); + } } return notEOF; } + private VectorizedRowBatch copyVrb(VectorizedRowBatch srcVrb, VectorizedRowBatch destVrb) { + destVrb.size = srcVrb.size; + destVrb.selectedInUse = srcVrb.selectedInUse; + destVrb.endOfFile = srcVrb.endOfFile; + int selectedLength = srcVrb.selected.length; + if (selectedLength > 0) { + destVrb.selected = new int[selectedLength]; + System.arraycopy(srcVrb.selected, 0, destVrb.selected, 0, selectedLength); + } + if (srcVrb.numCols > 0) { + System.arraycopy(srcVrb.cols, 0, destVrb.cols, 0, srcVrb.numCols); + } + return destVrb; + } } diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index bc75ec0b56..91851ec900 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -120,6 +120,7 @@ import org.apache.hadoop.hive.ql.exec.tez.TezTask; import org.apache.hadoop.hive.ql.exec.util.DAGTraversal; import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface; +import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.ContentSummaryInputFormat; @@ -3804,6 +3805,14 @@ public static void clearWorkMap(Configuration conf) { gWorkMap.get(conf).clear(); } + /** + * RecordReaderStatus. + * + */ + public static enum RecordReaderStatus { + CURRENT, NEXT, EOF + } + /** * Skip header lines in the table file when reading the record. * @@ -3819,18 +3828,52 @@ public static void clearWorkMap(Configuration conf) { * @param value * Value of current reading record. * - * @return Return true if there are 0 or more records left in the file - * after skipping all headers, otherwise return false. + * @return RecordReaderStatus.CURRENT -> if the current key/value pair after skipping headers + * are ready to be utilized. + * RecordReaderStatus.NEXT -> if 0 or more records left in the file + * after skipping all headers but the current key/value pair is already utilized. + * RecordReaderStatus.EOF -> if no more row to be fetched. */ - public static boolean skipHeader(RecordReader currRecReader, int headerCount, K key, V value) - throws IOException { - while (headerCount > 0) { - if (!currRecReader.next(key, value)) { - return false; + public static RecordReaderStatus skipHeader(RecordReader currRecReader, int headerCount, K key, V value) + throws IOException { + if (headerCount <= 0) { + LOG.debug("[skipHeader] Noop as headerCount={}", headerCount); + return RecordReaderStatus.NEXT; + } + LOG.debug("[skipHeader] HeaderCount: {}", headerCount); + if (value instanceof VectorizedRowBatch) { + VectorizedRowBatch vrb = (VectorizedRowBatch)value; + int size = 0; + while (size < headerCount) { + headerCount -= size; + if (!currRecReader.next(key, value)) { + return RecordReaderStatus.EOF; + } + size = vrb.size; + LOG.debug("[skipHeader] VectorizedRowBatch: {}, size={}, headerCount={}", vrb, size, headerCount); + } + if (headerCount > 0) { + size -= headerCount; + vrb.selectedInUse = true; + vrb.selected = new int[size]; + int startIdx = headerCount; + for (int i = 0; i < size; i++, startIdx++) { + vrb.selected[i] = startIdx; + } + vrb.size = size; + LOG.debug("[skipHeader] Trimmed VectorizedRowBatch: {}, size={}, headerCount={}", vrb, size, headerCount); } - headerCount--; + return RecordReaderStatus.CURRENT; + } else { + LOG.debug("[skipHeader] Non-VectorizedRowBatch flow: {}", value); + while (headerCount > 0) { + if (!currRecReader.next(key, value)) { + return RecordReaderStatus.EOF; + } + headerCount--; + } + return RecordReaderStatus.NEXT; } - return true; } /** diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java index 7f3ef37258..b509e451ca 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/HiveContextAwareRecordReader.java @@ -29,6 +29,7 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.io.HiveIOExceptionHandlerUtil; import org.apache.hadoop.hive.ql.exec.Utilities; +import org.apache.hadoop.hive.ql.exec.Utilities.RecordReaderStatus; import org.apache.hadoop.hive.ql.exec.FooterBuffer; import org.apache.hadoop.hive.ql.io.IOContext.Comparison; import org.apache.hadoop.hive.ql.plan.PartitionDesc; @@ -68,6 +69,7 @@ private String genericUDFClassName = null; private final List stopComparisons = new ArrayList(); private Map pathToPartitionInfo; + private RecordReaderStatus rrStatus = RecordReaderStatus.NEXT; protected RecordReader recordReader; protected JobConf jobConf; @@ -346,19 +348,25 @@ public boolean doNext(K key, V value) throws IOException { } // If input contains header, skip header. - if (!Utilities.skipHeader(recordReader, headerCount, key, value)) { + rrStatus = Utilities.skipHeader(recordReader, headerCount, key, value); + if (rrStatus == RecordReaderStatus.EOF) { return false; } if (footerCount > 0) { footerBuffer = new FooterBuffer(); - if (!footerBuffer.initializeBuffer(jobConf, recordReader, footerCount, key, value)) { + if (!footerBuffer.initializeBuffer(jobConf, recordReader, rrStatus, footerCount, key, value)) { return false; } + rrStatus = RecordReaderStatus.NEXT; } } if (footerBuffer == null) { - // Table files don't have footer rows. + if (rrStatus == RecordReaderStatus.CURRENT) { + // If status is current means, already key/value fetched and yet to utilize by caller. + rrStatus = RecordReaderStatus.NEXT; + return true; + } return recordReader.next(key, value); } else { return footerBuffer.updateBuffer(jobConf, recordReader, key, value); diff --git a/ql/src/test/queries/clientpositive/file_with_header_footer.q b/ql/src/test/queries/clientpositive/file_with_header_footer.q index 8913e54ad0..d075ea5c6e 100644 --- a/ql/src/test/queries/clientpositive/file_with_header_footer.q +++ b/ql/src/test/queries/clientpositive/file_with_header_footer.q @@ -9,8 +9,10 @@ dfs -copyFromLocal ../../data/files/header_footer_table_3 hdfs:///tmp/test/heade CREATE EXTERNAL TABLE header_footer_table_1 (name string, message string, id int) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LOCATION 'hdfs:///tmp/test/header_footer_table_1' tblproperties ("skip.header.line.count"="1", "skip.footer.line.count"="2"); +SELECT COUNT(*) FROM header_footer_table_1; SELECT * FROM header_footer_table_1; +SELECT COUNT(*) FROM header_footer_table_1 WHERE id < 50; SELECT * FROM header_footer_table_1 WHERE id < 50; CREATE EXTERNAL TABLE header_footer_table_2 (name string, message string, id int) PARTITIONED BY (year int, month int, day int) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' tblproperties ("skip.header.line.count"="1", "skip.footer.line.count"="2"); @@ -21,14 +23,18 @@ ALTER TABLE header_footer_table_2 ADD PARTITION (year=2012, month=1, day=2) loca ALTER TABLE header_footer_table_2 ADD PARTITION (year=2012, month=1, day=3) location 'hdfs:///tmp/test/header_footer_table_2/2012/01/03'; +SELECT COUNT(*) FROM header_footer_table_2; SELECT * FROM header_footer_table_2; +SELECT COUNT(*) FROM header_footer_table_2 WHERE id < 50; SELECT * FROM header_footer_table_2 WHERE id < 50; CREATE EXTERNAL TABLE emptytable (name string, message string, id int) ROW FORMAT DELIMITED FIELDS TERMINATED BY '\t' LOCATION 'hdfs:///tmp/test/header_footer_table_3' tblproperties ("skip.header.line.count"="1", "skip.footer.line.count"="2"); +SELECT COUNT(*) FROM emptytable; SELECT * FROM emptytable; +SELECT COUNT(*) FROM emptytable WHERE id < 50; SELECT * FROM emptytable WHERE id < 50; DROP TABLE header_footer_table_1; diff --git a/ql/src/test/results/clientpositive/llap/file_with_header_footer.q.out b/ql/src/test/results/clientpositive/llap/file_with_header_footer.q.out index e16df6beaa..60ac0ee977 100644 --- a/ql/src/test/results/clientpositive/llap/file_with_header_footer.q.out +++ b/ql/src/test/results/clientpositive/llap/file_with_header_footer.q.out @@ -8,6 +8,15 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Input: hdfs://### HDFS PATH ### POSTHOOK: Output: database:default POSTHOOK: Output: default@header_footer_table_1 +PREHOOK: query: SELECT COUNT(*) FROM header_footer_table_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@header_footer_table_1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM header_footer_table_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@header_footer_table_1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 PREHOOK: query: SELECT * FROM header_footer_table_1 PREHOOK: type: QUERY PREHOOK: Input: default@header_footer_table_1 @@ -27,6 +36,15 @@ xifa2 phd 13 chuan2 hadoop 14 shanyu2 senior 15 david3 oozie 22 +PREHOOK: query: SELECT COUNT(*) FROM header_footer_table_1 WHERE id < 50 +PREHOOK: type: QUERY +PREHOOK: Input: default@header_footer_table_1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM header_footer_table_1 WHERE id < 50 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@header_footer_table_1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 PREHOOK: query: SELECT * FROM header_footer_table_1 WHERE id < 50 PREHOOK: type: QUERY PREHOOK: Input: default@header_footer_table_1 @@ -81,6 +99,21 @@ POSTHOOK: type: ALTERTABLE_ADDPARTS POSTHOOK: Input: hdfs://### HDFS PATH ### POSTHOOK: Output: default@header_footer_table_2 POSTHOOK: Output: default@header_footer_table_2@year=2012/month=1/day=3 +PREHOOK: query: SELECT COUNT(*) FROM header_footer_table_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@header_footer_table_2 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM header_footer_table_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@header_footer_table_2 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3 +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 PREHOOK: query: SELECT * FROM header_footer_table_2 PREHOOK: type: QUERY PREHOOK: Input: default@header_footer_table_2 @@ -106,6 +139,21 @@ xifa2 phd 13 2012 1 2 chuan2 hadoop 14 2012 1 2 shanyu2 senior 15 2012 1 2 david3 oozie 22 2012 1 3 +PREHOOK: query: SELECT COUNT(*) FROM header_footer_table_2 WHERE id < 50 +PREHOOK: type: QUERY +PREHOOK: Input: default@header_footer_table_2 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM header_footer_table_2 WHERE id < 50 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@header_footer_table_2 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3 +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 PREHOOK: query: SELECT * FROM header_footer_table_2 WHERE id < 50 PREHOOK: type: QUERY PREHOOK: Input: default@header_footer_table_2 @@ -141,6 +189,15 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Input: hdfs://### HDFS PATH ### POSTHOOK: Output: database:default POSTHOOK: Output: default@emptytable +PREHOOK: query: SELECT COUNT(*) FROM emptytable +PREHOOK: type: QUERY +PREHOOK: Input: default@emptytable +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM emptytable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emptytable +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 PREHOOK: query: SELECT * FROM emptytable PREHOOK: type: QUERY PREHOOK: Input: default@emptytable @@ -149,6 +206,15 @@ POSTHOOK: query: SELECT * FROM emptytable POSTHOOK: type: QUERY POSTHOOK: Input: default@emptytable POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: SELECT COUNT(*) FROM emptytable WHERE id < 50 +PREHOOK: type: QUERY +PREHOOK: Input: default@emptytable +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM emptytable WHERE id < 50 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emptytable +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 PREHOOK: query: SELECT * FROM emptytable WHERE id < 50 PREHOOK: type: QUERY PREHOOK: Input: default@emptytable diff --git a/ql/src/test/results/clientpositive/spark/file_with_header_footer.q.out b/ql/src/test/results/clientpositive/spark/file_with_header_footer.q.out index e16df6beaa..60ac0ee977 100644 --- a/ql/src/test/results/clientpositive/spark/file_with_header_footer.q.out +++ b/ql/src/test/results/clientpositive/spark/file_with_header_footer.q.out @@ -8,6 +8,15 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Input: hdfs://### HDFS PATH ### POSTHOOK: Output: database:default POSTHOOK: Output: default@header_footer_table_1 +PREHOOK: query: SELECT COUNT(*) FROM header_footer_table_1 +PREHOOK: type: QUERY +PREHOOK: Input: default@header_footer_table_1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM header_footer_table_1 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@header_footer_table_1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 PREHOOK: query: SELECT * FROM header_footer_table_1 PREHOOK: type: QUERY PREHOOK: Input: default@header_footer_table_1 @@ -27,6 +36,15 @@ xifa2 phd 13 chuan2 hadoop 14 shanyu2 senior 15 david3 oozie 22 +PREHOOK: query: SELECT COUNT(*) FROM header_footer_table_1 WHERE id < 50 +PREHOOK: type: QUERY +PREHOOK: Input: default@header_footer_table_1 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM header_footer_table_1 WHERE id < 50 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@header_footer_table_1 +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 PREHOOK: query: SELECT * FROM header_footer_table_1 WHERE id < 50 PREHOOK: type: QUERY PREHOOK: Input: default@header_footer_table_1 @@ -81,6 +99,21 @@ POSTHOOK: type: ALTERTABLE_ADDPARTS POSTHOOK: Input: hdfs://### HDFS PATH ### POSTHOOK: Output: default@header_footer_table_2 POSTHOOK: Output: default@header_footer_table_2@year=2012/month=1/day=3 +PREHOOK: query: SELECT COUNT(*) FROM header_footer_table_2 +PREHOOK: type: QUERY +PREHOOK: Input: default@header_footer_table_2 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM header_footer_table_2 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@header_footer_table_2 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3 +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 PREHOOK: query: SELECT * FROM header_footer_table_2 PREHOOK: type: QUERY PREHOOK: Input: default@header_footer_table_2 @@ -106,6 +139,21 @@ xifa2 phd 13 2012 1 2 chuan2 hadoop 14 2012 1 2 shanyu2 senior 15 2012 1 2 david3 oozie 22 2012 1 3 +PREHOOK: query: SELECT COUNT(*) FROM header_footer_table_2 WHERE id < 50 +PREHOOK: type: QUERY +PREHOOK: Input: default@header_footer_table_2 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2 +PREHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3 +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM header_footer_table_2 WHERE id < 50 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@header_footer_table_2 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=1 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=2 +POSTHOOK: Input: default@header_footer_table_2@year=2012/month=1/day=3 +POSTHOOK: Output: hdfs://### HDFS PATH ### +11 PREHOOK: query: SELECT * FROM header_footer_table_2 WHERE id < 50 PREHOOK: type: QUERY PREHOOK: Input: default@header_footer_table_2 @@ -141,6 +189,15 @@ POSTHOOK: type: CREATETABLE POSTHOOK: Input: hdfs://### HDFS PATH ### POSTHOOK: Output: database:default POSTHOOK: Output: default@emptytable +PREHOOK: query: SELECT COUNT(*) FROM emptytable +PREHOOK: type: QUERY +PREHOOK: Input: default@emptytable +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM emptytable +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emptytable +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 PREHOOK: query: SELECT * FROM emptytable PREHOOK: type: QUERY PREHOOK: Input: default@emptytable @@ -149,6 +206,15 @@ POSTHOOK: query: SELECT * FROM emptytable POSTHOOK: type: QUERY POSTHOOK: Input: default@emptytable POSTHOOK: Output: hdfs://### HDFS PATH ### +PREHOOK: query: SELECT COUNT(*) FROM emptytable WHERE id < 50 +PREHOOK: type: QUERY +PREHOOK: Input: default@emptytable +PREHOOK: Output: hdfs://### HDFS PATH ### +POSTHOOK: query: SELECT COUNT(*) FROM emptytable WHERE id < 50 +POSTHOOK: type: QUERY +POSTHOOK: Input: default@emptytable +POSTHOOK: Output: hdfs://### HDFS PATH ### +0 PREHOOK: query: SELECT * FROM emptytable WHERE id < 50 PREHOOK: type: QUERY PREHOOK: Input: default@emptytable