diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java index 6d1ca7227d..2349cda1a3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcAcidRowBatchReader.java @@ -390,7 +390,8 @@ public void setBaseAndInnerReader( private OrcRawRecordMerger.KeyInterval findMinMaxKeys( OrcSplit orcSplit, Configuration conf, Reader.Options deleteEventReaderOptions) throws IOException { - if(!HiveConf.getBoolVar(conf, ConfVars.FILTER_DELETE_EVENTS)) { + final boolean noDeleteDeltas = getDeleteDeltaDirsFromSplit(orcSplit).length == 0; + if(!HiveConf.getBoolVar(conf, ConfVars.FILTER_DELETE_EVENTS) || noDeleteDeltas) { LOG.debug("findMinMaxKeys() " + ConfVars.FILTER_DELETE_EVENTS + "=false"); return new OrcRawRecordMerger.KeyInterval(null, null); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java index 91458ea87a..191e98d1e1 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestInputOutputFormat.java @@ -802,8 +802,8 @@ public void testFSCallsVectorizedOrcAcidRowBatchReader() throws IOException { int readsAfter = fs.statistics.getReadOps(); System.out.println("STATS TRACE END - " + testCaseName.getMethodName()); int delta = readsAfter - readsBefore; - //HIVE-16812 adds 1 read of the footer of each file - assertEquals(16, delta); + //HIVE-16812 adds 1 read of the footer of each file (only if delete delta exists) + assertEquals(8, delta); } finally { MockFileSystem.clearGlobalFiles(); } diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java index 3382288988..a8f18d1791 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedOrcAcidRowBatchReader.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hive.ql.exec.vector.LongColumnVector; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch; import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatchCtx; +import org.apache.hadoop.hive.ql.io.AcidInputFormat; import org.apache.hadoop.hive.ql.io.AcidOutputFormat; import org.apache.hadoop.hive.ql.io.AcidUtils; import org.apache.hadoop.hive.ql.io.BucketCodec; @@ -60,6 +61,8 @@ import static org.junit.Assert.*; +import com.google.common.collect.Lists; + /** * This class tests the VectorizedOrcAcidRowBatchReader by creating an actual split and a set * of delete delta files. The split is on an insert delta and there are multiple delete deltas @@ -549,7 +552,7 @@ private void testDeleteEventFiltering3() throws Exception { OrcSplit split = new OrcSplit(acidFilePath, null, stripe.getOffset() + 50, stripe.getLength() - 100, - new String[] {"localhost"}, null, false, true, new ArrayList<>(), + new String[] {"localhost"}, null, false, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, null); validateKeyInterval(split, new RecordIdentifier(1, 1, 1), @@ -560,7 +563,7 @@ private void testDeleteEventFiltering3() throws Exception { split = new OrcSplit(acidFilePath, null, stripe.getOffset() + 50, stripe.getLength() - 100, - new String[] {"localhost"}, null, false, true, new ArrayList<>(), + new String[] {"localhost"}, null, false, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, null); validateKeyInterval(split, new RecordIdentifier(1, 1, 1), @@ -572,7 +575,7 @@ private void testDeleteEventFiltering3() throws Exception { split = new OrcSplit(acidFilePath, null, stripe.getOffset(), stripe.getLength() - 50, - new String[] {"localhost"}, null, false, true, new ArrayList<>(), + new String[] {"localhost"}, null, false, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, null); // The key interval for the 1st stripe @@ -588,7 +591,7 @@ private void testDeleteEventFiltering3() throws Exception { split = new OrcSplit(acidFilePath, null, stripe.getOffset(), stripe.getLength() + 50, - new String[] {"localhost"}, null, false, true, new ArrayList<>(), + new String[] {"localhost"}, null, false, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, null); // The key interval for the last 2 stripes @@ -601,7 +604,7 @@ private void testDeleteEventFiltering3() throws Exception { split = new OrcSplit(acidFilePath, null, stripe.getOffset() - 50, stripe.getLength() + 50, - new String[] {"localhost"}, null, false, true, new ArrayList<>(), + new String[] {"localhost"}, null, false, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, null); // The key interval for the last stripe @@ -612,7 +615,7 @@ private void testDeleteEventFiltering3() throws Exception { split = new OrcSplit(acidFilePath, null, stripes.get(0).getOffset() + 50, reader.getContentLength() - 50, - new String[] {"localhost"}, null, false, true, new ArrayList<>(), + new String[] {"localhost"}, null, false, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, null); // The key interval for the last 2 stripes @@ -623,7 +626,7 @@ private void testDeleteEventFiltering3() throws Exception { split = new OrcSplit(acidFilePath, null, stripes.get(0).getOffset(), reader.getContentLength(), - new String[] {"localhost"}, null, false, true, new ArrayList<>(), + new String[] {"localhost"}, null, false, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, null); // The key interval for all 3 stripes @@ -874,7 +877,7 @@ private void testDeleteEventOriginalFiltering2() throws Exception { OrcSplit split = new OrcSplit(originalFilePath, null, stripe.getOffset() + 50, stripe.getLength() - 100, - new String[] {"localhost"}, null, true, true, new ArrayList<>(), + new String[] {"localhost"}, null, true, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, syntheticProps); validateKeyInterval(split, new RecordIdentifier(0, bucketProperty, 2), @@ -885,7 +888,7 @@ private void testDeleteEventOriginalFiltering2() throws Exception { split = new OrcSplit(originalFilePath, null, stripe.getOffset() + 50, stripe.getLength() - 100, - new String[] {"localhost"}, null, true, true, new ArrayList<>(), + new String[] {"localhost"}, null, true, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, syntheticProps); validateKeyInterval(split, new RecordIdentifier(0, bucketProperty, 3), @@ -897,7 +900,7 @@ private void testDeleteEventOriginalFiltering2() throws Exception { split = new OrcSplit(originalFilePath, null, stripe.getOffset(), stripe.getLength() - 50, - new String[] {"localhost"}, null, true, true, new ArrayList<>(), + new String[] {"localhost"}, null, true, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, syntheticProps); // The key interval for the 1st stripe @@ -909,7 +912,7 @@ private void testDeleteEventOriginalFiltering2() throws Exception { split = new OrcSplit(originalFilePath, null, stripe.getOffset(), stripe.getLength() + 50, - new String[] {"localhost"}, null, true, true, new ArrayList<>(), + new String[] {"localhost"}, null, true, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, syntheticProps); // The key interval for the last 2 stripes @@ -922,7 +925,7 @@ private void testDeleteEventOriginalFiltering2() throws Exception { split = new OrcSplit(originalFilePath, null, stripe.getOffset() - 50, stripe.getLength() + 50, - new String[] {"localhost"}, null, true, true, new ArrayList<>(), + new String[] {"localhost"}, null, true, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, syntheticProps); // The key interval for the last stripe @@ -933,7 +936,7 @@ private void testDeleteEventOriginalFiltering2() throws Exception { split = new OrcSplit(originalFilePath, null, stripes.get(0).getOffset() + 50, reader.getContentLength() - 50, - new String[] {"localhost"}, null, true, true, new ArrayList<>(), + new String[] {"localhost"}, null, true, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, syntheticProps); // The key interval for the last 2 stripes @@ -944,7 +947,7 @@ private void testDeleteEventOriginalFiltering2() throws Exception { split = new OrcSplit(originalFilePath, null, stripes.get(0).getOffset(), reader.getContentLength(), - new String[] {"localhost"}, null, true, true, new ArrayList<>(), + new String[] {"localhost"}, null, true, true, Lists.newArrayList(new AcidInputFormat.DeltaMetaData()), fileLength, fileLength, root, syntheticProps); // The key interval for all 3 stripes