.../io/encoding/BufferedDataBlockEncoder.java | 2 + .../apache/hadoop/hbase/io/hfile/TestSeekTo.java | 95 ++++++++++++++++++++-- 2 files changed, 90 insertions(+), 7 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index 445b897..3958c7c 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -151,6 +151,8 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { lastCommonPrefix = nextState.lastCommonPrefix; nextKvOffset = nextState.nextKvOffset; memstoreTS = nextState.memstoreTS; + tagsOffset = nextState.tagsOffset; + tagsLength = nextState.tagsLength; } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java index c407635..9149b2a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java @@ -19,25 +19,69 @@ package org.apache.hadoop.hbase.io.hfile; import java.io.IOException; - +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.HBaseTestCase; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellUtil; +import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.Tag; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.testclassification.SmallTests; import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Before; +import org.junit.Test; import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.Parameterized.Parameters; /** * Test {@link HFileScanner#seekTo(byte[])} and its variants. */ @Category(SmallTests.class) -public class TestSeekTo extends HBaseTestCase { +@RunWith(Parameterized.class) +public class TestSeekTo { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private final DataBlockEncoding encoding; + + @Parameters + public static Collection parameters() { + List paramList = new ArrayList(); + for (DataBlockEncoding encoding : DataBlockEncoding.values()) { + // Remove after HBASE-13939 + if (encoding != DataBlockEncoding.PREFIX_TREE) { + paramList.add(new Object[] { encoding }); + } + } + return paramList; + } static boolean switchKVs = false; + public TestSeekTo(DataBlockEncoding encoding) { + this.encoding = encoding; + } + + @Before + public void setUp() { + // reset + switchKVs = false; + } + + static KeyValue toKV(String row, TagUsage tagUsage) { if (tagUsage == TagUsage.NO_TAG) { return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), Bytes.toBytes("qualifier"), @@ -52,7 +96,7 @@ public class TestSeekTo extends HBaseTestCase { if (!switchKVs) { switchKVs = true; return new KeyValue(Bytes.toBytes(row), Bytes.toBytes("family"), - Bytes.toBytes("qualifier"), Bytes.toBytes("value")); + Bytes.toBytes("qualifier"), HConstants.LATEST_TIMESTAMP, Bytes.toBytes("value")); } else { switchKVs = false; Tag t = new Tag((byte) 1, "myTag1"); @@ -68,15 +112,17 @@ public class TestSeekTo extends HBaseTestCase { } Path makeNewFile(TagUsage tagUsage) throws IOException { - Path ncTFile = new Path(this.testDir, "basic.hfile"); + Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile"); + FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); + Configuration conf = TEST_UTIL.getConfiguration(); if (tagUsage != TagUsage.NO_TAG) { conf.setInt("hfile.format.version", 3); } else { conf.setInt("hfile.format.version", 2); } - FSDataOutputStream fout = this.fs.create(ncTFile); int blocksize = toKV("a", tagUsage).getLength() * 3; HFileContext context = new HFileContextBuilder().withBlockSize(blocksize) + .withDataBlockEncoding(encoding) .withIncludesTags(true).build(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout) .withFileContext(context) @@ -96,6 +142,7 @@ public class TestSeekTo extends HBaseTestCase { return ncTFile; } + @Test public void testSeekBefore() throws Exception { testSeekBeforeInternals(TagUsage.NO_TAG); testSeekBeforeInternals(TagUsage.ONLY_TAG); @@ -104,6 +151,8 @@ public class TestSeekTo extends HBaseTestCase { protected void testSeekBeforeInternals(TagUsage tagUsage) throws IOException { Path p = makeNewFile(tagUsage); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Configuration conf = TEST_UTIL.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); @@ -129,6 +178,15 @@ public class TestSeekTo extends HBaseTestCase { assertEquals("g", toRowStr(scanner.getKeyValue())); assertEquals(true, scanner.seekBefore(toKV("j", tagUsage).getKey())); assertEquals("i", toRowStr(scanner.getKeyValue())); + Cell cell = scanner.getKeyValue(); + if (tagUsage != TagUsage.NO_TAG && cell.getTagsLength() > 0) { + Iterator tagsIterator = CellUtil.tagsIterator(cell.getTagsArray(), cell.getTagsOffset(), + cell.getTagsLength()); + while (tagsIterator.hasNext()) { + Tag next = tagsIterator.next(); + assertEquals("myTag1", Bytes.toString(next.getValue())); + } + } assertEquals(true, scanner.seekBefore(toKV("k", tagUsage).getKey())); assertEquals("i", toRowStr(scanner.getKeyValue())); assertEquals(true, scanner.seekBefore(toKV("l", tagUsage).getKey())); @@ -137,6 +195,7 @@ public class TestSeekTo extends HBaseTestCase { reader.close(); } + @Test public void testSeekBeforeWithReSeekTo() throws Exception { testSeekBeforeWithReSeekToInternals(TagUsage.NO_TAG); testSeekBeforeWithReSeekToInternals(TagUsage.ONLY_TAG); @@ -145,6 +204,8 @@ public class TestSeekTo extends HBaseTestCase { protected void testSeekBeforeWithReSeekToInternals(TagUsage tagUsage) throws IOException { Path p = makeNewFile(tagUsage); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Configuration conf = TEST_UTIL.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); @@ -224,8 +285,16 @@ public class TestSeekTo extends HBaseTestCase { // reseekTo k assertEquals(0, scanner.reseekTo(toKV("k", tagUsage).getKey())); assertEquals("k", toRowStr(scanner.getKeyValue())); + deleteTestDir(fs); } + protected void deleteTestDir(FileSystem fs) throws IOException { + Path dataTestDir = TEST_UTIL.getDataTestDir(); + if (fs.exists(dataTestDir)) { + fs.delete(dataTestDir, true); + } + } + @Test public void testSeekTo() throws Exception { testSeekToInternals(TagUsage.NO_TAG); testSeekToInternals(TagUsage.ONLY_TAG); @@ -234,6 +303,8 @@ public class TestSeekTo extends HBaseTestCase { protected void testSeekToInternals(TagUsage tagUsage) throws IOException { Path p = makeNewFile(tagUsage); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Configuration conf = TEST_UTIL.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf); reader.loadFileInfo(); assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount()); @@ -249,10 +320,17 @@ public class TestSeekTo extends HBaseTestCase { assertEquals("g", toRowStr(scanner.getKeyValue())); assertEquals(1, scanner.seekTo(toKV("l", tagUsage).getKey())); - assertEquals("k", toRowStr(scanner.getKeyValue())); + if (encoding == DataBlockEncoding.PREFIX_TREE) { + // TODO : Fix this + assertEquals(null, scanner.getKeyValue()); + } else { + assertEquals("k", toRowStr(scanner.getKeyValue())); + } reader.close(); } + + @Test public void testBlockContainingKey() throws Exception { testBlockContainingKeyInternals(TagUsage.NO_TAG); testBlockContainingKeyInternals(TagUsage.ONLY_TAG); @@ -261,6 +339,8 @@ public class TestSeekTo extends HBaseTestCase { protected void testBlockContainingKeyInternals(TagUsage tagUsage) throws IOException { Path p = makeNewFile(tagUsage); + FileSystem fs = TEST_UTIL.getTestFileSystem(); + Configuration conf = TEST_UTIL.getConfiguration(); HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf); reader.loadFileInfo(); HFileBlockIndex.BlockIndexReader blockIndexReader = @@ -290,6 +370,7 @@ public class TestSeekTo extends HBaseTestCase { toKV("l", tagUsage).getKey(), 0, klen)); reader.close(); + deleteTestDir(fs); }