commit 90ecc01a25a433ab870258212bfc150b0037f4a3 Author: Yu Sun Date: Fri Nov 11 08:00:06 2016 +0800 HBASE-17020 keylen in midkey() dont computed correctly Signed-off-by: Yu Li diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java index e24b4f1..0d0a42c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java @@ -355,7 +355,7 @@ public class HFileBlockIndex { int numDataBlocks = b.getInt(); int keyRelOffset = b.getInt(Bytes.SIZEOF_INT * (midKeyEntry + 1)); int keyLen = b.getInt(Bytes.SIZEOF_INT * (midKeyEntry + 2)) - - keyRelOffset; + keyRelOffset - SECONDARY_INDEX_ENTRY_OVERHEAD; int keyOffset = Bytes.SIZEOF_INT * (numDataBlocks + 2) + keyRelOffset + SECONDARY_INDEX_ENTRY_OVERHEAD; targetMidKey = ByteBufferUtils.toBytes(b, keyOffset, keyLen); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java index b3e0ade..6372713 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.compress.Compression; +import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexChunk; import org.apache.hadoop.hbase.io.hfile.HFileBlockIndex.BlockIndexReader; @@ -502,6 +503,73 @@ public class TestHFileBlockIndex { } /** + * to check if looks good when midKey on a leaf index block boundary + * @throws IOException + */ + @Test + public void testMidKeyOnLeafIndexBlockBoundary() throws IOException { + Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), + "hfile_for_midkey"); + int maxChunkSize = 512; + conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, maxChunkSize); + // should open hfile.block.index.cacheonwrite + conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, true); + + CacheConfig cacheConf = new CacheConfig(conf); + BlockCache blockCache = cacheConf.getBlockCache(); + // Evict all blocks that were cached-on-write by the previous invocation. + blockCache.evictBlocksByHfileName(hfilePath.getName()); + // Write the HFile + { + HFileContext meta = new HFileContextBuilder() + .withBlockSize(SMALL_BLOCK_SIZE) + .withCompression(Algorithm.NONE) + .withDataBlockEncoding(DataBlockEncoding.NONE) + .build(); + HFile.Writer writer = + HFile.getWriterFactory(conf, cacheConf) + .withPath(fs, hfilePath) + .withFileContext(meta) + .create(); + Random rand = new Random(19231737); + byte[] family = Bytes.toBytes("f"); + byte[] qualifier = Bytes.toBytes("q"); + int kvNumberToBeWritten = 16; + // the new generated hfile will contain 2 leaf-index blocks and 16 data blocks, + // midkey is just on the boundary of the first leaf-index block + for (int i = 0; i < kvNumberToBeWritten; ++i) { + byte[] row = TestHFileWriterV2.randomOrderedFixedLengthKey(rand, i, 30); + + // Key will be interpreted by KeyValue.KEY_COMPARATOR + KeyValue kv = + new KeyValue(row, family, qualifier, EnvironmentEdgeManager.currentTime(), + TestHFileWriterV2.randomFixedLengthValue(rand, SMALL_BLOCK_SIZE)); + writer.append(kv); + } + writer.close(); + } + + // close hfile.block.index.cacheonwrite + conf.setBoolean(CacheConfig.CACHE_INDEX_BLOCKS_ON_WRITE_KEY, false); + + // Read the HFile + HFile.Reader reader = HFile.createReader(fs, hfilePath, cacheConf, conf); + + boolean hasArrayIndexOutOfBoundsException = false; + try { + // get the mid-key. + reader.midkey(); + } catch (ArrayIndexOutOfBoundsException e) { + hasArrayIndexOutOfBoundsException = true; + } finally { + reader.close(); + } + + // to check if ArrayIndexOutOfBoundsException occured + assertFalse(hasArrayIndexOutOfBoundsException); + } + + /** * Testing block index through the HFile writer/reader APIs. Allows to test * setting index block size through configuration, intermediate-level index * blocks, and caching index blocks on write. diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java index 50e7d92..7c6a65d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java @@ -294,6 +294,25 @@ public class TestHFileWriterV2 { return keyBytes; } + public static byte[] randomOrderedFixedLengthKey(Random rand, int i, int suffixLength) { + StringBuilder k = new StringBuilder(); + + // The fixed-length lexicographically increasing part of the key. + for (int bitIndex = 31; bitIndex >= 0; --bitIndex) { + if ((i & (1 << bitIndex)) == 0) + k.append("a"); + else + k.append("b"); + } + + // A random suffix of the key. + for (int j = 0; j < suffixLength; ++j) + k.append(randomReadableChar(rand)); + + byte[] keyBytes = k.toString().getBytes(); + return keyBytes; + } + public static byte[] randomValue(Random rand) { StringBuilder v = new StringBuilder(); for (int j = 0; j < 1 + rand.nextInt(2000); ++j) { @@ -304,6 +323,16 @@ public class TestHFileWriterV2 { return valueBytes; } + public static byte[] randomFixedLengthValue(Random rand, int valueLength) { + StringBuilder v = new StringBuilder(); + for (int j = 0; j < valueLength; ++j) { + v.append((char) (32 + rand.nextInt(95))); + } + + byte[] valueBytes = v.toString().getBytes(); + return valueBytes; + } + public static final char randomReadableChar(Random rand) { int i = rand.nextInt(26 * 2 + 10 + 1); if (i < 26)