.../hbase/io/encoding/CopyKeyDataBlockEncoder.java | 4 ++-- .../hadoop/hbase/io/encoding/DataBlockEncoder.java | 2 +- .../hbase/io/encoding/DiffKeyDeltaEncoder.java | 4 ++-- .../hbase/io/encoding/FastDiffDeltaEncoder.java | 4 ++-- .../hbase/io/encoding/PrefixKeyDeltaEncoder.java | 4 ++-- .../hbase/codec/prefixtree/PrefixTreeCodec.java | 4 ++-- .../hbase/codec/prefixtree/PrefixTreeSeeker.java | 4 +--- .../hadoop/hbase/io/hfile/HFileReaderV2.java | 19 +++++++--------- .../hbase/io/encoding/TestDataBlockEncoders.java | 25 ++++++++++++++++------ .../apache/hadoop/hbase/io/hfile/TestSeekTo.java | 5 +---- 10 files changed, 39 insertions(+), 36 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java index 1c757f5..e52ee6e 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java @@ -66,13 +66,13 @@ public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder { } @Override - public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { + public Cell getFirstKeyInBlock(ByteBuffer block) { int keyLength = block.getInt(Bytes.SIZEOF_INT); ByteBuffer dup = block.duplicate(); int pos = 3 * Bytes.SIZEOF_INT; dup.position(pos); dup.limit(pos + keyLength); - return dup.slice(); + return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java index 0e85380..66d6e58 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java @@ -90,7 +90,7 @@ public interface DataBlockEncoder { * @param block encoded block we want index, the position will not change * @return First key in block. */ - ByteBuffer getFirstKeyInBlock(ByteBuffer block); + Cell getFirstKeyInBlock(ByteBuffer block); /** * Create a HFileBlock seeker which find KeyValues within a block. diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java index 14048e4..af68347 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java @@ -305,7 +305,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { + public Cell getFirstKeyInBlock(ByteBuffer block) { block.mark(); block.position(Bytes.SIZEOF_INT); byte familyLength = block.get(); @@ -350,7 +350,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder { block.get(result.array(), pos, Bytes.SIZEOF_BYTE); block.reset(); - return result; + return new KeyValue.KeyOnlyKeyValue(result.array(), 0, keyLength); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java index 5e28479..4de2d3e 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java @@ -354,7 +354,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { + public Cell getFirstKeyInBlock(ByteBuffer block) { block.mark(); block.position(Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE); int keyLength = ByteBufferUtils.readCompressedInt(block); @@ -365,7 +365,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder { ByteBuffer dup = block.duplicate(); dup.position(pos); dup.limit(pos + keyLength); - return dup.slice(); + return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java index 8c17102..02470fb 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java @@ -172,7 +172,7 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { + public Cell getFirstKeyInBlock(ByteBuffer block) { block.mark(); block.position(Bytes.SIZEOF_INT); int keyLength = ByteBufferUtils.readCompressedInt(block); @@ -187,7 +187,7 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder { ByteBuffer dup = block.duplicate(); dup.position(pos); dup.limit(pos + keyLength); - return dup.slice(); + return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength); } @Override diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java index afcb526..68da713 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java @@ -114,7 +114,7 @@ public class PrefixTreeCodec implements DataBlockEncoder{ @Override - public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { + public Cell getFirstKeyInBlock(ByteBuffer block) { block.rewind(); PrefixTreeArraySearcher searcher = null; try { @@ -123,7 +123,7 @@ public class PrefixTreeCodec implements DataBlockEncoder{ if (!searcher.positionAtFirstCell()) { return null; } - return KeyValueUtil.copyKeyToNewByteBuffer(searcher.current()); + return searcher.current(); } finally { DecoderFactory.checkIn(searcher); } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java index b95055c..4db10cc 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java @@ -259,9 +259,7 @@ public class PrefixTreeSeeker implements EncodedSeeker { @Override public int compareKey(KVComparator comparator, Cell key) { - ByteBuffer bb = getKeyDeepCopy(); - return comparator.compare(key, - new KeyValue.KeyOnlyKeyValue(bb.array(), bb.arrayOffset(), bb.limit())); + return comparator.compare(key, ptSearcher.current()); } /** * Cloned version of the PrefixTreeCell where except the value part, the rest diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java index 1775230..746fd0f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java @@ -562,7 +562,7 @@ public class HFileReaderV2 extends AbstractHFileReader { super(r, cacheBlocks, pread, isCompaction); } - protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock); + protected abstract Cell getFirstKeyInBlock(HFileBlock curBlock); protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, Cell nextIndexedKey, boolean rewind, Cell key, boolean seekBefore) throws IOException; @@ -654,12 +654,9 @@ public class HFileReaderV2 extends AbstractHFileReader { if (seekToBlock == null) { return false; } - ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock); + Cell firstKey = getFirstKeyInBlock(seekToBlock); - if (reader.getComparator() - .compareOnlyKeyPortion( - new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(), - firstKey.limit()), key) >= 0) { + if (reader.getComparator().compareOnlyKeyPortion(firstKey, key) >= 0) { long previousBlockOffset = seekToBlock.getPrevBlockOffset(); // The key we are interested in if (previousBlockOffset == -1) { @@ -676,8 +673,7 @@ public class HFileReaderV2 extends AbstractHFileReader { // TODO shortcut: seek forward in this block to the last key of the // block. } - Cell firstKeyInCurrentBlock = new KeyValue.KeyOnlyKeyValue(Bytes.getBytes(firstKey)); - loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true); + loadBlockAndSeekToKey(seekToBlock, firstKey, true, key, true); return true; } @@ -1117,7 +1113,7 @@ public class HFileReaderV2 extends AbstractHFileReader { } @Override - protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) { + protected Cell getFirstKeyInBlock(HFileBlock curBlock) { ByteBuffer buffer = curBlock.getBufferWithoutHeader(); // It is safe to manipulate this buffer because we own the buffer object. buffer.rewind(); @@ -1126,7 +1122,8 @@ public class HFileReaderV2 extends AbstractHFileReader { ByteBuffer keyBuff = buffer.slice(); keyBuff.limit(klen); keyBuff.rewind(); - return keyBuff; + return new KeyValue.KeyOnlyKeyValue(keyBuff.array(), keyBuff.arrayOffset() + + keyBuff.position(), klen); } @Override @@ -1303,7 +1300,7 @@ public class HFileReaderV2 extends AbstractHFileReader { } @Override - protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) { + protected Cell getFirstKeyInBlock(HFileBlock curBlock) { return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java index 1f3525d..dbcb18b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java @@ -30,6 +30,9 @@ import java.util.Collection; import java.util.List; import java.util.Random; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; @@ -298,23 +301,31 @@ public class TestDataBlockEncoders { DataBlockEncoder encoder = encoding.getEncoder(); ByteBuffer encodedBuffer = encodeKeyValues(encoding, sampleKv, getEncodingContext(Compression.Algorithm.NONE, encoding)); - ByteBuffer keyBuffer = encoder.getFirstKeyInBlock(encodedBuffer); + Cell key = encoder.getFirstKeyInBlock(encodedBuffer); + KeyValue keyBuffer = null; + if(encoding == DataBlockEncoding.PREFIX_TREE) { + // This is not an actual case. So the Prefix tree block is not loaded in case of Prefix_tree + // Just copy only the key part to form a keyBuffer + byte[] serializedKey = CellUtil.getCellKeySerializedAsKeyValueKey(key); + keyBuffer = new KeyValue.KeyOnlyKeyValue(serializedKey); + } else { + keyBuffer = KeyValueUtil.ensureKeyValue(key); + } KeyValue firstKv = sampleKv.get(0); - if (0 != Bytes.compareTo(keyBuffer.array(), keyBuffer.arrayOffset(), keyBuffer.limit(), - firstKv.getBuffer(), firstKv.getKeyOffset(), firstKv.getKeyLength())) { + if (0 != CellComparator.compare(keyBuffer, firstKv, true)) { int commonPrefix = 0; - int length = Math.min(keyBuffer.limit(), firstKv.getKeyLength()); + int length = Math.min(keyBuffer.getKeyLength(), firstKv.getKeyLength()); while (commonPrefix < length - && keyBuffer.array()[keyBuffer.arrayOffset() + commonPrefix] == firstKv.getBuffer()[firstKv - .getKeyOffset() + commonPrefix]) { + && keyBuffer.getBuffer()[keyBuffer.getKeyOffset() + commonPrefix] == firstKv + .getBuffer()[firstKv.getKeyOffset() + commonPrefix]) { commonPrefix++; } fail(String.format("Bug in '%s' commonPrefix %d", encoder.toString(), commonPrefix)); } } } - + private void checkSeekingConsistency(List encodedSeekers, boolean seekBefore, KeyValue keyValue) { ByteBuffer expectedKeyValue = null; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java index a7712b2..1a190fd 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java @@ -64,10 +64,7 @@ public class TestSeekTo { public static Collection parameters() { List paramList = new ArrayList(); for (DataBlockEncoding encoding : DataBlockEncoding.values()) { - // Remove after HBASE-13939 - if (encoding != DataBlockEncoding.PREFIX_TREE) { - paramList.add(new Object[] { encoding }); - } + paramList.add(new Object[] { encoding }); } return paramList; }