.../hbase/io/encoding/CopyKeyDataBlockEncoder.java | 4 ++-- .../hadoop/hbase/io/encoding/DataBlockEncoder.java | 2 +- .../hbase/io/encoding/DiffKeyDeltaEncoder.java | 4 ++-- .../hbase/io/encoding/FastDiffDeltaEncoder.java | 4 ++-- .../hbase/io/encoding/PrefixKeyDeltaEncoder.java | 4 ++-- .../java/org/apache/hadoop/hbase/util/Bytes.java | 2 -- .../hbase/codec/prefixtree/PrefixTreeCodec.java | 4 ++-- .../hbase/codec/prefixtree/PrefixTreeSeeker.java | 4 +--- .../hadoop/hbase/io/hfile/HFileReaderImpl.java | 18 ++++++++-------- .../hbase/io/encoding/TestDataBlockEncoders.java | 24 +++++++++++++++------- 10 files changed, 38 insertions(+), 32 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java index 6fa01c0..d4bfd46 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/CopyKeyDataBlockEncoder.java @@ -66,13 +66,13 @@ public class CopyKeyDataBlockEncoder extends BufferedDataBlockEncoder { } @Override - public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { + public Cell getFirstKeyInBlock(ByteBuffer block) { int keyLength = block.getInt(Bytes.SIZEOF_INT); ByteBuffer dup = block.duplicate(); int pos = 3 * Bytes.SIZEOF_INT; dup.position(pos); dup.limit(pos + keyLength); - return dup.slice(); + return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java index f084689..a67af15 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java @@ -90,7 +90,7 @@ public interface DataBlockEncoder { * @param block encoded block we want index, the position will not change * @return First key in block. */ - ByteBuffer getFirstKeyInBlock(ByteBuffer block); + Cell getFirstKeyInBlock(ByteBuffer block); /** * Create a HFileBlock seeker which find KeyValues within a block. diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java index c55400b..c75eb99 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DiffKeyDeltaEncoder.java @@ -305,7 +305,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { + public Cell getFirstKeyInBlock(ByteBuffer block) { block.mark(); block.position(Bytes.SIZEOF_INT); byte familyLength = block.get(); @@ -350,7 +350,7 @@ public class DiffKeyDeltaEncoder extends BufferedDataBlockEncoder { block.get(result.array(), pos, Bytes.SIZEOF_BYTE); block.reset(); - return result; + return new KeyValue.KeyOnlyKeyValue(result.array(), 0, keyLength); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java index 3000993..857b2d3 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/FastDiffDeltaEncoder.java @@ -354,7 +354,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { + public Cell getFirstKeyInBlock(ByteBuffer block) { block.mark(); block.position(Bytes.SIZEOF_INT + Bytes.SIZEOF_BYTE); int keyLength = ByteBufferUtils.readCompressedInt(block); @@ -365,7 +365,7 @@ public class FastDiffDeltaEncoder extends BufferedDataBlockEncoder { ByteBuffer dup = block.duplicate(); dup.position(pos); dup.limit(pos + keyLength); - return dup.slice(); + return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java index 1350366..436f5ae 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/PrefixKeyDeltaEncoder.java @@ -172,7 +172,7 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder { } @Override - public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { + public Cell getFirstKeyInBlock(ByteBuffer block) { block.mark(); block.position(Bytes.SIZEOF_INT); int keyLength = ByteBufferUtils.readCompressedInt(block); @@ -187,7 +187,7 @@ public class PrefixKeyDeltaEncoder extends BufferedDataBlockEncoder { ByteBuffer dup = block.duplicate(); dup.position(pos); dup.limit(pos + keyLength); - return dup.slice(); + return new KeyValue.KeyOnlyKeyValue(dup.array(), dup.arrayOffset() + pos, keyLength); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java index 683b559..732692d 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java @@ -2114,8 +2114,6 @@ public class Bytes implements Comparable { int low = 0; int high = arr.length - 1; - KeyValue.KeyOnlyKeyValue r = new KeyValue.KeyOnlyKeyValue(); - r.setKey(key, offset, length); while (low <= high) { int mid = (low + high) >>> 1; // we have to compare in this order, because the comparator order diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java index f703eef..ee7e343 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeCodec.java @@ -114,7 +114,7 @@ public class PrefixTreeCodec implements DataBlockEncoder { @Override - public ByteBuffer getFirstKeyInBlock(ByteBuffer block) { + public Cell getFirstKeyInBlock(ByteBuffer block) { block.rewind(); PrefixTreeArraySearcher searcher = null; try { @@ -123,7 +123,7 @@ public class PrefixTreeCodec implements DataBlockEncoder { if (!searcher.positionAtFirstCell()) { return null; } - return KeyValueUtil.copyKeyToNewByteBuffer(searcher.current()); + return searcher.current(); } finally { DecoderFactory.checkIn(searcher); } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java index a4b4c353..bb2afa0 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java @@ -225,10 +225,8 @@ public class PrefixTreeSeeker implements EncodedSeeker { @Override public int compareKey(CellComparator comparator, Cell key) { - // can't optimize this, make a copy of the key - ByteBuffer bb = getKeyDeepCopy(); return comparator.compare(key, - new KeyValue.KeyOnlyKeyValue(bb.array(), bb.arrayOffset(), bb.limit())); + ptSearcher.current()); } /** * Cloned version of the PrefixTreeCell where except the value part, the rest diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index c6655c1..ab85746 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -758,12 +758,10 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { if (seekToBlock == null) { return false; } - ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock); + Cell firstKey = getFirstKeyInBlock(seekToBlock); if (reader.getComparator() - .compareKeyIgnoresMvcc( - new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(), - firstKey.limit()), key) >= 0) { + .compareKeyIgnoresMvcc(firstKey, key) >= 0) { long previousBlockOffset = seekToBlock.getPrevBlockOffset(); // The key we are interested in if (previousBlockOffset == -1) { @@ -780,8 +778,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { // TODO shortcut: seek forward in this block to the last key of the // block. } - Cell firstKeyInCurrentBlock = new KeyValue.KeyOnlyKeyValue(Bytes.getBytes(firstKey)); - loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true); + loadBlockAndSeekToKey(seekToBlock, firstKey, true, key, true); return true; } @@ -1046,7 +1043,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { this.nextIndexedKey = null; } - protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) { + protected Cell getFirstKeyInBlock(HFileBlock curBlock) { ByteBuffer buffer = curBlock.getBufferWithoutHeader(); // It is safe to manipulate this buffer because we own the buffer object. buffer.rewind(); @@ -1055,7 +1052,10 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { ByteBuffer keyBuff = buffer.slice(); keyBuff.limit(klen); keyBuff.rewind(); - return keyBuff; + // Create a KeyOnlyKv now. + // TODO : Will change when Buffer backed cells come + return new KeyValue.KeyOnlyKeyValue(keyBuff.array(), keyBuff.arrayOffset() + + keyBuff.position(), klen); } @Override @@ -1564,7 +1564,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { } } - protected ByteBuffer getFirstKeyInBlock(HFileBlock curBlock) { + protected Cell getFirstKeyInBlock(HFileBlock curBlock) { return dataBlockEncoder.getFirstKeyInBlock(getEncodedBuffer(curBlock)); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java index fbe8164..2d84ca6 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java @@ -30,7 +30,9 @@ import java.util.Collection; import java.util.List; import java.util.Random; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; @@ -300,23 +302,31 @@ public class TestDataBlockEncoders { DataBlockEncoder encoder = encoding.getEncoder(); ByteBuffer encodedBuffer = encodeKeyValues(encoding, sampleKv, getEncodingContext(Compression.Algorithm.NONE, encoding)); - ByteBuffer keyBuffer = encoder.getFirstKeyInBlock(encodedBuffer); + Cell key = encoder.getFirstKeyInBlock(encodedBuffer); + KeyValue keyBuffer = null; + if(encoding == DataBlockEncoding.PREFIX_TREE) { + // This is not an actual case. So the Prefix tree block is not loaded in case of Prefix_tree + // Just copy only the key part to form a keyBuffer + byte[] serializedKey = CellUtil.getCellKeySerializedAsKeyValueKey(key); + keyBuffer = KeyValueUtil.createKeyValueFromKey(serializedKey); + } else { + keyBuffer = KeyValueUtil.ensureKeyValue(key); + } KeyValue firstKv = sampleKv.get(0); - if (0 != Bytes.compareTo(keyBuffer.array(), keyBuffer.arrayOffset(), keyBuffer.limit(), - firstKv.getBuffer(), firstKv.getKeyOffset(), firstKv.getKeyLength())) { + if (0 != CellComparator.COMPARATOR.compareKeyIgnoresMvcc(keyBuffer, firstKv)) { int commonPrefix = 0; - int length = Math.min(keyBuffer.limit(), firstKv.getKeyLength()); + int length = Math.min(keyBuffer.getKeyLength(), firstKv.getKeyLength()); while (commonPrefix < length - && keyBuffer.array()[keyBuffer.arrayOffset() + commonPrefix] == firstKv.getBuffer()[firstKv - .getKeyOffset() + commonPrefix]) { + && keyBuffer.getBuffer()[keyBuffer.getKeyOffset() + commonPrefix] == firstKv + .getBuffer()[firstKv.getKeyOffset() + commonPrefix]) { commonPrefix++; } fail(String.format("Bug in '%s' commonPrefix %d", encoder.toString(), commonPrefix)); } } } - + private void checkSeekingConsistency(List encodedSeekers, boolean seekBefore, KeyValue keyValue) { ByteBuffer expectedKeyValue = null;