.../apache/hadoop/hbase/CachedOffheapKeyValue.java | 137 +++++++++++++++++++++ .../java/org/apache/hadoop/hbase/CellUtil.java | 2 +- .../hadoop/hbase/MemstoreOffheapKeyValue.java | 50 ++++++++ .../org/apache/hadoop/hbase/OffheapKeyValue.java | 81 +++++------- .../apache/hadoop/hbase/codec/KeyValueCodec.java | 4 +- .../hadoop/hbase/codec/KeyValueCodecWithTags.java | 4 +- .../hadoop/hbase/io/encoding/RowIndexSeekerV1.java | 4 +- .../hbase/util/test/RedundantKVGenerator.java | 8 +- .../apache/hadoop/hbase/TestOffheapKeyValue.java | 8 +- .../hadoop/hbase/io/TestTagCompressionContext.java | 4 +- .../hadoop/hbase/io/hfile/HFileReaderImpl.java | 4 +- .../hbase/io/hfile/TestScannerFromBucketCache.java | 18 +-- .../apache/hadoop/hbase/io/hfile/TestSeekTo.java | 4 +- .../hadoop/hbase/protobuf/TestProtobufUtil.java | 4 +- .../wal/TestWALCellCodecWithCompression.java | 6 +- .../hadoop/hbase/wal/TestWALReaderOnSecureWAL.java | 4 +- 16 files changed, 254 insertions(+), 88 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CachedOffheapKeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CachedOffheapKeyValue.java new file mode 100644 index 0000000..77702b5 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CachedOffheapKeyValue.java @@ -0,0 +1,137 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ByteBufferUtils; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ClassSize; + +/** + * This Cell is an implementation of {@link ByteBufferCell} where the data resides in off heap + * memory. Some of the attributes are cached like rowLen, keyLen. + */ +@InterfaceAudience.Private +public class CachedOffheapKeyValue extends OffheapKeyValue { + + private final short rowLen; + private final int keyLen; + // TODO : See if famLen can be cached or not? + + private static final int FIXED_OVERHEAD = + OffheapKeyValue.FIXED_OVERHEAD + +Bytes.SIZEOF_INT + Bytes.SIZEOF_SHORT; + + public CachedOffheapKeyValue(ByteBuffer buf, int offset, int length, boolean hasTags, + long seqId) { + assert buf.isDirect(); + this.buf = buf; + this.offset = offset; + this.length = length; + rowLen = ByteBufferUtils.toShort(this.buf, this.offset + KeyValue.ROW_OFFSET); + keyLen = ByteBufferUtils.toInt(this.buf, this.offset); + this.hasTags = hasTags; + this.seqId = seqId; + } + + public CachedOffheapKeyValue(ByteBuffer buf, int offset, int length) { + assert buf.isDirect(); + this.buf = buf; + this.offset = offset; + this.length = length; + rowLen = ByteBufferUtils.toShort(this.buf, this.offset + KeyValue.ROW_OFFSET); + keyLen = ByteBufferUtils.toInt(this.buf, this.offset); + int tagsLen = + this.length - (this.keyLen + getValueLength() + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE); + this.hasTags = tagsLen > 0; + } + + @Override + public short getRowLength() { + return this.rowLen; + } + + protected int getFamilyLengthPosition() { + return this.offset + KeyValue.ROW_KEY_OFFSET + rowLen; + } + + protected int getQualifierLength(int rlength, int flength) { + return this.keyLen - (int) KeyValue.getKeyDataStructureSize(rlength, flength, 0); + } + + @Override + public long getTimestamp() { + int offset = getTimestampOffset(this.keyLen); + return ByteBufferUtils.toLong(this.buf, offset); + } + + @Override + public byte getTypeByte() { + return ByteBufferUtils.toByte(this.buf, this.offset + this.keyLen - 1 + KeyValue.ROW_OFFSET); + } + + @Override + public int getTagsLength() { + if (!hasTags) { + return 0; + } + int tagsLen = + this.length - (this.keyLen + getValueLength() + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE); + if (tagsLen > 0) { + // There are some Tag bytes in the byte[]. So reduce 2 bytes which is + // added to denote the tags + // length + tagsLen -= KeyValue.TAGS_LENGTH_SIZE; + } + return tagsLen; + } + + @Override + public ByteBuffer getRowByteBuffer() { + return this.buf; + } + + @Override + public int getValuePosition() { + return this.offset + KeyValue.ROW_OFFSET + this.keyLen; + } + + @Override + public long heapSize() { + return ClassSize.align(FIXED_OVERHEAD + ClassSize.align(length)); + } + + @Override + public int getSerializedSize(boolean withTags) { + if (withTags) { + return this.length; + } + return this.keyLen + this.getValueLength() + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE; + } + + protected int getTimestampOffset() { + return this.offset + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + this.keyLen + - KeyValue.TIMESTAMP_TYPE_SIZE; + } + + @Override + public long heapOverhead() { + return FIXED_OVERHEAD; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index 7a9fe66..ad16607 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -3202,6 +3202,6 @@ public final class CellUtil { newKv.setSequenceId(cell.getSequenceId()); return newKv; } - return new OffheapKeyValue(buf, offset, len, tagsLen > 0, cell.getSequenceId()); + return new MemstoreOffheapKeyValue(buf, offset, len, tagsLen > 0, cell.getSequenceId()); } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/MemstoreOffheapKeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/MemstoreOffheapKeyValue.java new file mode 100644 index 0000000..8e62797 --- /dev/null +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/MemstoreOffheapKeyValue.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import java.nio.ByteBuffer; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ByteBufferUtils; + +/** + * This Cell is an implementation of {@link ByteBufferCell} where the data resides in off heap + * memory when the memstore is configured to work with offheap + */ +@InterfaceAudience.Private +public class MemstoreOffheapKeyValue extends OffheapKeyValue { + public MemstoreOffheapKeyValue(ByteBuffer buf, int offset, int length, boolean hasTags, + long seqId) { + assert buf.isDirect(); + this.buf = buf; + this.offset = offset; + this.length = length; + this.hasTags = hasTags; + this.seqId = seqId; + } + + public MemstoreOffheapKeyValue(ByteBuffer buf, int offset, int length) { + assert buf.isDirect(); + this.buf = buf; + this.offset = offset; + this.length = length; + int tagsLen = this.length - (ByteBufferUtils.toInt(this.buf, this.offset) + getValueLength() + + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE); + this.hasTags = tagsLen > 0; + } +} diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/OffheapKeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/OffheapKeyValue.java index ab1f6ef..c3d07e9 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/OffheapKeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/OffheapKeyValue.java @@ -31,43 +31,18 @@ import org.apache.hadoop.hbase.util.ClassSize; * memory. */ @InterfaceAudience.Private -public class OffheapKeyValue extends ByteBufferCell implements ExtendedCell { - - protected final ByteBuffer buf; - protected final int offset; - protected final int length; - protected final boolean hasTags; - private final short rowLen; - private final int keyLen; - private long seqId = 0; - // TODO : See if famLen can be cached or not? - - private static final int FIXED_OVERHEAD = ClassSize.OBJECT + ClassSize.REFERENCE - + (3 * Bytes.SIZEOF_INT) + Bytes.SIZEOF_SHORT - + Bytes.SIZEOF_BOOLEAN + Bytes.SIZEOF_LONG; - - public OffheapKeyValue(ByteBuffer buf, int offset, int length, boolean hasTags, long seqId) { - assert buf.isDirect(); - this.buf = buf; - this.offset = offset; - this.length = length; - rowLen = ByteBufferUtils.toShort(this.buf, this.offset + KeyValue.ROW_OFFSET); - keyLen = ByteBufferUtils.toInt(this.buf, this.offset); - this.hasTags = hasTags; - this.seqId = seqId; - } +public abstract class OffheapKeyValue extends ByteBufferCell implements ExtendedCell { - public OffheapKeyValue(ByteBuffer buf, int offset, int length) { - assert buf.isDirect(); - this.buf = buf; - this.offset = offset; - this.length = length; - rowLen = ByteBufferUtils.toShort(this.buf, this.offset + KeyValue.ROW_OFFSET); - keyLen = ByteBufferUtils.toInt(this.buf, this.offset); - int tagsLen = this.length - - (this.keyLen + getValueLength() + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE); - this.hasTags = tagsLen > 0; - } + protected ByteBuffer buf; + protected int offset; + protected int length; + protected boolean hasTags; + protected long seqId = 0; + + static final int FIXED_OVERHEAD = ClassSize.OBJECT + ClassSize.REFERENCE + + (2 * Bytes.SIZEOF_INT) // offset, length + + Bytes.SIZEOF_BOOLEAN // hasTags + + Bytes.SIZEOF_LONG; // seqId @Override public byte[] getRowArray() { @@ -81,7 +56,7 @@ public class OffheapKeyValue extends ByteBufferCell implements ExtendedCell { @Override public short getRowLength() { - return this.rowLen; + return ByteBufferUtils.toShort(this.buf, this.offset + KeyValue.ROW_OFFSET); } @Override @@ -99,8 +74,9 @@ public class OffheapKeyValue extends ByteBufferCell implements ExtendedCell { return getFamilyLength(getFamilyLengthPosition()); } - private int getFamilyLengthPosition() { - return this.offset + KeyValue.ROW_KEY_OFFSET + rowLen; + protected int getFamilyLengthPosition() { + return this.offset + KeyValue.ROW_KEY_OFFSET + + ByteBufferUtils.toShort(this.buf, this.offset + KeyValue.ROW_OFFSET); } private byte getFamilyLength(int famLenPos) { @@ -122,23 +98,25 @@ public class OffheapKeyValue extends ByteBufferCell implements ExtendedCell { return getQualifierLength(getRowLength(), getFamilyLength()); } - private int getQualifierLength(int rlength, int flength) { - return this.keyLen - (int) KeyValue.getKeyDataStructureSize(rlength, flength, 0); + protected int getQualifierLength(int rlength, int flength) { + return ByteBufferUtils.toInt(this.buf, this.offset) + - (int) KeyValue.getKeyDataStructureSize(rlength, flength, 0); } @Override public long getTimestamp() { - int offset = getTimestampOffset(this.keyLen); + int offset = getTimestampOffset(ByteBufferUtils.toInt(this.buf, this.offset)); return ByteBufferUtils.toLong(this.buf, offset); } - private int getTimestampOffset(int keyLen) { + protected int getTimestampOffset(int keyLen) { return this.offset + KeyValue.ROW_OFFSET + keyLen - KeyValue.TIMESTAMP_TYPE_SIZE; } @Override public byte getTypeByte() { - return ByteBufferUtils.toByte(this.buf, this.offset + this.keyLen - 1 + KeyValue.ROW_OFFSET); + return ByteBufferUtils.toByte(this.buf, + this.offset + ByteBufferUtils.toInt(this.buf, this.offset) - 1 + KeyValue.ROW_OFFSET); } @Override @@ -180,8 +158,8 @@ public class OffheapKeyValue extends ByteBufferCell implements ExtendedCell { if(!hasTags) { return 0; } - int tagsLen = this.length - - (this.keyLen + getValueLength() + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE); + int tagsLen = this.length - (ByteBufferUtils.toInt(this.buf, this.offset) + getValueLength() + + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE); if (tagsLen > 0) { // There are some Tag bytes in the byte[]. So reduce 2 bytes which is // added to denote the tags @@ -228,7 +206,7 @@ public class OffheapKeyValue extends ByteBufferCell implements ExtendedCell { @Override public int getValuePosition() { - return this.offset + KeyValue.ROW_OFFSET + this.keyLen; + return this.offset + KeyValue.ROW_OFFSET + ByteBufferUtils.toInt(this.buf, this.offset); } @Override @@ -262,7 +240,8 @@ public class OffheapKeyValue extends ByteBufferCell implements ExtendedCell { if (withTags) { return this.length; } - return this.keyLen + this.getValueLength() + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE; + return ByteBufferUtils.toInt(this.buf, this.offset) + this.getValueLength() + + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE; } @Override @@ -281,9 +260,9 @@ public class OffheapKeyValue extends ByteBufferCell implements ExtendedCell { Bytes.SIZEOF_LONG); } - private int getTimestampOffset() { - return this.offset + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + this.keyLen - - KeyValue.TIMESTAMP_TYPE_SIZE; + protected int getTimestampOffset() { + return this.offset + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE + + ByteBufferUtils.toInt(this.buf, this.offset) - KeyValue.TIMESTAMP_TYPE_SIZE; } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java index afe4e11..1f60fd8 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodec.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.NoTagsKeyValue; -import org.apache.hadoop.hbase.OffheapKeyValue; +import org.apache.hadoop.hbase.CachedOffheapKeyValue; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.util.ByteBufferUtils; @@ -111,7 +111,7 @@ public class KeyValueCodec implements Codec { protected Cell createCell(ByteBuffer bb, int pos, int len) { // We know there is not going to be any tags. - return new OffheapKeyValue(bb, pos, len, false, 0); + return new CachedOffheapKeyValue(bb, pos, len, false, 0); } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java index de7f36c..aae913b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/codec/KeyValueCodecWithTags.java @@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; -import org.apache.hadoop.hbase.OffheapKeyValue; +import org.apache.hadoop.hbase.CachedOffheapKeyValue; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.util.ByteBufferUtils; @@ -94,7 +94,7 @@ public class KeyValueCodecWithTags implements Codec { @Override protected Cell createCell(ByteBuffer bb, int pos, int len) { - return new OffheapKeyValue(bb, pos, len); + return new CachedOffheapKeyValue(bb, pos, len); } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java index 9ad098c..8c3a1e4 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/RowIndexSeekerV1.java @@ -25,7 +25,7 @@ import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.OffheapKeyValue; +import org.apache.hadoop.hbase.CachedOffheapKeyValue; import org.apache.hadoop.hbase.SizeCachedKeyValue; import org.apache.hadoop.hbase.SizeCachedNoTagsKeyValue; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -382,7 +382,7 @@ public class RowIndexSeekerV1 extends AbstractEncodedSeeker { currentBuffer.asSubByteBuffer(startOffset, cellBufSize, tmpPair); ByteBuffer buf = tmpPair.getFirst(); if (buf.isDirect()) { - ret = new OffheapKeyValue(buf, tmpPair.getSecond(), cellBufSize, + ret = new CachedOffheapKeyValue(buf, tmpPair.getSecond(), cellBufSize, tagsLength > 0, seqId); } else { if (tagsLength > 0) { diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java index 7dc3d5a..6b67550 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/test/RedundantKVGenerator.java @@ -28,7 +28,7 @@ import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.OffheapKeyValue; +import org.apache.hadoop.hbase.CachedOffheapKeyValue; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.ByteBufferUtils; @@ -370,7 +370,7 @@ public class RedundantKVGenerator { ByteBuffer offheapKVBB = ByteBuffer.allocateDirect(keyValue.getLength()); ByteBufferUtils.copyFromArrayToBuffer(offheapKVBB, keyValue.getBuffer(), keyValue.getOffset(), keyValue.getLength()); - OffheapKeyValue offheapKV = + CachedOffheapKeyValue offheapKV = new ExtendedOffheapKeyValue(offheapKVBB, 0, keyValue.getLength(), true, 0); result.add(offheapKV); } else { @@ -378,7 +378,7 @@ public class RedundantKVGenerator { ByteBuffer offheapKVBB = ByteBuffer.allocateDirect(keyValue.getLength()); ByteBufferUtils.copyFromArrayToBuffer(offheapKVBB, keyValue.getBuffer(), keyValue.getOffset(), keyValue.getLength()); - OffheapKeyValue offheapKV = + CachedOffheapKeyValue offheapKV = new ExtendedOffheapKeyValue(offheapKVBB, 0, keyValue.getLength(), false, 0); result.add(offheapKV); } @@ -389,7 +389,7 @@ public class RedundantKVGenerator { return result; } - static class ExtendedOffheapKeyValue extends OffheapKeyValue { + static class ExtendedOffheapKeyValue extends CachedOffheapKeyValue { public ExtendedOffheapKeyValue(ByteBuffer buf, int offset, int length, boolean hasTags, long seqId) { super(buf, offset, length, hasTags, seqId); diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestOffheapKeyValue.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestOffheapKeyValue.java index ec44408..66f1d0f 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestOffheapKeyValue.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestOffheapKeyValue.java @@ -56,7 +56,7 @@ public class TestOffheapKeyValue { KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0l, Type.Put, row1); ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length); ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length); - ByteBufferCell offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l); + ByteBufferCell offheapKV = new CachedOffheapKeyValue(buf, 0, buf.capacity(), false, 0l); assertEquals( ROW1, ByteBufferUtils.toStringBinary(offheapKV.getRowByteBuffer(), @@ -99,7 +99,7 @@ public class TestOffheapKeyValue { kvCell = new KeyValue(row1, fam2, qual2, 0l, Type.Put, row1); buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length); ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length); - offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l); + offheapKV = new CachedOffheapKeyValue(buf, 0, buf.capacity(), false, 0l); assertEquals( FAM2, ByteBufferUtils.toStringBinary(offheapKV.getFamilyByteBuffer(), @@ -112,7 +112,7 @@ public class TestOffheapKeyValue { kvCell = new KeyValue(row1, fam1, nullQualifier, 0L, Type.Put, row1); buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length); ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length); - offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), false, 0l); + offheapKV = new CachedOffheapKeyValue(buf, 0, buf.capacity(), false, 0l); assertEquals( ROW1, ByteBufferUtils.toStringBinary(offheapKV.getRowByteBuffer(), @@ -138,7 +138,7 @@ public class TestOffheapKeyValue { KeyValue kvCell = new KeyValue(row1, fam1, qual1, 0l, Type.Put, row1, tags); ByteBuffer buf = ByteBuffer.allocateDirect(kvCell.getBuffer().length); ByteBufferUtils.copyFromArrayToBuffer(buf, kvCell.getBuffer(), 0, kvCell.getBuffer().length); - ByteBufferCell offheapKV = new OffheapKeyValue(buf, 0, buf.capacity(), true, 0l); + ByteBufferCell offheapKV = new CachedOffheapKeyValue(buf, 0, buf.capacity(), true, 0l); assertEquals( ROW1, ByteBufferUtils.toStringBinary(offheapKV.getRowByteBuffer(), diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java index 0a14443..abe79ba 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/io/TestTagCompressionContext.java @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.OffheapKeyValue; +import org.apache.hadoop.hbase.CachedOffheapKeyValue; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.ArrayBackedTag; import org.apache.hadoop.hbase.ByteBufferCell; @@ -166,7 +166,7 @@ public class TestTagCompressionContext { KeyValue kv = new KeyValue(ROW, CF, Q, 1234L, V, tags); ByteBuffer dbb = ByteBuffer.allocateDirect(kv.getBuffer().length); ByteBufferUtils.copyFromArrayToBuffer(dbb, kv.getBuffer(), 0, kv.getBuffer().length); - OffheapKeyValue offheapKV = new OffheapKeyValue(dbb, 0, kv.getBuffer().length, true, 0); + CachedOffheapKeyValue offheapKV = new CachedOffheapKeyValue(dbb, 0, kv.getBuffer().length, true, 0); return offheapKV; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java index 4cf1bf2..41eca26 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderImpl.java @@ -36,7 +36,7 @@ import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.OffheapKeyValue; +import org.apache.hadoop.hbase.CachedOffheapKeyValue; import org.apache.hadoop.hbase.SizeCachedKeyValue; import org.apache.hadoop.hbase.SizeCachedNoTagsKeyValue; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -957,7 +957,7 @@ public class HFileReaderImpl implements HFile.Reader, Configurable { } else { ByteBuffer buf = blockBuffer.asSubByteBuffer(cellBufSize); if (buf.isDirect()) { - ret = new OffheapKeyValue(buf, buf.position(), cellBufSize, currTagsLen > 0, seqId); + ret = new CachedOffheapKeyValue(buf, buf.position(), cellBufSize, currTagsLen > 0, seqId); } else { if (currTagsLen > 0) { ret = new SizeCachedKeyValue(buf.array(), buf.arrayOffset() + buf.position(), diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java index 898f3bb..ded9988 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestScannerFromBucketCache.java @@ -35,7 +35,7 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.OffheapKeyValue; +import org.apache.hadoop.hbase.CachedOffheapKeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Durability; import org.apache.hadoop.hbase.client.Put; @@ -119,14 +119,14 @@ public class TestScannerFromBucketCache { List actual = performScan(row1, fam1); // Verify result for (int i = 0; i < expected.size(); i++) { - assertFalse(actual.get(i) instanceof OffheapKeyValue); + assertFalse(actual.get(i) instanceof CachedOffheapKeyValue); assertTrue(CellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); } // do the scan again and verify. This time it should be from the lru cache actual = performScan(row1, fam1); // Verify result for (int i = 0; i < expected.size(); i++) { - assertFalse(actual.get(i) instanceof OffheapKeyValue); + assertFalse(actual.get(i) instanceof CachedOffheapKeyValue); assertTrue(CellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); } @@ -157,7 +157,7 @@ public class TestScannerFromBucketCache { List actual = performScan(row1, fam1); // Verify result for (int i = 0; i < expected.size(); i++) { - assertFalse(actual.get(i) instanceof OffheapKeyValue); + assertFalse(actual.get(i) instanceof CachedOffheapKeyValue); assertTrue(CellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); } // Wait for the bucket cache threads to move the data to offheap @@ -166,7 +166,7 @@ public class TestScannerFromBucketCache { actual = performScan(row1, fam1); // Verify result for (int i = 0; i < expected.size(); i++) { - assertTrue(actual.get(i) instanceof OffheapKeyValue); + assertTrue(actual.get(i) instanceof CachedOffheapKeyValue); assertTrue(CellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); } @@ -198,7 +198,7 @@ public class TestScannerFromBucketCache { List actual = performScan(row1, fam1); // Verify result for (int i = 0; i < expected.size(); i++) { - assertFalse(actual.get(i) instanceof OffheapKeyValue); + assertFalse(actual.get(i) instanceof CachedOffheapKeyValue); assertTrue(CellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); } // Wait for the bucket cache threads to move the data to offheap @@ -218,7 +218,7 @@ public class TestScannerFromBucketCache { if (i != 5) { // the last cell fetched will be of type shareable but not offheap because // the MBB is copied to form a single cell - assertTrue(actual.get(i) instanceof OffheapKeyValue); + assertTrue(actual.get(i) instanceof CachedOffheapKeyValue); } } @@ -250,14 +250,14 @@ public class TestScannerFromBucketCache { List actual = performScan(row1, fam1); // Verify result for (int i = 0; i < expected.size(); i++) { - assertFalse(actual.get(i) instanceof OffheapKeyValue); + assertFalse(actual.get(i) instanceof CachedOffheapKeyValue); assertTrue(CellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); } // do the scan again and verify. This time it should be from the bucket cache in onheap mode actual = performScan(row1, fam1); // Verify result for (int i = 0; i < expected.size(); i++) { - assertFalse(actual.get(i) instanceof OffheapKeyValue); + assertFalse(actual.get(i) instanceof CachedOffheapKeyValue); assertTrue(CellUtil.equalsIgnoreMvccVersion(expected.get(i), actual.get(i))); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java index 9ec6dc9..6a41b5d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java @@ -41,7 +41,7 @@ import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.OffheapKeyValue; +import org.apache.hadoop.hbase.CachedOffheapKeyValue; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagUtil; import org.apache.hadoop.hbase.ArrayBackedTag; @@ -215,7 +215,7 @@ public class TestSeekTo { // seekBefore d, so the scanner points to c assertTrue(scanner.seekBefore(toKV("d", tagUsage))); - assertFalse(scanner.getCell() instanceof OffheapKeyValue); + assertFalse(scanner.getCell() instanceof CachedOffheapKeyValue); assertEquals("c", toRowStr(scanner.getCell())); // reseekTo e and g assertEquals(0, scanner.reseekTo(toKV("c", tagUsage))); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java index c094ccb..733f0e0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/protobuf/TestProtobufUtil.java @@ -29,7 +29,7 @@ import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.OffheapKeyValue; +import org.apache.hadoop.hbase.CachedOffheapKeyValue; import org.apache.hadoop.hbase.client.Append; import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Get; @@ -331,7 +331,7 @@ public class TestProtobufUtil { kv3.getLength()); ByteBuffer dbb = ByteBuffer.allocateDirect(arr.length); dbb.put(arr); - OffheapKeyValue offheapKV = new OffheapKeyValue(dbb, kv1.getLength(), kv2.getLength()); + CachedOffheapKeyValue offheapKV = new CachedOffheapKeyValue(dbb, kv1.getLength(), kv2.getLength()); CellProtos.Cell cell = org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toCell(offheapKV); Cell newOffheapKV = org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.toCell(cell); assertTrue(CellComparator.COMPARATOR.compare(offheapKV, newOffheapKV) == 0); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALCellCodecWithCompression.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALCellCodecWithCompression.java index ba5bfa3..379c991 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALCellCodecWithCompression.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/wal/TestWALCellCodecWithCompression.java @@ -30,7 +30,7 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.OffheapKeyValue; +import org.apache.hadoop.hbase.CachedOffheapKeyValue; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.TagUtil; import org.apache.hadoop.hbase.ArrayBackedTag; @@ -109,7 +109,7 @@ public class TestWALCellCodecWithCompression { return new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, tags); } - private OffheapKeyValue createOffheapKV(int noOfTags) { + private CachedOffheapKeyValue createOffheapKV(int noOfTags) { byte[] row = Bytes.toBytes("myRow"); byte[] cf = Bytes.toBytes("myCF"); byte[] q = Bytes.toBytes("myQualifier"); @@ -121,6 +121,6 @@ public class TestWALCellCodecWithCompression { KeyValue kv = new KeyValue(row, cf, q, HConstants.LATEST_TIMESTAMP, value, tags); ByteBuffer dbb = ByteBuffer.allocateDirect(kv.getBuffer().length); dbb.put(kv.getBuffer()); - return new OffheapKeyValue(dbb, 0, kv.getBuffer().length); + return new CachedOffheapKeyValue(dbb, 0, kv.getBuffer().length); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java index 0562fd9..88d63e1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/wal/TestWALReaderOnSecureWAL.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.OffheapKeyValue; +import org.apache.hadoop.hbase.CachedOffheapKeyValue; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; import org.apache.hadoop.hbase.shaded.protobuf.generated.ZooKeeperProtos.SplitLogTask.RecoveryMode; @@ -124,7 +124,7 @@ public class TestWALReaderOnSecureWAL { if (offheap) { ByteBuffer bb = ByteBuffer.allocateDirect(kv.getBuffer().length); bb.put(kv.getBuffer()); - OffheapKeyValue offheapKV = new OffheapKeyValue(bb, 0, kv.getLength()); + CachedOffheapKeyValue offheapKV = new CachedOffheapKeyValue(bb, 0, kv.getLength()); kvs.add(offheapKV); } else { kvs.add(kv);