.../java/org/apache/hadoop/hbase/KeyValueUtil.java | 2 +- .../java/org/apache/hadoop/hbase/TestKeyValue.java | 301 +++++++++++++++++++++ 2 files changed, 302 insertions(+), 1 deletion(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java index dde15bc..7cbfdd6 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java @@ -574,7 +574,7 @@ public class KeyValueUtil { // write value out.write(cell.getValueArray(), cell.getValueOffset(), vlen); // write tags if we have to - if (withTags) { + if (withTags && tlen > 0) { // 2 bytes tags length followed by tags bytes // tags length is serialized with 2 bytes only(short way) even if the type is int. As this // is non -ve numbers, we save the sign bit. See HBASE-11437 diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java index 5daeefb..e2d3023 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestKeyValue.java @@ -18,6 +18,10 @@ */ package org.apache.hadoop.hbase; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.util.Arrays; import java.util.Collections; @@ -634,4 +638,301 @@ public class TestKeyValue extends TestCase { assertNotEquals(kvA1.hashCode(), kvB.hashCode()); } + + public void testKeyValueSerialization() throws Exception { + KeyValue kvA1 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"), + Bytes.toBytes("1")); + KeyValue kvA2 = new KeyValue(Bytes.toBytes("key"), Bytes.toBytes("cf"), Bytes.toBytes("qualA"), + Bytes.toBytes("2")); + MockKeyValue mkvA1 = new MockKeyValue(kvA1.getBuffer(), 0, kvA1.getBuffer().length); + MockKeyValue mkvA2 = new MockKeyValue(kvA2.getBuffer(), 0, kvA2.getBuffer().length); + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + DataOutputStream os = new DataOutputStream(byteArrayOutputStream); + KeyValueUtil.oswrite(mkvA1, os, true); + KeyValueUtil.oswrite(mkvA2, os, true); + DataInputStream is = new DataInputStream(new ByteArrayInputStream( + byteArrayOutputStream.toByteArray())); + KeyValue deSerKV1 = KeyValue.iscreate(is); + assertTrue(kvA1.equals(deSerKV1)); + KeyValue deSerKV2 = KeyValue.iscreate(is); + assertTrue(kvA2.equals(deSerKV2)); + } + + private class MockKeyValue implements Cell { + private final byte[] bytes; + private final int offset; + private final int length; + + public MockKeyValue(byte[] bytes, int offset, int length) { + this.bytes = bytes; + this.offset = offset; + this.length = length; + } + + /** + * This returns the offset where the tag actually starts. + */ + @Override + public int getTagsOffset() { + int tagsLen = getTagsLength(); + if (tagsLen == 0) { + return this.offset + this.length; + } + return this.offset + this.length - tagsLen; + } + + // used to achieve atomic operations in the memstore. + @Override + public long getMvccVersion() { + return this.getSequenceId(); + } + + /** + * used to achieve atomic operations in the memstore. + */ + @Override + public long getSequenceId() { + return 0; + } + + /** + * This returns the total length of the tag bytes + */ + @Override + public int getTagsLength() { + int tagsLen = this.length + - (getKeyLength() + getValueLength() + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE); + if (tagsLen > 0) { + // There are some Tag bytes in the byte[]. So reduce 2 bytes which is + // added to denote the tags + // length + tagsLen -= KeyValue.TAGS_LENGTH_SIZE; + } + return tagsLen; + } + + /** + * + * @return Timestamp + */ + @Override + public long getTimestamp() { + return getTimestamp(getKeyLength()); + } + + /** + * @return KeyValue.TYPE byte representation + */ + @Override + public byte getTypeByte() { + return this.bytes[this.offset + getKeyLength() - 1 + KeyValue.ROW_OFFSET]; + } + + /** + * @param keylength + * Pass if you have it to save on a int creation. + * @return Timestamp + */ + long getTimestamp(final int keylength) { + int tsOffset = getTimestampOffset(keylength); + return Bytes.toLong(this.bytes, tsOffset); + } + + /** + * @return Key offset in backing buffer.. + */ + public int getKeyOffset() { + return this.offset + KeyValue.ROW_OFFSET; + } + + /** + * @return Length of key portion. + */ + public int getKeyLength() { + return Bytes.toInt(this.bytes, this.offset); + } + + /** + * @return the backing array of the entire KeyValue (all KeyValue fields are + * in a single array) + */ + @Override + public byte[] getValueArray() { + return bytes; + } + + /** + * @return the value offset + */ + @Override + public int getValueOffset() { + int voffset = getKeyOffset() + getKeyLength(); + return voffset; + } + + /** + * @return Value length + */ + @Override + public int getValueLength() { + int vlength = Bytes.toInt(this.bytes, this.offset + Bytes.SIZEOF_INT); + return vlength; + } + + /** + * @return the backing array of the entire KeyValue (all KeyValue fields are + * in a single array) + */ + @Override + public byte[] getRowArray() { + return bytes; + } + + /** + * @return Row offset + */ + @Override + public int getRowOffset() { + return getKeyOffset() + Bytes.SIZEOF_SHORT; + } + + /** + * @return Row length + */ + @Override + public short getRowLength() { + return Bytes.toShort(this.bytes, getKeyOffset()); + } + + /** + * @return the backing array of the entire KeyValue (all KeyValue fields are + * in a single array) + */ + @Override + public byte[] getFamilyArray() { + return bytes; + } + + /** + * @return Family offset + */ + @Override + public int getFamilyOffset() { + return getFamilyOffset(getRowLength()); + } + + /** + * @return Family offset + */ + private int getFamilyOffset(int rlength) { + return this.offset + KeyValue.ROW_OFFSET + Bytes.SIZEOF_SHORT + rlength + Bytes.SIZEOF_BYTE; + } + + /** + * @return Family length + */ + @Override + public byte getFamilyLength() { + return getFamilyLength(getFamilyOffset()); + } + + /** + * @return Family length + */ + public byte getFamilyLength(int foffset) { + return this.bytes[foffset - 1]; + } + + /** + * @return the backing array of the entire KeyValue (all KeyValue fields are + * in a single array) + */ + @Override + public byte[] getQualifierArray() { + return bytes; + } + + /** + * @return Qualifier offset + */ + @Override + public int getQualifierOffset() { + return getQualifierOffset(getFamilyOffset()); + } + + /** + * @return Qualifier offset + */ + private int getQualifierOffset(int foffset) { + return foffset + getFamilyLength(foffset); + } + + /** + * @return Qualifier length + */ + @Override + public int getQualifierLength() { + return getQualifierLength(getRowLength(), getFamilyLength()); + } + + /** + * @return Qualifier length + */ + private int getQualifierLength(int rlength, int flength) { + return getKeyLength() - (int) KeyValue.getKeyDataStructureSize(rlength, flength, 0); + } + + /** + * @return Timestamp offset + */ + public int getTimestampOffset() { + return getTimestampOffset(getKeyLength()); + } + + /** + * @param keylength + * Pass if you have it to save on a int creation. + * @return Timestamp offset + */ + private int getTimestampOffset(final int keylength) { + return getKeyOffset() + keylength - KeyValue.TIMESTAMP_TYPE_SIZE; + } + + @Override + @Deprecated + public byte[] getValue() { + // TODO Auto-generated method stub + return null; + } + + @Override + @Deprecated + public byte[] getFamily() { + // TODO Auto-generated method stub + return null; + } + + @Override + @Deprecated + public byte[] getQualifier() { + // TODO Auto-generated method stub + return null; + } + + @Override + @Deprecated + public byte[] getRow() { + // TODO Auto-generated method stub + return null; + } + + /** + * @return the backing array of the entire KeyValue (all KeyValue fields are + * in a single array) + */ + @Override + public byte[] getTagsArray() { + return bytes; + } + } }