diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index d79aff2..481c612 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator; import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.io.TagCompressionContext; import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.HFileContext; @@ -74,7 +75,27 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { return internalDecodeKeyValues(source, 0, 0, decodingCtx); } - protected static class SeekerState { + protected static class SeekerStateMembers { + private short rowLength; + private int familyOffset = -1; + private byte familyLength; + private int qualifierOffset = -1; + private int qualifierLength; + private long timestamp; + private byte typeByte; + + @Override + public String toString() { + return "rowLength =" + " " + rowLength + ", familyOffset =" + " " + familyOffset + + ", familyLength =" + " " + familyLength + ", qualifierOffset =" + " " + qualifierOffset + + ", qualifierLength =" + " " + qualifierLength + ", timeStamp =" + " " + timestamp + + ", typeByte =" + " " + typeByte; + } + } + + protected static class SeekerState implements Cell { + protected ByteBuffer currentBuffer; + protected TagCompressionContext tagCompressionContext; protected int valueOffset = -1; protected int keyLength; protected int valueLength; @@ -90,6 +111,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { protected long memstoreTS; protected int nextKvOffset; + protected SeekerStateMembers currentStateMembers = new SeekerStateMembers(); protected boolean isValid() { return valueOffset != -1; @@ -98,7 +120,9 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { protected void invalidate() { valueOffset = -1; tagsCompressedLength = 0; + currentStateMembers = new SeekerStateMembers(); uncompressTags = true; + currentBuffer = null; } protected void ensureSpaceForKey() { @@ -127,6 +151,21 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { } } + protected void extractOffsetsAndLengthsFromKey(byte[] keyBuffer, long memTS) { + currentStateMembers.rowLength = Bytes.toShort(keyBuffer); + currentStateMembers.familyOffset = Bytes.SIZEOF_SHORT + getRowLength() + Bytes.SIZEOF_BYTE; + currentStateMembers.familyLength = keyBuffer[getFamilyOffset() - Bytes.SIZEOF_BYTE]; + currentStateMembers.qualifierOffset = currentStateMembers.familyOffset + + currentStateMembers.familyLength; + currentStateMembers.qualifierLength = keyLength + - (int) KeyValue.getKeyDataStructureSize(currentStateMembers.rowLength, + currentStateMembers.familyLength, 0); + int tsOffset = keyLength - KeyValue.TIMESTAMP_TYPE_SIZE; + currentStateMembers.timestamp = Bytes.toLong(keyBuffer, tsOffset); + currentStateMembers.typeByte = keyBuffer[keyLength - 1]; + memstoreTS = memTS; + } + /** * Copy the state from the next one into this instance (the previous state * placeholder). Used to save the previous state when we are advancing the @@ -146,6 +185,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { keyBuffer, nextState.lastCommonPrefix, nextState.keyLength - nextState.lastCommonPrefix); } + currentStateMembers = nextState.currentStateMembers; valueOffset = nextState.valueOffset; keyLength = nextState.keyLength; @@ -153,8 +193,323 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { lastCommonPrefix = nextState.lastCommonPrefix; nextKvOffset = nextState.nextKvOffset; memstoreTS = nextState.memstoreTS; + currentBuffer = nextState.currentBuffer; + if (nextState.tagCompressionContext != null) { + tagCompressionContext = nextState.tagCompressionContext; + } + } + + @Override + public byte[] getRowArray() { + return keyBuffer; + } + + @Override + public int getRowOffset() { + return Bytes.SIZEOF_SHORT; + } + + @Override + public short getRowLength() { + return currentStateMembers.rowLength; + } + + @Override + public byte[] getFamilyArray() { + return keyBuffer; + } + + @Override + public int getFamilyOffset() { + return currentStateMembers.familyOffset; + } + + @Override + public byte getFamilyLength() { + return currentStateMembers.familyLength; + } + + @Override + public byte[] getQualifierArray() { + return keyBuffer; + } + + @Override + public int getQualifierOffset() { + return currentStateMembers.qualifierOffset; + } + + @Override + public int getQualifierLength() { + return currentStateMembers.qualifierLength; + } + + @Override + public long getTimestamp() { + return currentStateMembers.timestamp; + } + + @Override + public byte getTypeByte() { + return currentStateMembers.typeByte; + } + + @Override + public long getMvccVersion() { + return memstoreTS; + } + + @Override + public byte[] getValueArray() { + return currentBuffer.array(); + } + + @Override + public int getValueOffset() { + return currentBuffer.arrayOffset() + valueOffset; + } + + @Override + public int getValueLength() { + return valueLength; + } + + @Override + public byte[] getTagsArray() { + if (tagCompressionContext != null) { + return tagsBuffer; + } + return currentBuffer.array(); + } + + @Override + public int getTagsOffset() { + if (tagCompressionContext != null) { + return 0; + } + return currentBuffer.arrayOffset() + tagsOffset; + } + + @Override + public short getTagsLength() { + return (short) tagsLength; + } + + @Override + @Deprecated + public byte[] getValue() { + // TODO Auto-generated method stub + return null; + } + + @Override + @Deprecated + public byte[] getFamily() { + // TODO Auto-generated method stub + return null; + } + + @Override + @Deprecated + public byte[] getQualifier() { + // TODO Auto-generated method stub + return null; + } + + @Override + @Deprecated + public byte[] getRow() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String toString() { + KeyValue kv = KeyValueUtil.copyToNewKeyValue(this); + if (kv == null) { + return "null"; + } + return kv.toString(); } + public Cell shallowCopy() { + return new KeyOnlyClonedSeekerState(this); + } + } + + protected static class KeyOnlyClonedSeekerState implements Cell { + private byte[] keyOnlyBuffer; + private ByteBuffer currentBuffer;; + private short rowLength; + private int familyOffset; + private byte familyLength; + private int qualifierOffset; + private int qualifierLength; + private long timestamp; + private byte typeByte; + private int valueOffset; + private int valueLength; + private int tagsLength; + private int tagsOffset; + private byte[] tagsBuffer; + private long memstoreTS; + private TagCompressionContext tagCompressionContext; + + protected KeyOnlyClonedSeekerState(SeekerState s) { + keyOnlyBuffer = new byte[s.keyLength]; + this.currentBuffer = s.currentBuffer; + this.tagCompressionContext = s.tagCompressionContext; + this.rowLength = s.currentStateMembers.rowLength; + this.familyOffset = s.currentStateMembers.familyOffset; + this.familyLength = s.currentStateMembers.familyLength; + this.qualifierOffset = s.currentStateMembers.qualifierOffset; + this.qualifierLength = s.currentStateMembers.qualifierLength; + this.timestamp = s.currentStateMembers.timestamp; + this.typeByte = s.currentStateMembers.typeByte; + this.valueLength = s.valueLength; + this.valueOffset = s.valueOffset; + this.memstoreTS = s.memstoreTS; + this.tagsOffset = s.tagsOffset; + this.tagsLength = s.tagsLength; + System.arraycopy(s.keyBuffer, 0, keyOnlyBuffer, 0, s.keyLength); + if (s.tagCompressionContext != null) { + tagsBuffer = new byte[s.tagsLength]; + System.arraycopy(s.tagsBuffer, 0, tagsBuffer, 0, s.tagsLength); + } + } + + @Override + public byte[] getRowArray() { + return keyOnlyBuffer; + } + + @Override + public byte[] getFamilyArray() { + return keyOnlyBuffer; + } + + @Override + public byte[] getQualifierArray() { + return keyOnlyBuffer; + } + + @Override + public int getRowOffset() { + return Bytes.SIZEOF_SHORT; + } + + @Override + public short getRowLength() { + return rowLength; + } + + @Override + public int getFamilyOffset() { + return familyOffset; + } + + @Override + public byte getFamilyLength() { + return familyLength; + } + + @Override + public int getQualifierOffset() { + return qualifierOffset; + } + + @Override + public int getQualifierLength() { + return qualifierLength; + } + + @Override + public long getTimestamp() { + return timestamp; + } + + @Override + public byte getTypeByte() { + return typeByte; + } + + @Override + public long getMvccVersion() { + return memstoreTS; + } + + @Override + public byte[] getValueArray() { + return currentBuffer.array(); + } + + @Override + public int getValueOffset() { + return currentBuffer.arrayOffset() + valueOffset; + } + + @Override + public int getValueLength() { + return valueLength; + } + + @Override + public byte[] getTagsArray() { + if (tagCompressionContext != null) { + return tagsBuffer; + } + return currentBuffer.array(); + } + + @Override + public int getTagsOffset() { + if (tagCompressionContext != null) { + return 0; + } + return currentBuffer.arrayOffset() + tagsOffset; + } + + @Override + public short getTagsLength() { + return (short) tagsLength; + } + + @Override + @Deprecated + public byte[] getValue() { + // TODO Auto-generated method stub + return null; + } + + @Override + @Deprecated + public byte[] getFamily() { + // TODO Auto-generated method stub + return null; + } + + @Override + @Deprecated + public byte[] getQualifier() { + // TODO Auto-generated method stub + return null; + } + + @Override + @Deprecated + public byte[] getRow() { + // TODO Auto-generated method stub + return null; + } + + @Override + public String toString() { + KeyValue kv = KeyValueUtil.copyToNewKeyValue(this); + if (kv == null) { + return "null"; + } + return kv.toString(); + } } protected abstract static class @@ -208,7 +563,12 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { this.tagCompressionContext.clear(); } currentBuffer = buffer; + current.currentBuffer = currentBuffer; + if(tagCompressionContext != null) { + current.tagCompressionContext = tagCompressionContext; + } decodeFirst(); + current.extractOffsetsAndLengthsFromKey(current.keyBuffer, current.memstoreTS); previous.invalidate(); } @@ -260,11 +620,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { @Override public Cell getKeyValue() { - ByteBuffer kvBuf = getKeyValueBuffer(); - KeyValue kv = new KeyValue(kvBuf.array(), kvBuf.arrayOffset(), kvBuf.array().length - - kvBuf.arrayOffset()); - kv.setMvccVersion(current.memstoreTS); - return kv; + return current.shallowCopy(); } @Override @@ -274,6 +630,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { tagCompressionContext.clear(); } decodeFirst(); + current.extractOffsetsAndLengthsFromKey(current.keyBuffer, current.memstoreTS); previous.invalidate(); } @@ -283,6 +640,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { return false; } decodeNext(); + current.extractOffsetsAndLengthsFromKey(current.keyBuffer, current.memstoreTS); previous.invalidate(); return true; } @@ -416,6 +774,7 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { if (currentBuffer.hasRemaining()) { previous.copyFromNext(current); decodeNext(); + current.extractOffsetsAndLengthsFromKey(current.keyBuffer, current.memstoreTS); } else { break; }