.../apache/hadoop/hbase/util/ByteBufferUtils.java | 64 +++++++++++++++ .../org/apache/hadoop/hbase/util/UnsafeAccess.java | 8 ++ .../codec/prefixtree/PrefixTreeBlockMeta.java | 96 +++++++++------------- .../hbase/codec/prefixtree/PrefixTreeSeeker.java | 1 + .../codec/prefixtree/decode/DecoderFactory.java | 12 +-- .../prefixtree/decode/PrefixTreeArrayScanner.java | 11 ++- .../codec/prefixtree/decode/PrefixTreeCell.java | 50 +++++++---- .../prefixtree/decode/column/ColumnNodeReader.java | 10 ++- .../prefixtree/decode/column/ColumnReader.java | 4 +- .../codec/prefixtree/decode/row/RowNodeReader.java | 24 +++--- .../decode/timestamp/MvccVersionDecoder.java | 6 +- .../decode/timestamp/TimestampDecoder.java | 6 +- .../apache/hadoop/hbase/util/vint/UFIntTool.java | 12 +++ .../apache/hadoop/hbase/util/vint/UVIntTool.java | 16 ++++ .../apache/hadoop/hbase/util/vint/UVLongTool.java | 16 ++++ .../codec/prefixtree/column/TestColumnBuilder.java | 3 +- .../hbase/codec/prefixtree/row/TestRowEncoder.java | 2 +- .../prefixtree/timestamp/TestTimestampEncoder.java | 3 +- 18 files changed, 240 insertions(+), 104 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java index 33e5cc6..37c680f 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java @@ -632,6 +632,14 @@ public final class ByteBufferUtils { } } + public static byte toByte(ByteBuffer buffer, int offset) { + if (UnsafeAccess.isAvailable()) { + return UnsafeAccess.toByte(buffer, offset); + } else { + return buffer.get(offset); + } + } + /** * Copies the bytes from given array's offset to length part into the given buffer. Puts the bytes * to buffer's current position. This also advances the position in the 'out' buffer by 'length' @@ -674,4 +682,60 @@ public final class ByteBufferUtils { } } } + + public static String toStringBinary(final ByteBuffer b, int off, int len) { + StringBuilder result = new StringBuilder(); + // Just in case we are passed a 'len' that is > buffer length... + if (off >= b.capacity()) + return result.toString(); + if (off + len > b.capacity()) + len = b.capacity() - off; + for (int i = off; i < off + len; ++i) { + int ch = b.get(i) & 0xFF; + if ((ch >= '0' && ch <= '9') || (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') + || " `~!@#$%^&*()-_=+[]{}|;:'\",.<>/?".indexOf(ch) >= 0) { + result.append((char) ch); + } else { + result.append(String.format("\\x%02X", ch)); + } + } + return result.toString(); + } + + /** + * Search sorted array "a" for byte "key". I can't remember if I wrote this or + * copied it from somewhere. (mcorgan) + * + * @param a + * Array to search. Entries must be sorted and unique. + * @param fromIndex + * First index inclusive of "a" to include in the search. + * @param toIndex + * Last index exclusive of "a" to include in the search. + * @param key + * The byte to search for. + * @return The index of key if found. If not found, return -(index + 1), where + * negative indicates "not found" and the "index + 1" handles the "-0" + * case. + */ + public static int unsignedBinarySearch(ByteBuffer a, int fromIndex, int toIndex, byte key) { + int unsignedKey = key & 0xff; + int low = fromIndex; + int high = toIndex - 1; + + while (low <= high) { + int mid = (low + high) >>> 1; + int midVal = ByteBufferUtils.toByte(a, mid) & 0xff; + + if (midVal < unsignedKey) { + low = mid + 1; + } else if (midVal > unsignedKey) { + high = mid - 1; + } else { + return mid; // key found + } + } + return -(low + 1); // key not found. + } + } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAccess.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAccess.java index deb9a1a..368c1c3 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAccess.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/UnsafeAccess.java @@ -91,6 +91,14 @@ public final class UnsafeAccess { } } + public static byte toByte(ByteBuffer buf, int offset) { + if (buf.isDirect()) { + return theUnsafe.getByte(((DirectBuffer) buf).address() + offset); + } else { + return theUnsafe.getByte(buf.array(), BYTE_ARRAY_BASE_OFFSET + buf.arrayOffset() + offset); + } + } + /** * Converts a byte array to an int value considering it was written in big-endian format. * @param bytes byte array diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeBlockMeta.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeBlockMeta.java index 8410cf3..d7f404c 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeBlockMeta.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeBlockMeta.java @@ -25,6 +25,7 @@ import java.nio.ByteBuffer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.codec.prefixtree.encode.other.LongEncoder; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.vint.UVIntTool; import org.apache.hadoop.hbase.util.vint.UVLongTool; @@ -56,8 +57,6 @@ public class PrefixTreeBlockMeta { /**************** transient fields *********************/ - - protected int arrayOffset; protected int bufferOffset; @@ -116,7 +115,6 @@ public class PrefixTreeBlockMeta { public PrefixTreeBlockMeta(InputStream is) throws IOException{ this.version = VERSION; - this.arrayOffset = 0; this.bufferOffset = 0; readVariableBytesFromInputStream(is); } @@ -129,9 +127,8 @@ public class PrefixTreeBlockMeta { } public void initOnBlock(ByteBuffer buffer) { - arrayOffset = buffer.arrayOffset(); bufferOffset = buffer.position(); - readVariableBytesFromArray(buffer.array(), arrayOffset + bufferOffset); + readVariableBytesFromArray(buffer, bufferOffset); } @@ -263,79 +260,79 @@ public class PrefixTreeBlockMeta { numUniqueTags = UVIntTool.getInt(is); } - public void readVariableBytesFromArray(byte[] bytes, int offset) { + public void readVariableBytesFromArray(ByteBuffer buf, int offset) { int position = offset; - version = UVIntTool.getInt(bytes, position); + version = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(version); - numMetaBytes = UVIntTool.getInt(bytes, position); + numMetaBytes = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numMetaBytes); - numKeyValueBytes = UVIntTool.getInt(bytes, position); + numKeyValueBytes = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numKeyValueBytes); - setIncludesMvccVersion(bytes[position]); + setIncludesMvccVersion(ByteBufferUtils.toByte(buf, position)); ++position; - numRowBytes = UVIntTool.getInt(bytes, position); + numRowBytes = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numRowBytes); - numFamilyBytes = UVIntTool.getInt(bytes, position); + numFamilyBytes = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numFamilyBytes); - numQualifierBytes = UVIntTool.getInt(bytes, position); + numQualifierBytes = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numQualifierBytes); - numTagsBytes = UVIntTool.getInt(bytes, position); + numTagsBytes = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numTagsBytes); - numTimestampBytes = UVIntTool.getInt(bytes, position); + numTimestampBytes = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numTimestampBytes); - numMvccVersionBytes = UVIntTool.getInt(bytes, position); + numMvccVersionBytes = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numMvccVersionBytes); - numValueBytes = UVIntTool.getInt(bytes, position); + numValueBytes = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numValueBytes); - nextNodeOffsetWidth = UVIntTool.getInt(bytes, position); + nextNodeOffsetWidth = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(nextNodeOffsetWidth); - familyOffsetWidth = UVIntTool.getInt(bytes, position); + familyOffsetWidth = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(familyOffsetWidth); - qualifierOffsetWidth = UVIntTool.getInt(bytes, position); + qualifierOffsetWidth = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(qualifierOffsetWidth); - tagsOffsetWidth = UVIntTool.getInt(bytes, position); + tagsOffsetWidth = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(tagsOffsetWidth); - timestampIndexWidth = UVIntTool.getInt(bytes, position); + timestampIndexWidth = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(timestampIndexWidth); - mvccVersionIndexWidth = UVIntTool.getInt(bytes, position); + mvccVersionIndexWidth = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(mvccVersionIndexWidth); - valueOffsetWidth = UVIntTool.getInt(bytes, position); + valueOffsetWidth = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(valueOffsetWidth); - valueLengthWidth = UVIntTool.getInt(bytes, position); + valueLengthWidth = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(valueLengthWidth); - rowTreeDepth = UVIntTool.getInt(bytes, position); + rowTreeDepth = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(rowTreeDepth); - maxRowLength = UVIntTool.getInt(bytes, position); + maxRowLength = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(maxRowLength); - maxQualifierLength = UVIntTool.getInt(bytes, position); + maxQualifierLength = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(maxQualifierLength); - maxTagsLength = UVIntTool.getInt(bytes, position); + maxTagsLength = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(maxTagsLength); - minTimestamp = UVLongTool.getLong(bytes, position); + minTimestamp = UVLongTool.getLong(buf, position); position += UVLongTool.numBytes(minTimestamp); - timestampDeltaWidth = UVIntTool.getInt(bytes, position); + timestampDeltaWidth = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(timestampDeltaWidth); - minMvccVersion = UVLongTool.getLong(bytes, position); + minMvccVersion = UVLongTool.getLong(buf, position); position += UVLongTool.numBytes(minMvccVersion); - mvccVersionDeltaWidth = UVIntTool.getInt(bytes, position); + mvccVersionDeltaWidth = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(mvccVersionDeltaWidth); - setAllSameType(bytes[position]); + setAllSameType(ByteBufferUtils.toByte(buf, position)); ++position; - allTypes = bytes[position]; + allTypes = ByteBufferUtils.toByte(buf, position); ++position; - numUniqueRows = UVIntTool.getInt(bytes, position); + numUniqueRows = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numUniqueRows); - numUniqueFamilies = UVIntTool.getInt(bytes, position); + numUniqueFamilies = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numUniqueFamilies); - numUniqueQualifiers = UVIntTool.getInt(bytes, position); + numUniqueQualifiers = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numUniqueQualifiers); - numUniqueTags = UVIntTool.getInt(bytes, position); + numUniqueTags = UVIntTool.getInt(buf, position); position += UVIntTool.numBytes(numUniqueTags); } @@ -405,8 +402,6 @@ public class PrefixTreeBlockMeta { return false; if (allTypes != other.allTypes) return false; - if (arrayOffset != other.arrayOffset) - return false; if (bufferOffset != other.bufferOffset) return false; if (valueLengthWidth != other.valueLengthWidth) @@ -483,7 +478,6 @@ public class PrefixTreeBlockMeta { int result = 1; result = prime * result + (allSameType ? 1231 : 1237); result = prime * result + allTypes; - result = prime * result + arrayOffset; result = prime * result + bufferOffset; result = prime * result + valueLengthWidth; result = prime * result + valueOffsetWidth; @@ -525,9 +519,7 @@ public class PrefixTreeBlockMeta { @Override public String toString() { StringBuilder builder = new StringBuilder(); - builder.append("PtBlockMeta [arrayOffset="); - builder.append(arrayOffset); - builder.append(", bufferOffset="); + builder.append("PtBlockMeta [bufferOffset="); builder.append(bufferOffset); builder.append(", version="); builder.append(version); @@ -602,12 +594,8 @@ public class PrefixTreeBlockMeta { /************** absolute getters *******************/ - public int getAbsoluteMetaOffset() { - return arrayOffset + bufferOffset; - } - public int getAbsoluteRowOffset() { - return getAbsoluteMetaOffset() + numMetaBytes; + return getBufferOffset() + numMetaBytes; } public int getAbsoluteFamilyOffset() { @@ -749,14 +737,6 @@ public class PrefixTreeBlockMeta { this.numMetaBytes = numMetaBytes; } - public int getArrayOffset() { - return arrayOffset; - } - - public void setArrayOffset(int arrayOffset) { - this.arrayOffset = arrayOffset; - } - public int getBufferOffset() { return bufferOffset; } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java index 8a45f13..94c3b97 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java @@ -101,6 +101,7 @@ public class PrefixTreeSeeker implements EncodedSeeker { if (cell == null) { return null; } + // TODO : Create cells for blocks backed by DBBs return new ClonedPrefixTreeCell(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength(), cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(), diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/DecoderFactory.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/DecoderFactory.java index f8f7c99..c8da3a3 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/DecoderFactory.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/DecoderFactory.java @@ -23,20 +23,22 @@ import java.nio.ByteBuffer; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta; import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellSearcher; - +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; /** * Static wrapper class for the ArraySearcherPool. */ @InterfaceAudience.Private public class DecoderFactory { - + private static final Log LOG = LogFactory.getLog(DecoderFactory.class); private static final ArraySearcherPool POOL = new ArraySearcherPool(); //TODO will need a PrefixTreeSearcher on top of CellSearcher public static PrefixTreeArraySearcher checkOut(final ByteBuffer buffer, boolean includeMvccVersion) { if (buffer.isDirect()) { - throw new IllegalArgumentException("DirectByteBuffers not supported yet"); + // This won't happen now. when we have DBB backed blocks we will have this scenario + LOG.trace("DirectByteBuffers passed to Prefix Tree searcher"); // TODO implement PtByteBufferBlockScanner } @@ -66,7 +68,7 @@ public class DecoderFactory { searcher = new PrefixTreeArraySearcher(blockMeta, blockMeta.getRowTreeDepth(), blockMeta.getMaxRowLength(), blockMeta.getMaxQualifierLength(), blockMeta.getMaxTagsLength()); - searcher.initOnBlock(blockMeta, buffer.array(), includeMvccVersion); + searcher.initOnBlock(blockMeta, buffer, includeMvccVersion); return searcher; } @@ -83,7 +85,7 @@ public class DecoderFactory { qualifierBufferLength, tagBufferLength); } //this is where we parse the BlockMeta - searcher.initOnBlock(blockMeta, buffer.array(), includeMvccVersion); + searcher.initOnBlock(blockMeta, buffer, includeMvccVersion); return searcher; } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java index 1e91eb2..aba1543 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.codec.prefixtree.decode; +import java.nio.ByteBuffer; + import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellScanner; @@ -27,6 +29,7 @@ import org.apache.hadoop.hbase.codec.prefixtree.decode.row.RowNodeReader; import org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp.MvccVersionDecoder; import org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp.TimestampDecoder; import org.apache.hadoop.hbase.codec.prefixtree.encode.other.ColumnNodeType; +import org.apache.hadoop.hbase.util.ByteBufferUtils; /** * Extends PtCell and manipulates its protected fields. Could alternatively contain a PtCell and @@ -103,7 +106,7 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne return true; } - public void initOnBlock(PrefixTreeBlockMeta blockMeta, byte[] block, + public void initOnBlock(PrefixTreeBlockMeta blockMeta, ByteBuffer block, boolean includeMvccVersion) { this.block = block; this.blockMeta = blockMeta; @@ -358,8 +361,8 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne /***************** helper methods **************************/ protected void appendCurrentTokenToRowBuffer() { - System.arraycopy(block, currentRowNode.getTokenArrayOffset(), rowBuffer, rowLength, - currentRowNode.getTokenLength()); + ByteBufferUtils.copyFromBufferToArray(rowBuffer, block, currentRowNode.getTokenArrayOffset(), + rowLength, currentRowNode.getTokenLength()); rowLength += currentRowNode.getTokenLength(); } @@ -502,7 +505,7 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne /**************** getters ***************************/ - public byte[] getTreeBytes() { + public ByteBuffer getTreeBytes() { return block; } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java index f06634c..4fa42af 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.codec.prefixtree.decode; +import java.nio.ByteBuffer; + import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; @@ -25,16 +27,17 @@ import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.SettableSequenceId; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; /** - * As the PrefixTreeArrayScanner moves through the tree bytes, it changes the values in the fields - * of this class so that Cell logic can be applied, but without allocating new memory for every Cell - * iterated through. + * As the PrefixTreeArrayScanner moves through the tree bytes, it changes the + * values in the fields of this class so that Cell logic can be applied, but + * without allocating new memory for every Cell iterated through. */ @InterfaceAudience.Private public class PrefixTreeCell implements Cell, SettableSequenceId, Comparable { - // Create a reference here? Can be removed too + // Create a reference here? Can be removed too protected CellComparator comparator = CellComparator.COMPARATOR; /********************** static **********************/ @@ -46,13 +49,15 @@ public class PrefixTreeCell implements Cell, SettableSequenceId, Comparable= 0) {// found it, but need to adjust for position of fan in overall block return fanIndexInBlock - fanOffset; @@ -269,8 +273,8 @@ public class RowNodeReader { @Override public String toString() { StringBuilder sb = new StringBuilder(); - sb.append("fan:" + Bytes.toStringBinary(block, fanOffset, fanOut)); - sb.append(",token:" + Bytes.toStringBinary(block, tokenOffset, tokenLength)); + sb.append("fan:" + ByteBufferUtils.toStringBinary(block, fanOffset, fanOut)); + sb.append(",token:" + ByteBufferUtils.toStringBinary(block, tokenOffset, tokenLength)); sb.append(",numCells:" + numCells); sb.append(",fanIndex:"+fanIndex); if(fanIndex>=0){ diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/timestamp/MvccVersionDecoder.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/timestamp/MvccVersionDecoder.java index 8d51e92..0cd7944 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/timestamp/MvccVersionDecoder.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/timestamp/MvccVersionDecoder.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp; +import java.nio.ByteBuffer; + import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta; import org.apache.hadoop.hbase.util.vint.UFIntTool; @@ -29,7 +31,7 @@ import org.apache.hadoop.hbase.util.vint.UFIntTool; public class MvccVersionDecoder { protected PrefixTreeBlockMeta blockMeta; - protected byte[] block; + protected ByteBuffer block; /************** construct ***********************/ @@ -37,7 +39,7 @@ public class MvccVersionDecoder { public MvccVersionDecoder() { } - public void initOnBlock(PrefixTreeBlockMeta blockMeta, byte[] block) { + public void initOnBlock(PrefixTreeBlockMeta blockMeta, ByteBuffer block) { this.block = block; this.blockMeta = blockMeta; } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/timestamp/TimestampDecoder.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/timestamp/TimestampDecoder.java index 47a975a..9a75621 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/timestamp/TimestampDecoder.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/timestamp/TimestampDecoder.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp; +import java.nio.ByteBuffer; + import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.codec.prefixtree.PrefixTreeBlockMeta; import org.apache.hadoop.hbase.util.vint.UFIntTool; @@ -29,7 +31,7 @@ import org.apache.hadoop.hbase.util.vint.UFIntTool; public class TimestampDecoder { protected PrefixTreeBlockMeta blockMeta; - protected byte[] block; + protected ByteBuffer block; /************** construct ***********************/ @@ -37,7 +39,7 @@ public class TimestampDecoder { public TimestampDecoder() { } - public void initOnBlock(PrefixTreeBlockMeta blockMeta, byte[] block) { + public void initOnBlock(PrefixTreeBlockMeta blockMeta, ByteBuffer block) { this.block = block; this.blockMeta = blockMeta; } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UFIntTool.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UFIntTool.java index a3da9f0..5bda9da 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UFIntTool.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UFIntTool.java @@ -20,8 +20,10 @@ package org.apache.hadoop.hbase.util.vint; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ByteBufferUtils; /** * UFInt is an abbreviation for Unsigned Fixed-width Integer. @@ -113,4 +115,14 @@ public class UFIntTool { return value; } + public static long fromBytes(final ByteBuffer bytes, final int offset, final int width) { + long value = 0; + value |= ByteBufferUtils.toByte(bytes, offset + 0) & 0xff;// these seem to do ok without casting the byte to int + for (int i = 1; i < width; ++i) { + value <<= 8; + value |= ByteBufferUtils.toByte(bytes, i + offset) & 0xff; + } + return value; + } + } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UVIntTool.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UVIntTool.java index aeebd2c..37c6d7c 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UVIntTool.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UVIntTool.java @@ -21,8 +21,10 @@ package org.apache.hadoop.hbase.util.vint; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ByteBufferUtils; /** * Simple Variable Length Integer encoding. Left bit of 0 means we are on the last byte. If left @@ -99,6 +101,20 @@ public class UVIntTool { return value; } + public static int getInt(ByteBuffer buffer, int offset) { + int value = 0; + for (int i = 0;; ++i) { + byte b = ByteBufferUtils.toByte(buffer, offset + i); + int shifted = BYTE_7_RIGHT_BITS_SET & b;// kill leftmost bit + shifted <<= 7 * i; + value |= shifted; + if (b >= 0) { + break; + } + } + return value; + } + public static int getInt(InputStream is) throws IOException { int value = 0; int i = 0; diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UVLongTool.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UVLongTool.java index b55e0f6..2aedf66 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UVLongTool.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/util/vint/UVLongTool.java @@ -21,8 +21,10 @@ package org.apache.hadoop.hbase.util.vint; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.nio.ByteBuffer; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.util.ByteBufferUtils; /** * Simple Variable Length Integer encoding. Left bit of 0 means we are on the last byte. If left @@ -97,6 +99,20 @@ public class UVLongTool{ return value; } + public static long getLong(ByteBuffer bytes, int offset) { + long value = 0; + for (int i = 0;; ++i) { + byte b = ByteBufferUtils.toByte(bytes, offset + i); + long shifted = BYTE_7_RIGHT_BITS_SET & b;// kill leftmost bit + shifted <<= 7 * i; + value |= shifted; + if (b >= 0) { + break; + }// first bit was 0, so that's the last byte in the VarLong + } + return value; + } + public static long getLong(InputStream is) throws IOException { long value = 0; int i = 0; diff --git a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/column/TestColumnBuilder.java b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/column/TestColumnBuilder.java index c33a953..d22f201 100644 --- a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/column/TestColumnBuilder.java +++ b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/column/TestColumnBuilder.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.codec.prefixtree.column; import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collection; import java.util.List; @@ -103,7 +104,7 @@ public class TestColumnBuilder { bytes = baos.toByteArray(); buffer = new byte[blockMeta.getMaxQualifierLength()]; reader = new ColumnReader(buffer, ColumnNodeType.QUALIFIER); - reader.initOnBlock(blockMeta, bytes); + reader.initOnBlock(blockMeta, ByteBuffer.wrap(bytes)); List builderNodes = Lists.newArrayList(); builder.appendNodes(builderNodes, true, true); diff --git a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestRowEncoder.java b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestRowEncoder.java index ec11551..c02c404 100644 --- a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestRowEncoder.java +++ b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/row/TestRowEncoder.java @@ -99,7 +99,7 @@ public class TestRowEncoder { searcher = new PrefixTreeArraySearcher(blockMetaReader, blockMetaReader.getRowTreeDepth(), blockMetaReader.getMaxRowLength(), blockMetaReader.getMaxQualifierLength(), blockMetaReader.getMaxTagsLength()); - searcher.initOnBlock(blockMetaReader, outputBytes, includeMemstoreTS); + searcher.initOnBlock(blockMetaReader, buffer, includeMemstoreTS); } @Test diff --git a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/timestamp/TestTimestampEncoder.java b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/timestamp/TestTimestampEncoder.java index 65cbcc9..562a131 100644 --- a/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/timestamp/TestTimestampEncoder.java +++ b/hbase-prefix-tree/src/test/java/org/apache/hadoop/hbase/codec/prefixtree/timestamp/TestTimestampEncoder.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.codec.prefixtree.timestamp; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Collection; import org.apache.hadoop.hbase.testclassification.MiscTests; @@ -62,7 +63,7 @@ public class TestTimestampEncoder { blockMeta.setTimestampFields(encoder); bytes = encoder.getByteArray(); decoder = new TimestampDecoder(); - decoder.initOnBlock(blockMeta, bytes); + decoder.initOnBlock(blockMeta, ByteBuffer.wrap(bytes)); } @Test