.../ipc/TestPayloadCarryingRpcController.java | 43 ++++ .../main/java/org/apache/hadoop/hbase/Cell.java | 18 ++ .../org/apache/hadoop/hbase/CellComparator.java | 264 ++++++++++++++++----- .../java/org/apache/hadoop/hbase/CellUtil.java | 141 ++++++++--- .../java/org/apache/hadoop/hbase/KeyValue.java | 200 ++++++++++++++-- .../java/org/apache/hadoop/hbase/KeyValueUtil.java | 18 +- .../src/main/java/org/apache/hadoop/hbase/Tag.java | 60 ++++- .../io/encoding/BufferedDataBlockEncoder.java | 88 ++++++- .../apache/hadoop/hbase/util/ByteBufferUtils.java | 240 +++++++++++++++++++ .../java/org/apache/hadoop/hbase/util/Bytes.java | 23 +- .../java/org/apache/hadoop/hbase/TestCellUtil.java | 43 ++++ .../hbase/codec/prefixtree/PrefixTreeSeeker.java | 42 ++++ .../codec/prefixtree/decode/PrefixTreeCell.java | 44 ++++ .../hadoop/hbase/io/hfile/HFileReaderV2.java | 32 ++- .../hadoop/hbase/io/hfile/HFileReaderV3.java | 3 +- .../hadoop/hbase/regionserver/ColumnCount.java | 33 ++- .../hadoop/hbase/regionserver/ColumnTracker.java | 12 + .../hbase/regionserver/ExplicitColumnTracker.java | 101 +++++++- .../hbase/regionserver/ScanQueryMatcher.java | 93 ++++++-- .../regionserver/ScanWildcardColumnTracker.java | 83 ++++++- .../hadoop/hbase/regionserver/StoreFile.java | 2 +- .../hadoop/hbase/regionserver/StoreScanner.java | 23 +- 22 files changed, 1430 insertions(+), 176 deletions(-) diff --git a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestPayloadCarryingRpcController.java b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestPayloadCarryingRpcController.java index e6d6f43..fe17219 100644 --- a/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestPayloadCarryingRpcController.java +++ b/hbase-client/src/test/java/org/apache/hadoop/hbase/ipc/TestPayloadCarryingRpcController.java @@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -206,6 +207,48 @@ public class TestPayloadCarryingRpcController { // unused return null; } + + @Override + public ByteBuffer getQualifierBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getRowBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getFamilyBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getValueBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getTagBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public long getTimeStampFromBuffer() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public byte getTypeBytesFromBuffer() { + // TODO Auto-generated method stub + return 0; + } }; } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/Cell.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/Cell.java index 32b4789..79dc8f2 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/Cell.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/Cell.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase; +import java.nio.ByteBuffer; + import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; @@ -235,4 +237,20 @@ public interface Cell { */ @Deprecated byte[] getRow(); + + // This is just a trial to show how the BB will working + ByteBuffer getQualifierBuffer(); + + ByteBuffer getRowBuffer(); + + ByteBuffer getFamilyBuffer(); + + ByteBuffer getValueBuffer(); + + ByteBuffer getTagBuffer(); + + long getTimeStampFromBuffer(); + + byte getTypeBytesFromBuffer(); + } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java index 2f635a4..277a017 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java @@ -19,11 +19,13 @@ package org.apache.hadoop.hbase; import java.io.Serializable; +import java.nio.ByteBuffer; import java.util.Comparator; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import com.google.common.primitives.Longs; @@ -43,6 +45,8 @@ import com.google.common.primitives.Longs; public class CellComparator implements Comparator, Serializable{ private static final long serialVersionUID = -8760041766259623329L; + // TODO : Ideally the cell should indicate whether to do a comparison based on the + // buffer or the byte[]. @Override public int compare(Cell a, Cell b) { return compareStatic(a, b, false); @@ -85,8 +89,14 @@ public class CellComparator implements Comparator, Serializable{ * @return Long.MAX_VALUE if there is no LOG_REPLAY_TAG */ private static long getReplaySeqNum(final Cell c) { - Tag tag = Tag.getTag(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength(), + Tag tag = null; + if(c.getTagsArray() != null) { + tag = Tag.getTag(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength(), TagType.LOG_REPLAY_TAG_TYPE); + } else { + tag = Tag.getTag(c.getTagBuffer(), c.getTagsOffset(), c.getTagsLength(), + TagType.LOG_REPLAY_TAG_TYPE); + } if (tag != null) { return Bytes.toLong(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength()); @@ -95,9 +105,15 @@ public class CellComparator implements Comparator, Serializable{ } public static int findCommonPrefixInRowPart(Cell left, Cell right, int rowCommonPrefix) { - return findCommonPrefix(left.getRowArray(), right.getRowArray(), left.getRowLength() - - rowCommonPrefix, right.getRowLength() - rowCommonPrefix, left.getRowOffset() - + rowCommonPrefix, right.getRowOffset() + rowCommonPrefix); + if(left.getRowArray() != null && right.getRowArray() != null) { + return findCommonPrefix(left.getRowArray(), right.getRowArray(), left.getRowLength() + - rowCommonPrefix, right.getRowLength() - rowCommonPrefix, left.getRowOffset() + + rowCommonPrefix, right.getRowOffset() + rowCommonPrefix); + } else { + return findCommonPrefix(left.getRowBuffer(), right.getRowBuffer(), left.getRowLength() + - rowCommonPrefix, right.getRowLength() - rowCommonPrefix, left.getRowOffset() + + rowCommonPrefix, right.getRowOffset() + rowCommonPrefix); + } } private static int findCommonPrefix(byte[] left, byte[] right, int leftLength, int rightLength, @@ -110,19 +126,44 @@ public class CellComparator implements Comparator, Serializable{ } return result; } + + private static int findCommonPrefix(ByteBuffer left, ByteBuffer right, int leftLength, int rightLength, + int leftOffset, int rightOffset) { + int length = Math.min(leftLength, rightLength); + int result = 0; + + while (result < length && left.get(leftOffset + result) == right.get(rightOffset + result)) { + result++; + } + return result; + } public static int findCommonPrefixInFamilyPart(Cell left, Cell right, int familyCommonPrefix) { - return findCommonPrefix(left.getFamilyArray(), right.getFamilyArray(), left.getFamilyLength() - - familyCommonPrefix, right.getFamilyLength() - familyCommonPrefix, left.getFamilyOffset() - + familyCommonPrefix, right.getFamilyOffset() + familyCommonPrefix); + if(left.getFamilyArray() != null && right.getFamilyArray() != null) { + return findCommonPrefix(left.getFamilyArray(), right.getFamilyArray(), left.getFamilyLength() + - familyCommonPrefix, right.getFamilyLength() - familyCommonPrefix, left.getFamilyOffset() + + familyCommonPrefix, right.getFamilyOffset() + familyCommonPrefix); + } else { + return findCommonPrefix(left.getFamilyBuffer(), right.getFamilyBuffer(), left.getFamilyLength() + - familyCommonPrefix, right.getFamilyLength() - familyCommonPrefix, left.getFamilyOffset() + + familyCommonPrefix, right.getFamilyOffset() + familyCommonPrefix); + + } } public static int findCommonPrefixInQualifierPart(Cell left, Cell right, int qualifierCommonPrefix) { - return findCommonPrefix(left.getQualifierArray(), right.getQualifierArray(), - left.getQualifierLength() - qualifierCommonPrefix, right.getQualifierLength() - - qualifierCommonPrefix, left.getQualifierOffset() + qualifierCommonPrefix, - right.getQualifierOffset() + qualifierCommonPrefix); + if(left.getQualifierArray() != null && right.getQualifierArray() != null) { + return findCommonPrefix(left.getQualifierArray(), right.getQualifierArray(), + left.getQualifierLength() - qualifierCommonPrefix, right.getQualifierLength() + - qualifierCommonPrefix, left.getQualifierOffset() + qualifierCommonPrefix, + right.getQualifierOffset() + qualifierCommonPrefix); + } else { + return findCommonPrefix(left.getQualifierBuffer(), right.getQualifierBuffer(), + left.getQualifierLength() - qualifierCommonPrefix, right.getQualifierLength() + - qualifierCommonPrefix, left.getQualifierOffset() + qualifierCommonPrefix, + right.getQualifierOffset() + qualifierCommonPrefix); + } } /**************** equals ****************************/ @@ -136,21 +177,58 @@ public class CellComparator implements Comparator, Serializable{ } public static boolean equalsRow(Cell a, Cell b){ - return Bytes.equals( - a.getRowArray(), a.getRowOffset(), a.getRowLength(), - b.getRowArray(), b.getRowOffset(), b.getRowLength()); + if (a.getRowArray() != null && b.getRowArray() != null) { + return Bytes.equals(a.getRowArray(), a.getRowOffset(), a.getRowLength(), b.getRowArray(), + b.getRowOffset(), b.getRowLength()); + } else if (((a.getRowArray() != null && a.getRowBuffer() == null) && b.getRowBuffer() != null)) { + return ByteBufferUtils.equals(a.getRowArray(), a.getRowOffset(), a.getRowLength(), + b.getRowBuffer(), b.getRowOffset(), b.getRowLength()); + } else if (a.getRowBuffer() != null && (b.getRowBuffer() == null && b.getRowArray() != null)) { + return ByteBufferUtils.equals(a.getRowBuffer(), a.getRowOffset(), a.getRowLength(), + b.getRowArray(), b.getRowOffset(), b.getRowLength()); + } else { + return ByteBufferUtils.equals(a.getRowBuffer(), a.getRowOffset(), a.getRowLength(), + b.getRowBuffer(), b.getRowOffset(), b.getRowLength()); + } } public static boolean equalsFamily(Cell a, Cell b){ - return Bytes.equals( - a.getFamilyArray(), a.getFamilyOffset(), a.getFamilyLength(), - b.getFamilyArray(), b.getFamilyOffset(), b.getFamilyLength()); + if(a.getFamilyArray() != null && b.getFamilyArray() != null) { + return Bytes.equals( + a.getFamilyArray(), a.getFamilyOffset(), a.getFamilyLength(), + b.getFamilyArray(), b.getFamilyOffset(), b.getFamilyLength()); + } else if (((a.getFamilyArray() != null && a.getFamilyBuffer() == null) && b.getFamilyBuffer() != null)) { + return ByteBufferUtils.equals(a.getFamilyArray(), a.getFamilyOffset(), a.getFamilyLength(), + b.getFamilyBuffer(), b.getFamilyOffset(), b.getFamilyLength()); + } else if (a.getFamilyBuffer() != null && (b.getFamilyBuffer() == null && b.getFamilyArray() != null)) { + return ByteBufferUtils.equals(a.getFamilyBuffer(), a.getFamilyOffset(), a.getFamilyLength(), + b.getFamilyArray(), b.getFamilyOffset(), b.getFamilyLength()); + } else { + return ByteBufferUtils.equals( + a.getFamilyBuffer(), a.getFamilyOffset(), a.getFamilyLength(), + b.getFamilyBuffer(), b.getFamilyOffset(), b.getFamilyLength()); + } } - public static boolean equalsQualifier(Cell a, Cell b){ - return Bytes.equals( - a.getQualifierArray(), a.getQualifierOffset(), a.getQualifierLength(), - b.getQualifierArray(), b.getQualifierOffset(), b.getQualifierLength()); + public static boolean equalsQualifier(Cell a, Cell b) { + if (a.getQualifierArray() != null && b.getQualifierArray() != null) { + return Bytes.equals(a.getQualifierArray(), a.getQualifierOffset(), a.getQualifierLength(), + b.getQualifierArray(), b.getQualifierOffset(), b.getQualifierLength()); + } else if (((a.getQualifierArray() != null && a.getQualifierBuffer() == null) && b + .getQualifierBuffer() != null)) { + return ByteBufferUtils.equals(a.getQualifierArray(), a.getQualifierOffset(), + a.getQualifierLength(), b.getQualifierBuffer(), b.getQualifierOffset(), + b.getQualifierLength()); + } else if (a.getQualifierBuffer() != null + && (b.getQualifierBuffer() == null && b.getQualifierArray() != null)) { + return ByteBufferUtils.equals(a.getQualifierBuffer(), a.getQualifierOffset(), + a.getQualifierLength(), b.getQualifierArray(), b.getQualifierOffset(), + b.getQualifierLength()); + } else { + return ByteBufferUtils.equals(a.getQualifierBuffer(), a.getQualifierOffset(), + a.getQualifierLength(), b.getQualifierBuffer(), b.getQualifierOffset(), + b.getQualifierLength()); + } } public static boolean equalsTimestamp(Cell a, Cell b){ @@ -162,31 +240,57 @@ public class CellComparator implements Comparator, Serializable{ } public static int compareColumns(final Cell left, final Cell right) { - int lfoffset = left.getFamilyOffset(); - int rfoffset = right.getFamilyOffset(); - int lclength = left.getQualifierLength(); - int rclength = right.getQualifierLength(); - int lfamilylength = left.getFamilyLength(); - int rfamilylength = right.getFamilyLength(); - int diff = compare(left.getFamilyArray(), lfoffset, lfamilylength, right.getFamilyArray(), - rfoffset, rfamilylength); + int diff = 0; + diff = compareFamilies(left, right); if (diff != 0) { return diff; } else { - return compare(left.getQualifierArray(), left.getQualifierOffset(), lclength, - right.getQualifierArray(), right.getQualifierOffset(), rclength); + return compareQualifiers(left, right); } } public static int compareFamilies(Cell left, Cell right) { - return Bytes.compareTo(left.getFamilyArray(), left.getFamilyOffset(), left.getFamilyLength(), - right.getFamilyArray(), right.getFamilyOffset(), right.getFamilyLength()); + if (left.getFamilyArray() != null && right.getFamilyArray() != null) { + return Bytes.compareTo(left.getFamilyArray(), left.getFamilyOffset(), + left.getFamilyLength(), right.getFamilyArray(), right.getFamilyOffset(), + right.getFamilyLength()); + } else if (((left.getFamilyArray() != null && left.getFamilyBuffer() == null) && right + .getFamilyBuffer() != null)) { + return ByteBufferUtils.compareTo(left.getFamilyArray(), left.getFamilyOffset(), + left.getFamilyLength(), right.getFamilyBuffer(), right.getFamilyOffset(), + right.getFamilyLength()); + } else if (left.getFamilyBuffer() != null + && (right.getFamilyBuffer() == null && right.getFamilyArray() != null)) { + return ByteBufferUtils.compareTo(left.getFamilyBuffer(), left.getFamilyOffset(), + left.getFamilyLength(), right.getFamilyArray(), right.getFamilyOffset(), + right.getFamilyLength()); + } else { + return ByteBufferUtils.compareTo(left.getFamilyBuffer(), left.getFamilyOffset(), + left.getFamilyLength(), right.getFamilyBuffer(), right.getFamilyOffset(), + right.getFamilyLength()); + } } public static int compareQualifiers(Cell left, Cell right) { - return Bytes.compareTo(left.getQualifierArray(), left.getQualifierOffset(), - left.getQualifierLength(), right.getQualifierArray(), right.getQualifierOffset(), - right.getQualifierLength()); + if (left.getQualifierArray() != null && right.getQualifierArray() != null) { + return Bytes.compareTo(left.getQualifierArray(), left.getQualifierOffset(), + left.getQualifierLength(), right.getQualifierArray(), right.getQualifierOffset(), + right.getQualifierLength()); + } else if (((left.getQualifierArray() != null && left.getQualifierBuffer() == null) && right + .getQualifierBuffer() != null)) { + return ByteBufferUtils.compareTo(left.getQualifierArray(), left.getQualifierOffset(), + left.getQualifierLength(), right.getQualifierBuffer(), right.getQualifierOffset(), + right.getQualifierLength()); + } else if (left.getQualifierBuffer() != null + && (right.getQualifierBuffer() == null && right.getQualifierArray() != null)) { + return ByteBufferUtils.compareTo(left.getQualifierBuffer(), left.getQualifierOffset(), + left.getQualifierLength(), right.getQualifierArray(), right.getQualifierOffset(), + right.getQualifierLength()); + } else { + return ByteBufferUtils.compareTo(left.getQualifierBuffer(), left.getQualifierOffset(), + left.getQualifierLength(), right.getQualifierBuffer(), right.getQualifierOffset(), + right.getQualifierLength()); + } } public int compareFlatKey(Cell left, Cell right) { @@ -198,8 +302,22 @@ public class CellComparator implements Comparator, Serializable{ } public static int compareRows(final Cell left, final Cell right) { - return Bytes.compareTo(left.getRowArray(), left.getRowOffset(), left.getRowLength(), - right.getRowArray(), right.getRowOffset(), right.getRowLength()); + // We will need to add all the 4 conditions here. This is not the right way + // to do + if (left.getRowArray() != null && right.getRowArray() != null) { + return Bytes.compareTo(left.getRowArray(), left.getRowOffset(), left.getRowLength(), + right.getRowArray(), right.getRowOffset(), right.getRowLength()); + } else if (((left.getRowArray() != null && left.getRowBuffer() == null) && right.getRowBuffer() != null)) { + return ByteBufferUtils.compareTo(left.getRowArray(), left.getRowOffset(), + left.getRowLength(), right.getRowBuffer(), right.getRowOffset(), right.getRowLength()); + } else if (left.getRowBuffer() != null + && (right.getRowBuffer() == null && right.getRowArray() != null)) { + return ByteBufferUtils.compareTo(left.getRowBuffer(), left.getRowOffset(), + left.getRowLength(), right.getRowArray(), right.getRowOffset(), right.getRowLength()); + } else { + return ByteBufferUtils.compareTo(left.getRowBuffer(), left.getRowOffset(), + left.getRowLength(), right.getRowBuffer(), right.getRowOffset(), right.getRowLength()); + } } public static int compareRows(byte[] left, int loffset, int llength, byte[] right, int roffset, @@ -220,10 +338,7 @@ public class CellComparator implements Comparator, Serializable{ boolean sameFamilySize = (leftCell.getFamilyLength() == rightCell.getFamilyLength()); if (!sameFamilySize) { // comparing column family is enough. - - return Bytes.compareTo(leftCell.getFamilyArray(), leftCell.getFamilyOffset(), - leftCell.getFamilyLength(), rightCell.getFamilyArray(), rightCell.getFamilyOffset(), - rightCell.getFamilyLength()); + return compareFamilies(leftCell, rightCell); } int diff = compareColumns(leftCell, rightCell); if (diff != 0) return diff; @@ -279,11 +394,28 @@ public class CellComparator implements Comparator, Serializable{ private static int calculateHashForKeyValue(Cell cell) { //pre-calculate the 3 hashes made of byte ranges - int rowHash = Bytes.hashCode(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); - int familyHash = - Bytes.hashCode(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()); - int qualifierHash = Bytes.hashCode(cell.getQualifierArray(), cell.getQualifierOffset(), - cell.getQualifierLength()); + int rowHash = 0; + int familyHash = 0; + int qualifierHash = 0; + if(cell.getRowArray() != null) { + rowHash = Bytes.hashCode(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); + } else { + rowHash = ByteBufferUtils.hashCode(cell.getRowBuffer(), cell.getRowOffset(), cell.getRowLength()); + } + if(cell.getFamilyArray() != null) { + familyHash = + Bytes.hashCode(cell.getFamilyArray(), cell.getFamilyOffset(), cell.getFamilyLength()); + } else { + familyHash = + ByteBufferUtils.hashCode(cell.getFamilyBuffer(), cell.getFamilyOffset(), cell.getFamilyLength()); + } + if(cell.getQualifierArray() != null) { + qualifierHash = Bytes.hashCode(cell.getQualifierArray(), cell.getQualifierOffset(), + cell.getQualifierLength()); + } else { + qualifierHash = ByteBufferUtils.hashCode(cell.getQualifierBuffer(), cell.getQualifierOffset(), + cell.getQualifierLength()); + } //combine the 6 sub-hashes int hash = 31 * rowHash + familyHash; @@ -315,25 +447,45 @@ public class CellComparator implements Comparator, Serializable{ } public static int compareCommonRowPrefix(Cell left, Cell right, int rowCommonPrefix) { - return compare(left.getRowArray(), left.getRowOffset() + rowCommonPrefix, left.getRowLength() + if(left.getRowArray() != null && right.getRowArray() != null) { + return compare(left.getRowArray(), left.getRowOffset() + rowCommonPrefix, left.getRowLength() - rowCommonPrefix, right.getRowArray(), right.getRowOffset() + rowCommonPrefix, right.getRowLength() - rowCommonPrefix); + } else { + return ByteBufferUtils.compareTo(left.getRowBuffer(), left.getRowOffset() + rowCommonPrefix, left.getRowLength() + - rowCommonPrefix, right.getRowBuffer(), right.getRowOffset() + rowCommonPrefix, + right.getRowLength() - rowCommonPrefix); + } } public static int compareCommonFamilyPrefix(Cell left, Cell right, int familyCommonPrefix) { - return compare(left.getFamilyArray(), left.getFamilyOffset() + familyCommonPrefix, - left.getFamilyLength() - familyCommonPrefix, right.getFamilyArray(), - right.getFamilyOffset() + familyCommonPrefix, - right.getFamilyLength() - familyCommonPrefix); + if(left.getFamilyArray() != null && right.getFamilyArray() != null) { + return compare(left.getFamilyArray(), left.getFamilyOffset() + familyCommonPrefix, + left.getFamilyLength() - familyCommonPrefix, right.getFamilyArray(), + right.getFamilyOffset() + familyCommonPrefix, right.getFamilyLength() + - familyCommonPrefix); + } else { + return ByteBufferUtils.compareTo(left.getFamilyBuffer(), left.getFamilyOffset() + familyCommonPrefix, + left.getFamilyLength() - familyCommonPrefix, right.getFamilyBuffer(), + right.getFamilyOffset() + familyCommonPrefix, right.getFamilyLength() + - familyCommonPrefix); + } } public static int compareCommonQualifierPrefix(Cell left, Cell right, int qualCommonPrefix) { - return compare(left.getQualifierArray(), left.getQualifierOffset() + qualCommonPrefix, - left.getQualifierLength() - qualCommonPrefix, right.getQualifierArray(), - right.getQualifierOffset() + qualCommonPrefix, right.getQualifierLength() - - qualCommonPrefix); + if (left.getQualifierArray() != null && right.getQualifierArray() != null) { + return compare(left.getQualifierArray(), left.getQualifierOffset() + qualCommonPrefix, + left.getQualifierLength() - qualCommonPrefix, right.getQualifierArray(), + right.getQualifierOffset() + qualCommonPrefix, right.getQualifierLength() + - qualCommonPrefix); + } else { + return ByteBufferUtils.compareTo(left.getQualifierBuffer(), left.getQualifierOffset() + + qualCommonPrefix, left.getQualifierLength() - qualCommonPrefix, + right.getQualifierBuffer(), right.getQualifierOffset() + qualCommonPrefix, + right.getQualifierLength() - qualCommonPrefix); + } } /***************** special cases ****************************/ diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java index ce0f546..35ee8b8 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellUtil.java @@ -26,9 +26,9 @@ import java.util.List; import java.util.Map.Entry; import java.util.NavigableMap; +import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; -import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.ByteRange; @@ -103,26 +103,50 @@ public final class CellUtil { /******************** copyTo **********************************/ public static int copyRowTo(Cell cell, byte[] destination, int destinationOffset) { - System.arraycopy(cell.getRowArray(), cell.getRowOffset(), destination, destinationOffset, - cell.getRowLength()); + if(cell.getRowArray() != null) { + System.arraycopy(cell.getRowArray(), cell.getRowOffset(), destination, destinationOffset, + cell.getRowLength()); + } else { + cell.getRowBuffer().position( + cell.getRowOffset() - cell.getRowBuffer().arrayOffset()); + cell.getRowBuffer().get(destination, destinationOffset, cell.getRowLength()); + } return destinationOffset + cell.getRowLength(); } public static int copyFamilyTo(Cell cell, byte[] destination, int destinationOffset) { - System.arraycopy(cell.getFamilyArray(), cell.getFamilyOffset(), destination, destinationOffset, - cell.getFamilyLength()); + if (cell.getFamilyArray() != null) { + System.arraycopy(cell.getFamilyArray(), cell.getFamilyOffset(), destination, + destinationOffset, cell.getFamilyLength()); + } else { + cell.getFamilyBuffer().position( + cell.getFamilyOffset() - cell.getFamilyBuffer().arrayOffset()); + cell.getFamilyBuffer().get(destination, destinationOffset, cell.getFamilyLength()); + } return destinationOffset + cell.getFamilyLength(); } public static int copyQualifierTo(Cell cell, byte[] destination, int destinationOffset) { - System.arraycopy(cell.getQualifierArray(), cell.getQualifierOffset(), destination, - destinationOffset, cell.getQualifierLength()); + if (cell.getQualifierArray() != null) { + System.arraycopy(cell.getQualifierArray(), cell.getQualifierOffset(), destination, + destinationOffset, cell.getQualifierLength()); + } else { + cell.getQualifierBuffer().position( + cell.getQualifierOffset() - cell.getQualifierBuffer().arrayOffset()); + cell.getQualifierBuffer().get(destination, destinationOffset, cell.getQualifierLength()); + } return destinationOffset + cell.getQualifierLength(); } public static int copyValueTo(Cell cell, byte[] destination, int destinationOffset) { - System.arraycopy(cell.getValueArray(), cell.getValueOffset(), destination, destinationOffset, - cell.getValueLength()); + if (cell.getValueArray() != null) { + System.arraycopy(cell.getValueArray(), cell.getValueOffset(), destination, destinationOffset, + cell.getValueLength()); + } else { + cell.getValueBuffer().position( + cell.getValueOffset() - cell.getValueBuffer().arrayOffset()); + cell.getValueBuffer().get(destination, destinationOffset, cell.getValueLength()); + } return destinationOffset + cell.getValueLength(); } @@ -134,8 +158,14 @@ public final class CellUtil { * @return position after tags */ public static int copyTagTo(Cell cell, byte[] destination, int destinationOffset) { - System.arraycopy(cell.getTagsArray(), cell.getTagsOffset(), destination, destinationOffset, - cell.getTagsLength()); + if (cell.getTagsArray() != null) { + System.arraycopy(cell.getTagsArray(), cell.getTagsOffset(), destination, destinationOffset, + cell.getTagsLength()); + } else { + cell.getTagBuffer().position( + cell.getTagsOffset() - cell.getTagBuffer().arrayOffset()); + cell.getTagBuffer().get(destination, destinationOffset, cell.getTagsLength()); + } return destinationOffset + cell.getTagsLength(); } @@ -339,44 +369,87 @@ public final class CellUtil { } public static boolean matchingRow(final Cell left, final byte[] buf) { - return Bytes.equals(left.getRowArray(), left.getRowOffset(), left.getRowLength(), buf, 0, - buf.length); + if (left.getRowArray() != null) { + return Bytes.equals(left.getRowArray(), left.getRowOffset(), left.getRowLength(), buf, 0, + buf.length); + } else { + // TODO : Need to find a better way. If in test cases only then no problem + ByteBuffer wrapBuf = ByteBuffer.wrap(buf); + return ByteBufferUtils.equals(left.getRowBuffer(), left.getRowOffset(), left.getRowLength(), + wrapBuf, 0, wrapBuf.capacity()); + } } public static boolean matchingRow(final Cell left, final byte[] buf, final int offset, final int length) { + if(left.getRowArray() != null) { return Bytes.equals(left.getRowArray(), left.getRowOffset(), left.getRowLength(), buf, offset, length); + } else { + ByteBuffer wrapBuf = ByteBuffer.wrap(buf); + return ByteBufferUtils.equals(left.getRowBuffer(), left.getRowOffset(), left.getRowLength(), + wrapBuf, offset, length); + } } public static boolean matchingFamily(final Cell left, final Cell right) { - return Bytes.equals(left.getFamilyArray(), left.getFamilyOffset(), left.getFamilyLength(), - right.getFamilyArray(), right.getFamilyOffset(), right.getFamilyLength()); + if (left.getFamilyArray() != null && right.getFamilyArray() != null) { + return Bytes.equals(left.getFamilyArray(), left.getFamilyOffset(), left.getFamilyLength(), + right.getFamilyArray(), right.getFamilyOffset(), right.getFamilyLength()); + } else { + return ByteBufferUtils.equals(left.getFamilyBuffer(), left.getFamilyOffset(), + left.getFamilyLength(), right.getFamilyBuffer(), right.getFamilyOffset(), + right.getFamilyLength()); + } } public static boolean matchingFamily(final Cell left, final byte[] buf) { - return Bytes.equals(left.getFamilyArray(), left.getFamilyOffset(), left.getFamilyLength(), buf, - 0, buf.length); + if (left.getFamilyArray() != null) { + return Bytes.equals(left.getFamilyArray(), left.getFamilyOffset(), left.getFamilyLength(), + buf, 0, buf.length); + } else { + ByteBuffer wrapBuf = ByteBuffer.wrap(buf); + return ByteBufferUtils.equals(left.getFamilyBuffer(), left.getFamilyOffset(), + left.getFamilyLength(), wrapBuf, 0, wrapBuf.capacity()); + } } public static boolean matchingFamily(final Cell left, final byte[] buf, final int offset, final int length) { - return Bytes.equals(left.getFamilyArray(), left.getFamilyOffset(), left.getFamilyLength(), buf, - offset, length); + if (left.getFamilyArray() != null) { + return Bytes.equals(left.getFamilyArray(), left.getFamilyOffset(), left.getFamilyLength(), + buf, offset, length); + } else { + ByteBuffer wrapBuf = ByteBuffer.wrap(buf); + return ByteBufferUtils.equals(left.getFamilyBuffer(), left.getFamilyOffset(), + left.getFamilyLength(), wrapBuf, offset, length); + } } public static boolean matchingQualifier(final Cell left, final Cell right) { - return Bytes.equals(left.getQualifierArray(), left.getQualifierOffset(), - left.getQualifierLength(), right.getQualifierArray(), right.getQualifierOffset(), - right.getQualifierLength()); + if (left.getQualifierArray() != null && right.getQualifierArray() != null) { + return Bytes.equals(left.getQualifierArray(), left.getQualifierOffset(), + left.getQualifierLength(), right.getQualifierArray(), right.getQualifierOffset(), + right.getQualifierLength()); + } else { + return ByteBufferUtils.equals(left.getQualifierBuffer(), left.getQualifierOffset(), + left.getQualifierLength(), right.getQualifierBuffer(), right.getQualifierOffset(), + right.getQualifierLength()); + } } public static boolean matchingQualifier(final Cell left, final byte[] buf) { if (buf == null) { return left.getQualifierLength() == 0; } - return Bytes.equals(left.getQualifierArray(), left.getQualifierOffset(), - left.getQualifierLength(), buf, 0, buf.length); + if (left.getQualifierArray() != null) { + return Bytes.equals(left.getQualifierArray(), left.getQualifierOffset(), + left.getQualifierLength(), buf, 0, buf.length); + } else { + ByteBuffer wrapBuf = ByteBuffer.wrap(buf); + return ByteBufferUtils.equals(left.getQualifierBuffer(), left.getQualifierOffset(), + left.getQualifierLength(), wrapBuf, 0, wrapBuf.capacity()); + } } public static boolean matchingQualifier(final Cell left, final byte[] buf, final int offset, @@ -384,8 +457,14 @@ public final class CellUtil { if (buf == null) { return left.getQualifierLength() == 0; } - return Bytes.equals(left.getQualifierArray(), left.getQualifierOffset(), - left.getQualifierLength(), buf, offset, length); + if (left.getQualifierArray() != null) { + return Bytes.equals(left.getQualifierArray(), left.getQualifierOffset(), + left.getQualifierLength(), buf, offset, length); + } else { + ByteBuffer wrapBuf = ByteBuffer.wrap(buf); + return ByteBufferUtils.equals(left.getQualifierBuffer(), left.getQualifierOffset(), + left.getQualifierLength(), wrapBuf, offset, length); + } } public static boolean matchingColumn(final Cell left, final byte[] fam, final byte[] qual) { @@ -413,8 +492,14 @@ public final class CellUtil { } public static boolean matchingValue(final Cell left, final byte[] buf) { - return Bytes.equals(left.getValueArray(), left.getValueOffset(), left.getValueLength(), buf, 0, - buf.length); + if (left.getValueArray() != null) { + return Bytes.equals(left.getValueArray(), left.getValueOffset(), left.getValueLength(), buf, + 0, buf.length); + } else { + ByteBuffer wrapBuf = ByteBuffer.wrap(buf); + return ByteBufferUtils.equals(left.getValueBuffer(), left.getValueOffset(), + left.getValueLength(), wrapBuf, 0, wrapBuf.capacity()); + } } /** diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java index 695f1f5..7ba7015 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -39,6 +39,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.util.StreamUtils; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.io.IOUtils; @@ -272,6 +273,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, private int offset = 0; // offset into bytes buffer KV starts at private int length = 0; // length of the KV starting from offset. + private ByteBuffer buffer = null; /** * @return True if a delete type, a {@link KeyValue.Type#Delete} or * a {KeyValue.Type#DeleteFamily} or a {@link KeyValue.Type#DeleteColumn} @@ -342,6 +344,12 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, this.offset = offset; this.length = length; } + + public KeyValue(final ByteBuffer buffer, final int offset, final int length) { + this.buffer = buffer; + this.offset = offset; + this.length = length; + } /** * Creates a KeyValue from the specified byte array, starting at offset, and @@ -578,6 +586,15 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, qlength, timestamp, type, value, voffset, vlength, null); } + public KeyValue(final ByteBuffer row, final int roffset, final int rlength, + final ByteBuffer family, final int foffset, final int flength, + final ByteBuffer qualifier, final int qoffset, final int qlength, + final long timestamp, final Type type, + final ByteBuffer value, final int voffset, final int vlength) { + this(row, roffset, rlength, family, foffset, flength, qualifier, qoffset, + qlength, timestamp, type, value, voffset, vlength, null); + } + /** * Constructs KeyValue structure filled with specified values. Uses the provided buffer as the * data buffer. @@ -612,6 +629,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, final byte [] value, final int voffset, final int vlength, final Tag[] tags) { this.bytes = buffer; + this.buffer = ByteBuffer.wrap(buffer); this.length = writeByteArray(buffer, boffset, row, roffset, rlength, family, foffset, flength, qualifier, qoffset, qlength, @@ -649,6 +667,21 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, this.bytes = createByteArray(row, roffset, rlength, family, foffset, flength, qualifier, qoffset, qlength, timestamp, type, value, voffset, vlength, tags); + this.buffer = ByteBuffer.wrap(bytes); + this.length = bytes.length; + this.offset = 0; + } + + public KeyValue(final Object row, final int roffset, final int rlength, + final Object family, final int foffset, final int flength, + final Object qualifier, final int qoffset, final int qlength, + final long timestamp, final Type type, + final Object value, final int voffset, final int vlength, + final List tags) { + this.bytes = createByteArray(row, roffset, rlength, + family, foffset, flength, qualifier, qoffset, qlength, + timestamp, type, value, voffset, vlength, tags); + this.buffer = ByteBuffer.wrap(bytes); this.length = bytes.length; this.offset = 0; } @@ -679,6 +712,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, this.bytes = createByteArray(row, roffset, rlength, family, foffset, flength, qualifier, qoffset, qlength, timestamp, type, value, voffset, vlength, tags, tagsOffset, tagsLength); + this.buffer = ByteBuffer.wrap(bytes); this.length = bytes.length; this.offset = 0; } @@ -725,6 +759,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, final int vlength, final int tagsLength) { this.bytes = createEmptyByteArray(rlength, flength, qlength, timestamp, type, vlength, tagsLength); + this.buffer = ByteBuffer.wrap(bytes); this.length = bytes.length; this.offset = 0; } @@ -736,6 +771,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, this.bytes = createByteArray(row, roffset, rlength, family, foffset, flength, qualifier, 0, qualifier == null ? 0 : qualifier.remaining(), ts, type, value, 0, value == null ? 0 : value.remaining(), tags); + this.buffer = ByteBuffer.wrap(bytes); this.length = bytes.length; this.offset = 0; } @@ -817,8 +853,8 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, * * @throws IllegalArgumentException an illegal value was passed */ - private static void checkParameters(final byte [] row, final int rlength, - final byte [] family, int flength, int qlength, int vlength) + private static void checkParameters(final Object row, final int rlength, + final Object family, int flength, int qlength, int vlength) throws IllegalArgumentException { if (rlength > Short.MAX_VALUE) { throw new IllegalArgumentException("Row > " + Short.MAX_VALUE); @@ -993,8 +1029,8 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, * @param qualifier can be a ByteBuffer or a byte[], or null. * @param value can be a ByteBuffer or a byte[], or null. */ - private static byte [] createByteArray(final byte [] row, final int roffset, - final int rlength, final byte [] family, final int foffset, int flength, + private static byte [] createByteArray(final Object row, final int roffset, + final int rlength, final Object family, final int foffset, int flength, final Object qualifier, final int qoffset, int qlength, final long timestamp, final Type type, final Object value, final int voffset, int vlength, List tags) { @@ -1013,21 +1049,33 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, int keyLength = (int) getKeyDataStructureSize(rlength, flength, qlength); byte[] bytes = new byte[(int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength, tagsLength)]; + ByteBuffer buffer = ByteBuffer.allocate((int) getKeyValueDataStructureSize(rlength, flength, qlength, vlength, + tagsLength)); // Write key, value and key row length. int pos = 0; pos = Bytes.putInt(bytes, pos, keyLength); + buffer.putInt(keyLength); pos = Bytes.putInt(bytes, pos, vlength); + pos = Bytes.putShort(bytes, pos, (short)(rlength & 0x0000ffff)); - pos = Bytes.putBytes(bytes, pos, row, roffset, rlength); + if(row instanceof ByteBuffer) { + pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) row, roffset, rlength); + } else { + pos = Bytes.putBytes(bytes, pos, (byte[])row, roffset, rlength); + } pos = Bytes.putByte(bytes, pos, (byte)(flength & 0x0000ff)); - if(flength != 0) { - pos = Bytes.putBytes(bytes, pos, family, foffset, flength); + if (flength != 0) { + if (family instanceof ByteBuffer) { + pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) family, foffset, flength); + } else { + pos = Bytes.putBytes(bytes, pos, (byte[]) family, foffset, flength); + } } if (qlength > 0) { if (qualifier instanceof ByteBuffer) { - pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) qualifier); + pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) qualifier, qoffset, qlength); } else { pos = Bytes.putBytes(bytes, pos, (byte[]) qualifier, qoffset, qlength); } @@ -1036,7 +1084,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, pos = Bytes.putByte(bytes, pos, type.getCode()); if (vlength > 0) { if (value instanceof ByteBuffer) { - pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) value); + pos = Bytes.putByteBuffer(bytes, pos, (ByteBuffer) value, voffset, vlength); } else { pos = Bytes.putBytes(bytes, pos, (byte[]) value, voffset, vlength); } @@ -1065,12 +1113,21 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, @Override public int hashCode() { byte[] b = getBuffer(); - int start = getOffset(), end = getOffset() + getLength(); - int h = b[start++]; - for (int i = start; i < end; i++) { - h = (h * 13) ^ b[i]; + if (b != null) { + int start = getOffset(), end = getOffset() + getLength(); + int h = b[start++]; + for (int i = start; i < end; i++) { + h = (h * 13) ^ b[i]; + } + return h; + } else { + int start = getOffset(), end = getOffset() + getLength(); + int h = this.buffer.get(start++); + for (int i = start; i < end; i++) { + h = (h * 13) ^ this.buffer.get(i); + } + return h; } - return h; } //--------------------------------------------------------------------------- @@ -1207,8 +1264,13 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, * @deprecated Since 0.98.0. Use Cell Interface instead. Do not presume single backing buffer. */ @Deprecated - public byte [] getBuffer() { - return this.bytes; + public byte[] getBuffer() { + // For now we can do this. + if (this.bytes != null) { + return this.bytes; + } else { + return this.buffer.array(); + } } /** @@ -1259,7 +1321,14 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, * @return Length of key portion. */ public int getKeyLength() { - return Bytes.toInt(this.bytes, this.offset); + int keyLength; + if (bytes != null) { + keyLength = Bytes.toInt(this.bytes, this.offset); + } else { + keyLength = ByteBufferUtils + .readAsInt(buffer, offset - buffer.arrayOffset(), Bytes.SIZEOF_INT); + } + return keyLength; } /** @@ -1284,7 +1353,12 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, */ @Override public int getValueLength() { - int vlength = Bytes.toInt(this.bytes, this.offset + Bytes.SIZEOF_INT); + int vlength = 0; + if(bytes != null) { + vlength = Bytes.toInt(this.bytes, this.offset + Bytes.SIZEOF_INT); + } else { + vlength = this.buffer.getInt(this.offset + Bytes.SIZEOF_INT - buffer.arrayOffset()); + } return vlength; } @@ -1309,7 +1383,13 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, */ @Override public short getRowLength() { - return Bytes.toShort(this.bytes, getKeyOffset()); + short rowLen = 0; + if (this.bytes != null) { + rowLen = Bytes.toShort(this.bytes, getKeyOffset()); + } else { + rowLen = ByteBufferUtils.readAsShort(buffer, getKeyOffset() - offset, Bytes.SIZEOF_SHORT); + } + return rowLen; } /** @@ -1347,7 +1427,11 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, * @return Family length */ public byte getFamilyLength(int foffset) { + if(this.bytes != null) { return this.bytes[foffset-1]; + } else { + return this.buffer.get(foffset - 1 - offset); + } } /** @@ -1487,7 +1571,12 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, */ @Override public long getTimestamp() { - return getTimestamp(getKeyLength()); + // For now call the getTimeStampBuffer from here + if (bytes != null) { + return getTimestamp(getKeyLength()); + } else { + return getTimeStampFromBuffer(); + } } /** @@ -1512,7 +1601,11 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, */ @Override public byte getTypeByte() { - return this.bytes[this.offset + getKeyLength() - 1 + ROW_OFFSET]; + if (bytes != null) { + return this.bytes[this.offset + getKeyLength() - 1 + ROW_OFFSET]; + } else { + return getTypeBytesFromBuffer(); + } } /** @@ -1931,6 +2024,31 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, byte [] right, int roffset, int rlength) { return Bytes.compareTo(left, loffset, llength, right, roffset, rlength); } + + public int compareRows(ByteBuffer left, int loffset, int llength, + byte [] right, int roffset, int rlength) { + return ByteBufferUtils.compareTo(left, loffset, llength, right, roffset, rlength); + } + + public int compareRows(byte [] left, int loffset, int llength, + ByteBuffer right, int roffset, int rlength) { + return ByteBufferUtils.compareTo(left, loffset, llength, right, roffset, rlength); + } + + /** + * Get the b[],o,l for left and right rowkey portions and compare. + * @param left + * @param loffset + * @param llength + * @param right + * @param roffset + * @param rlength + * @return 0 if equal, <0 if left smaller, >0 if right smaller + */ + public int compareRows(ByteBuffer left, int loffset, int llength, ByteBuffer right, + int roffset, int rlength) { + return ByteBufferUtils.compareTo(left, loffset, llength, right, roffset, rlength); + } int compareColumns(final Cell left, final short lrowlength, final Cell right, final short rrowlength) { @@ -2734,4 +2852,44 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, return super.equals(other); } } + + @Override + public ByteBuffer getQualifierBuffer() { + return buffer; + } + + @Override + public ByteBuffer getRowBuffer() { + return buffer; + } + + @Override + public ByteBuffer getFamilyBuffer() { + return buffer; + } + + @Override + public ByteBuffer getValueBuffer() { + return buffer; + } + + @Override + public ByteBuffer getTagBuffer() { + return buffer; + } + + @Override + public long getTimeStampFromBuffer() { + int keylength = getKeyLength(); + int tsOffset = getTimestampOffset(keylength); + long ts = ByteBufferUtils + .readAsLong(buffer, tsOffset - buffer.arrayOffset(), Bytes.SIZEOF_LONG); + return ts; + } + + @Override + public byte getTypeBytesFromBuffer() { + return this.buffer.get(this.offset + getKeyLength() - 1 + ROW_OFFSET - buffer.arrayOffset()); + } + } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java index 9e969e7..e46665b 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValueUtil.java @@ -24,8 +24,8 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.util.StreamUtils; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; @@ -274,6 +274,13 @@ public class KeyValueUtil { qlength, HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0); } + public static KeyValue createLastOnRow(final ByteBuffer row, final int roffset, final int rlength, + final ByteBuffer family, final int foffset, final int flength, final ByteBuffer qualifier, + final int qoffset, final int qlength) { + return new KeyValue(row, roffset, rlength, family, foffset, flength, qualifier, qoffset, + qlength, HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0); + } + /** * Creates a keyValue for the specified keyvalue larger than or equal to all other possible * KeyValues that have the same row, family, qualifer. Used for reseeking @@ -434,6 +441,15 @@ public class KeyValueUtil { foffset, flength, qualifier, qoffset, qlength, HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0); } + + public static KeyValue createFirstOnRow(final ByteBuffer row, + final int roffset, final int rlength, final ByteBuffer family, + final int foffset, final int flength, final ByteBuffer qualifier, + final int qoffset, final int qlength) { + return new KeyValue(row, roffset, rlength, family, + foffset, flength, qualifier, qoffset, qlength, + HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0); + } /** * Create a KeyValue for the specified row, family and qualifier that would be diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/Tag.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/Tag.java index 8d3c0b9..27e0494 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/Tag.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/Tag.java @@ -19,11 +19,13 @@ */ package org.apache.hadoop.hbase; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; /** * Tags are part of cells and helps to add metadata about the KVs. @@ -38,7 +40,8 @@ public class Tag { private static final int MAX_TAG_LENGTH = (2 * Short.MAX_VALUE) + 1 - TAG_LENGTH_SIZE; private final byte type; - private final byte[] bytes; + private byte[] bytes = null; + private ByteBuffer buffer = null; private int offset = 0; private int length = 0; @@ -72,6 +75,30 @@ public class Tag { Bytes.putBytes(bytes, pos, tag, 0, tag.length); this.type = tagType; } + + /** + * @param tagType + * @param tag + */ + public Tag(byte tagType, ByteBuffer tag) { + /** + * Format for a tag : taglength is serialized + * using 2 bytes only but as this will be unsigned, we can have max taglength of + * (Short.MAX_SIZE * 2) +1. It includes 1 byte type length and actual tag bytes length. + */ + int tagLength = tag.capacity() + TYPE_LENGTH_SIZE; + if (tagLength > MAX_TAG_LENGTH) { + throw new IllegalArgumentException( + "Invalid tag data being passed. Its length can not exceed " + MAX_TAG_LENGTH); + } + length = TAG_LENGTH_SIZE + tagLength; + bytes = new byte[length]; + int pos = ByteBufferUtils.putAsShort(buffer, 0, tagLength); + buffer.put(pos, tagType); + buffer.position(buffer.position() + 1); + ByteBufferUtils.copyFromBufferToBuffer(buffer, tag, buffer.position(), 0, tag.capacity()); + this.type = tagType; + } /** * Creates a Tag from the specified byte array and offset. Presumes @@ -112,6 +139,17 @@ public class Tag { this.length = length; this.type = bytes[offset + TAG_LENGTH_SIZE]; } + + public Tag(ByteBuffer buffer, int offset, int length) { + if (length > MAX_TAG_LENGTH) { + throw new IllegalArgumentException( + "Invalid tag data being passed. Its length can not exceed " + MAX_TAG_LENGTH); + } + this.buffer = buffer; + this.offset = offset; + this.length = length; + this.type = buffer.get(offset + TAG_LENGTH_SIZE); + } /** * @return The byte array backing this Tag. @@ -193,6 +231,26 @@ public class Tag { } return null; } + + /** + * Retrieve the first tag from the tags byte array matching the passed in tag type + * @param b + * @param offset + * @param length + * @param type + * @return null if there is no tag of the passed in tag type + */ + public static Tag getTag(ByteBuffer b, int offset, int length, byte type) { + int pos = offset; + while (pos < offset + length) { + int tagLen = ByteBufferUtils.readAsInt(b, pos, TAG_LENGTH_SIZE); + if(b.get(pos + TAG_LENGTH_SIZE) == type) { + return new Tag(b, pos, tagLen + TAG_LENGTH_SIZE); + } + pos += TAG_LENGTH_SIZE + tagLen; + } + return null; + } /** * Returns the total length of the entire tag entity diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index a4b3857..5870178 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -21,17 +21,17 @@ import java.io.DataOutputStream; import java.io.IOException; import java.nio.ByteBuffer; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.SettableSequenceId; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator; import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.SettableSequenceId; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.TagCompressionContext; import org.apache.hadoop.hbase.io.hfile.BlockType; @@ -316,6 +316,48 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { currentKey.getTimestamp(), currentKey.getTypeByte(), valueLength, valueOffset, memstoreTS, tagsOffset, tagsLength, tagCompressionContext, tagsBuffer); } + + @Override + public ByteBuffer getQualifierBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getRowBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getFamilyBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getValueBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getTagBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public long getTimeStampFromBuffer() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public byte getTypeBytesFromBuffer() { + // TODO Auto-generated method stub + return 0; + } } /** @@ -518,6 +560,48 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { return FIXED_OVERHEAD + rowLength + familyLength + qualifierLength + valueLength + tagsLength + KeyValue.TIMESTAMP_TYPE_SIZE; } + + @Override + public ByteBuffer getQualifierBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getRowBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getFamilyBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getValueBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getTagBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public long getTimeStampFromBuffer() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public byte getTypeBytesFromBuffer() { + // TODO Auto-generated method stub + return 0; + } } protected abstract static class diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java index b4c6690..b25bf92 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java @@ -96,6 +96,25 @@ public final class ByteBufferUtils { } return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i); } + + /** + * Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a + * {@link ByteBuffer}. + */ + public static long readVLong(ByteBuffer in, int offset) { + byte firstByte = in.get(offset); + int len = WritableUtils.decodeVIntSize(firstByte); + if (len == 1) { + return firstByte; + } + long i = 0; + for (int idx = 0; idx < len-1; idx++) { + byte b = in.get(); + i = i << 8; + i = i | (b & 0xFF); + } + return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i); + } /** @@ -350,6 +369,28 @@ public final class ByteBufferUtils { } } } + + /** + * Copy from one buffer to another from given offset. This will be absolute + * positional copying and won't affect the position of any of the buffers. + * + * @param out + * @param in + * @param sourceOffset + * @param destinationOffset + * @param length + */ + public static void copyFromBufferToBuffer(ByteBuffer out, ByteBuffer in, int sourceOffset, + int destinationOffset, int length) { + if (in.hasArray() && out.hasArray()) { + System.arraycopy(in.array(), sourceOffset + in.arrayOffset(), out.array(), out.arrayOffset() + + destinationOffset, length); + } else { + for (int i = 0; i < length; ++i) { + out.put((destinationOffset + i), in.get(sourceOffset + i)); + } + } + } /** * Find length of common prefix of two parts in the buffer @@ -454,4 +495,203 @@ public final class ByteBufferUtils { return output; } + public static int compareTo(ByteBuffer left, int leftOffset, int leftLength, ByteBuffer right, + int rightOffset, int rightLength) { + // Short circuit equal case + if (left == right && leftOffset == rightOffset && leftLength == rightLength) { + return 0; + } + // Bring WritableComparator code local + int end1 = leftOffset + leftLength; + int end2 = rightOffset + rightLength; + // Avoid this arrayOffset + for (int i = leftOffset - left.arrayOffset(), j = rightOffset - right.arrayOffset(); i < end1 + && j < end2; i++, j++) { + int a = (left.get(i) & 0xff); + int b = (right.get(j) & 0xff); + if (a != b) { + return a - b; + } + } + return leftLength - rightLength; + } + + public static int compareTo(ByteBuffer left, int leftOffset, int leftLength, byte[] right, + int rightOffset, int rightLength) { + // Bring WritableComparator code local + int end1 = leftOffset + leftLength; + int end2 = rightOffset + rightLength; + for (int i = leftOffset - left.arrayOffset(), j = rightOffset; i < end1 + && j < end2; i++, j++) { + int a = (left.get(i) & 0xff); + int b = (right[j] & 0xff); + if (a != b) { + return a - b; + } + } + return leftLength - rightLength; + } + + + public static int compareTo(byte[] left, int leftOffset, int leftLength, ByteBuffer right, + int rightOffset, int rightLength) { + // Bring WritableComparator code local + int end1 = leftOffset + leftLength; + int end2 = rightOffset + rightLength; + for (int i = leftOffset, j = rightOffset - right.arrayOffset(); i < end1 + && j < end2; i++, j++) { + int a = (left[i] & 0xff); + int b = (right.get(j) & 0xff); + if (a != b) { + return a - b; + } + } + return leftLength - rightLength; + } + + public static boolean equals(final ByteBuffer left, int leftOffset, int leftLen, final ByteBuffer right, + int rightOffset, int rightLen) { + // short circuit case + if (left == right && leftOffset == rightOffset && leftLen == rightLen) { + return true; + } + // different lengths fast check + if (leftLen != rightLen) { + return false; + } + if (leftLen == 0) { + return true; + } + + // Since we're often comparing adjacent sorted data, + // it's usual to have equal arrays except for the very last byte + // so check that first + if (left.get(leftOffset + leftLen - 1 - left.arrayOffset()) != right.get(rightOffset + rightLen + - 1 - right.arrayOffset())) + return false; + + return compareTo(left, leftOffset, leftLen, right, + rightOffset, rightLen) == 0; + } + + public static boolean equals(final ByteBuffer left, int leftOffset, int leftLen, final byte[] right, + int rightOffset, int rightLen) { + // different lengths fast check + if (leftLen != rightLen) { + return false; + } + if (leftLen == 0) { + return true; + } + + // Since we're often comparing adjacent sorted data, + // it's usual to have equal arrays except for the very last byte + // so check that first + if (left.get(leftOffset + leftLen - 1 - left.arrayOffset()) != right[rightOffset + rightLen + - 1]) { + return false; + } + + return compareTo(left, leftOffset, leftLen, right, + rightOffset, rightLen) == 0; + } + + public static boolean equals(final byte[] left, int leftOffset, int leftLen, final ByteBuffer right, + int rightOffset, int rightLen) { + // different lengths fast check + if (leftLen != rightLen) { + return false; + } + if (leftLen == 0) { + return true; + } + + // Since we're often comparing adjacent sorted data, + // it's usual to have equal arrays except for the very last byte + // so check that first + if (left[leftOffset + leftLen - 1] != right.get(rightOffset + rightLen + - 1- right.arrayOffset())) { + return false; + } + + return compareTo(left, leftOffset, leftLen, right, + rightOffset, rightLen) == 0; + } + + public static int hashCode(ByteBuffer buffer, int offset, int length) { + int hash = 1; + for (int i = offset; i < offset + length; i++) + hash = (31 * hash) + (int) buffer.get(i); + return hash; + } + + /** + * Converts a byte array to an int value + * @param bytes byte array + * @param offset offset into array + * @param length how many bytes should be considered for creating int + * @return the int value + * @throws IllegalArgumentException if there's not enough room in the array at the offset + * indicated. + */ + public static int readAsInt(ByteBuffer buffer, int offset, final int length) { + if (offset + length > buffer.capacity()) { + throw new IllegalArgumentException("offset (" + offset + ") + length (" + length + + ") exceed the" + " capacity of the array: " + buffer.capacity()); + } + int n = 0; + for(int i = offset; i < (offset + length); i++) { + n <<= 8; + n ^= buffer.get(i) & 0xFF; + } + return n; + } + + public static long readAsLong(ByteBuffer buffer, int offset, final int length) { + if (offset + length > buffer.capacity()) { + throw new IllegalArgumentException("offset (" + offset + ") + length (" + length + + ") exceed the" + " capacity of the array: " + buffer.capacity()); + } + long l = 0; + for(int i = offset; i < offset + length; i++) { + l <<= 8; + l ^= buffer.get(i) & 0xFF; + } + return l; + } + + public static short readAsShort(ByteBuffer buffer, int offset, final int length) { + if (offset + length > buffer.capacity()) { + throw new IllegalArgumentException("offset (" + offset + ") + length (" + length + + ") exceed the" + " capacity of the array: " + buffer.capacity()); + } + short n = 0; + n ^= buffer.get(offset) & 0xFF; + n <<= 8; + n ^= buffer.get(offset+1) & 0xFF; + return n; + } + + /** + * Put an int value as short out to the specified byte buffer position. Only the lower 2 bytes of + * the short will be put into the array. The caller of the API need to make sure they will not + * loose the value by doing so. This is useful to store an unsigned short which is represented as + * int in other parts. + * @param bytes the byte array + * @param offset position in the array + * @param val value to write out + * @return incremented offset + * @throws IllegalArgumentException if the byte array given doesn't have + * enough room at the offset specified. + */ + public static int putAsShort(ByteBuffer b, int offset, int val) { + if (b.capacity() - offset < Bytes.SIZEOF_SHORT) { + throw new IllegalArgumentException("Not enough room to put a short at" + + " offset " + offset + " in a " + b.capacity() + " byte buffer"); + } + b.put(offset + 1, (byte)val); + val >>= 8; + b.put(offset, (byte)val); + return offset + Bytes.SIZEOF_SHORT; + } } diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java index b085b3e..fdce297 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java @@ -39,23 +39,22 @@ import java.util.Comparator; import java.util.Iterator; import java.util.List; -import com.google.protobuf.ByteString; +import sun.misc.Unsafe; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.util.Bytes.LexicographicalComparerHolder.UnsafeComparer; import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.WritableComparator; import org.apache.hadoop.io.WritableUtils; -import sun.misc.Unsafe; - import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Lists; -import org.apache.hadoop.hbase.util.Bytes.LexicographicalComparerHolder.UnsafeComparer; +import com.google.protobuf.ByteString; /** * Utility class that handles byte arrays, conversions to/from other types, @@ -506,6 +505,15 @@ public class Bytes implements Comparable { buf.get(bytes, offset, len); return offset + len; } + + public static int putByteBuffer(byte[] bytes, int offset, ByteBuffer buf, int bufOffset, + int bufLength) { + buf.mark(); + buf.position(bufOffset - buf.arrayOffset()); + buf.get(bytes, offset, bufLength); + buf.reset(); + return offset + bufLength; + } /** * Returns a new byte array, copied from the given {@code buf}, @@ -1064,7 +1072,7 @@ public class Bytes implements Comparable { } return n; } - + /** * Put an int value out to the specified byte array position. * @param bytes the byte array @@ -1431,7 +1439,6 @@ public class Bytes implements Comparable { *

Uses reflection to gracefully fall back to the Java implementation if * {@code Unsafe} isn't available. */ - @VisibleForTesting static class LexicographicalComparerHolder { static final String UNSAFE_COMPARER_NAME = LexicographicalComparerHolder.class.getName() + "$UnsafeComparer"; diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java index 182c4db..6ef94ac 100644 --- a/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java +++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellUtil.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.NavigableMap; @@ -220,6 +221,48 @@ public class TestCellUtil { // TODO Auto-generated method stub return 0; } + + @Override + public ByteBuffer getQualifierBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getRowBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getFamilyBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getValueBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getTagBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public long getTimeStampFromBuffer() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public byte getTypeBytesFromBuffer() { + // TODO Auto-generated method stub + return 0; + } }; /** diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java index fb73443..1fd6bee 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java @@ -440,5 +440,47 @@ public class PrefixTreeSeeker implements EncodedSeeker { public String toString() { return KeyValueUtil.copyToNewKeyValue(this).toString(); } + + @Override + public ByteBuffer getQualifierBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getRowBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getFamilyBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getValueBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public ByteBuffer getTagBuffer() { + // TODO Auto-generated method stub + return null; + } + + @Override + public long getTimeStampFromBuffer() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public byte getTypeBytesFromBuffer() { + // TODO Auto-generated method stub + return 0; + } } } diff --git a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java index c29a704..cd3e4a2 100644 --- a/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java +++ b/hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java @@ -18,6 +18,8 @@ package org.apache.hadoop.hbase.codec.prefixtree.decode; +import java.nio.ByteBuffer; + import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellUtil; @@ -239,4 +241,46 @@ public class PrefixTreeCell implements Cell, SettableSequenceId, Comparable 0) { + // The current KV is smaller than the column the ExplicitColumnTracker + // is interested in, so seek to that column of interest. + return this.skipCount++ < this.lookAhead ? ScanQueryMatcher.MatchCode.SKIP + : ScanQueryMatcher.MatchCode.SEEK_NEXT_COL; + } + + // The current KV is bigger than the column the ExplicitColumnTracker + // is interested in. That means there is no more data for the column + // of interest. Advance the ExplicitColumnTracker state to next + // column of interest, and check again. + if (ret <= -1) { + ++this.index; + this.skipCount = 0; + if (done()) { + // No more to match, do not include, done with this row. + return ScanQueryMatcher.MatchCode.SEEK_NEXT_ROW; // done_row + } + // This is the recursive case. + this.column = this.columns[this.index]; + } + } while(true); + } + + @Override + public MatchCode checkVersions(ByteBuffer buffer, int offset, int length, long timestamp, byte type, + boolean ignoreCount) throws IOException { + return checkVersions((byte[])null, offset, length, timestamp, type, ignoreCount); + } + + @Override + public MatchCode getNextRowOrNextColumn(ByteBuffer buffer, int offset, int qualLength) { + doneWithColumn(buffer, offset,qualLength); + + if (getColumnHint() == null) { + return MatchCode.SEEK_NEXT_ROW; + } else { + return MatchCode.SEEK_NEXT_COL; + } + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java index 964fad8..80ecfa3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanQueryMatcher.java @@ -20,19 +20,21 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.NavigableSet; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.filter.Filter.ReturnCode; import org.apache.hadoop.hbase.io.TimeRange; import org.apache.hadoop.hbase.regionserver.DeleteTracker.DeleteResult; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; @@ -88,7 +90,7 @@ public class ScanQueryMatcher { /* row is not private for tests */ /** Row the query is on */ - byte [] row; + ByteBuffer row; int rowOffset; short rowLength; @@ -265,8 +267,14 @@ public class ScanQueryMatcher { if (filter != null && filter.filterAllRemaining()) { return MatchCode.DONE_SCAN; } - int ret = this.rowComparator.compareRows(row, this.rowOffset, this.rowLength, - cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()); + int ret = 0; + if (cell.getRowArray() != null) { + ret = this.rowComparator.compareRows(row, this.rowOffset, this.rowLength, cell.getRowArray(), + cell.getRowOffset(), cell.getRowLength()); + } else { + ret = this.rowComparator.compareRows(row, this.rowOffset, this.rowLength, + cell.getRowBuffer(), cell.getRowOffset(), cell.getRowLength()); + } if (!this.isReversed) { if (ret <= -1) { return MatchCode.DONE; @@ -388,8 +396,14 @@ public class ScanQueryMatcher { } // STEP 1: Check if the column is part of the requested columns - MatchCode colChecker = columns.checkColumn(cell.getQualifierArray(), - qualifierOffset, qualifierLength, typeByte); + MatchCode colChecker = null; + if (cell.getQualifierArray() != null) { + colChecker = columns.checkColumn(cell.getQualifierArray(), qualifierOffset, qualifierLength, + typeByte); + } else { + colChecker = columns.checkColumn(cell.getQualifierBuffer(), qualifierOffset, qualifierLength, + typeByte); + } if (colChecker == MatchCode.INCLUDE) { ReturnCode filterResponse = ReturnCode.SKIP; // STEP 2: Yes, the column is part of the requested columns. Check if filter is present @@ -431,10 +445,13 @@ public class ScanQueryMatcher { * In all the above scenarios, we return the column checker return value except for * FilterResponse (INCLUDE_AND_SEEK_NEXT_COL) and ColumnChecker(INCLUDE) */ - colChecker = - columns.checkVersions(cell.getQualifierArray(), qualifierOffset, - qualifierLength, timestamp, typeByte, - mvccVersion > maxReadPointToTrackVersions); + if (cell.getQualifierArray() != null) { + colChecker = columns.checkVersions(cell.getQualifierArray(), qualifierOffset, + qualifierLength, timestamp, typeByte, mvccVersion > maxReadPointToTrackVersions); + } else { + colChecker = columns.checkVersions(cell.getQualifierBuffer(), qualifierOffset, + qualifierLength, timestamp, typeByte, mvccVersion > maxReadPointToTrackVersions); + } //Optimize with stickyNextRow stickyNextRow = colChecker == MatchCode.INCLUDE_AND_SEEK_NEXT_ROW ? true : stickyNextRow; return (filterResponse == ReturnCode.INCLUDE_AND_NEXT_COL && @@ -449,14 +466,14 @@ public class ScanQueryMatcher { /** Handle partial-drop-deletes. As we match keys in order, when we have a range from which * we can drop deletes, we can set retainDeletesInOutput to false for the duration of this * range only, and maintain consistency. */ - private void checkPartialDropDeleteRange(byte [] row, int offset, short length) { + private void checkPartialDropDeleteRange(ByteBuffer row, int offset, short length) { // If partial-drop-deletes are used, initially, dropDeletesFromRow and dropDeletesToRow // are both set, and the matcher is set to retain deletes. We assume ordered keys. When // dropDeletesFromRow is leq current kv, we start dropping deletes and reset // dropDeletesFromRow; thus the 2nd "if" starts to apply. if ((dropDeletesFromRow != null) && ((dropDeletesFromRow == HConstants.EMPTY_START_ROW) - || (Bytes.compareTo(row, offset, length, + || (ByteBufferUtils.compareTo(row, offset, length, dropDeletesFromRow, 0, dropDeletesFromRow.length) >= 0))) { retainDeletesInOutput = false; dropDeletesFromRow = null; @@ -466,7 +483,7 @@ public class ScanQueryMatcher { // and reset dropDeletesToRow so that we don't do any more compares. if ((dropDeletesFromRow == null) && (dropDeletesToRow != null) && (dropDeletesToRow != HConstants.EMPTY_END_ROW) - && (Bytes.compareTo(row, offset, length, + && (ByteBufferUtils.compareTo(row, offset, length, dropDeletesToRow, 0, dropDeletesToRow.length) >= 0)) { retainDeletesInOutput = true; dropDeletesToRow = null; @@ -475,16 +492,30 @@ public class ScanQueryMatcher { public boolean moreRowsMayExistAfter(Cell kv) { if (this.isReversed) { - if (rowComparator.compareRows(kv.getRowArray(), kv.getRowOffset(), - kv.getRowLength(), stopRow, 0, stopRow.length) <= 0) { + int diff = 0; + if(kv.getRowArray() != null) { + diff = rowComparator.compareRows(kv.getRowArray(), kv.getRowOffset(), + kv.getRowLength(), stopRow, 0, stopRow.length); + } else { + diff = rowComparator.compareRows(kv.getRowBuffer(), kv.getRowOffset(), + kv.getRowLength(), stopRow, 0, stopRow.length); + } + if (diff <= 0) { return false; } else { return true; } } + int diff = 0; + if(kv.getRowArray() != null) { + diff = rowComparator.compareRows(kv.getRowArray(), kv.getRowOffset(), + kv.getRowLength(), stopRow, 0, stopRow.length); + } else { + diff = rowComparator.compareRows(kv.getRowBuffer(), kv.getRowOffset(), + kv.getRowLength(), stopRow, 0, stopRow.length); + } if (!Bytes.equals(stopRow , HConstants.EMPTY_END_ROW) && - rowComparator.compareRows(kv.getRowArray(),kv.getRowOffset(), - kv.getRowLength(), stopRow, 0, stopRow.length) >= 0) { + diff >= 0) { // KV >= STOPROW // then NO there is nothing left. return false; @@ -497,7 +528,7 @@ public class ScanQueryMatcher { * Set current row * @param row */ - public void setRow(byte [] row, int offset, short length) { + public void setRow(ByteBuffer row, int offset, short length) { checkPartialDropDeleteRange(row, offset, length); this.row = row; this.rowOffset = offset; @@ -539,15 +570,25 @@ public class ScanQueryMatcher { public Cell getKeyForNextColumn(Cell kv) { ColumnCount nextColumn = columns.getColumnHint(); if (nextColumn == null) { - return KeyValueUtil.createLastOnRow( - kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), - kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), - kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()); + if(kv.getRowArray() != null) { + return KeyValueUtil.createLastOnRow(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), + kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), + kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength()); + } else { + return KeyValueUtil.createLastOnRow(kv.getRowBuffer(), kv.getRowOffset(), kv.getRowLength(), + kv.getFamilyBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(), + kv.getQualifierBuffer(), kv.getQualifierOffset(), kv.getQualifierLength()); + } } else { - return KeyValueUtil.createFirstOnRow( - kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), - kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), - nextColumn.getBuffer(), nextColumn.getOffset(), nextColumn.getLength()); + if(kv.getRowArray() != null) { + return KeyValueUtil.createFirstOnRow(kv.getRowArray(), kv.getRowOffset(), + kv.getRowLength(), kv.getFamilyArray(), kv.getFamilyOffset(), kv.getFamilyLength(), + nextColumn.getBytes(), nextColumn.getOffset(), nextColumn.getLength()); + } else { + return KeyValueUtil.createFirstOnRow(kv.getRowBuffer(), kv.getRowOffset(), + kv.getRowLength(), kv.getFamilyBuffer(), kv.getFamilyOffset(), kv.getFamilyLength(), + nextColumn.getBuffer(), nextColumn.getOffset(), nextColumn.getLength()); + } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java index 85b36fb..800223b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ScanWildcardColumnTracker.java @@ -20,11 +20,13 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; +import java.nio.ByteBuffer; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.regionserver.ScanQueryMatcher.MatchCode; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; /** @@ -32,7 +34,8 @@ import org.apache.hadoop.hbase.util.Bytes; */ @InterfaceAudience.Private public class ScanWildcardColumnTracker implements ColumnTracker { - private byte [] columnBuffer = null; + private byte [] columnBytes = null; + private ByteBuffer columnBuffer = null; private int columnOffset = 0; private int columnLength = 0; private int currentCount = 0; @@ -78,7 +81,7 @@ public class ScanWildcardColumnTracker implements ColumnTracker { public ScanQueryMatcher.MatchCode checkVersions(byte[] bytes, int offset, int length, long timestamp, byte type, boolean ignoreCount) throws IOException { - if (columnBuffer == null) { + if (columnBytes == null) { // first iteration. resetBuffer(bytes, offset, length); if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE; @@ -86,7 +89,7 @@ public class ScanWildcardColumnTracker implements ColumnTracker { return checkVersion(type, timestamp); } int cmp = Bytes.compareTo(bytes, offset, length, - columnBuffer, columnOffset, columnLength); + columnBytes, columnOffset, columnLength); if (cmp == 0) { if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE; @@ -118,11 +121,20 @@ public class ScanWildcardColumnTracker implements ColumnTracker { } private void resetBuffer(byte[] bytes, int offset, int length) { - columnBuffer = bytes; + columnBytes = bytes; + columnOffset = offset; + columnLength = length; + currentCount = 0; + } + + private void resetBuffer(ByteBuffer buffer, int offset, int length) { + columnBuffer = buffer; columnOffset = offset; columnLength = length; currentCount = 0; } + + /** * Check whether this version should be retained. @@ -152,8 +164,9 @@ public class ScanWildcardColumnTracker implements ColumnTracker { @Override public void reset() { - columnBuffer = null; + columnBytes = null; resetTSAndType(); + columnBuffer = null; } private void resetTSAndType() { @@ -181,6 +194,7 @@ public class ScanWildcardColumnTracker implements ColumnTracker { * * @return The column count. */ + @Override public ColumnCount getColumnHint() { return null; } @@ -194,12 +208,69 @@ public class ScanWildcardColumnTracker implements ColumnTracker { return false; } + @Override public MatchCode getNextRowOrNextColumn(byte[] bytes, int offset, int qualLength) { return MatchCode.SEEK_NEXT_COL; } + @Override public boolean isDone(long timestamp) { return minVersions <= 0 && isExpired(timestamp); } + + @Override + public MatchCode checkColumn(ByteBuffer buffer, int offset, int length, byte type) + throws IOException { + return MatchCode.INCLUDE; + } + + @Override + public MatchCode checkVersions(ByteBuffer buffer, int offset, int length, long timestamp, + byte type, boolean ignoreCount) throws IOException { + + if (columnBuffer == null) { + // first iteration. + resetBuffer(buffer, offset, length); + if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE; + // do not count a delete marker as another version + return checkVersion(type, timestamp); + } + int cmp = ByteBufferUtils.compareTo(buffer, offset, length, + columnBuffer, columnOffset, columnLength); + if (cmp == 0) { + if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE; + + //If column matches, check if it is a duplicate timestamp + if (sameAsPreviousTSAndType(timestamp, type)) { + return ScanQueryMatcher.MatchCode.SKIP; + } + return checkVersion(type, timestamp); + } + + resetTSAndType(); + + // new col > old col + if (cmp > 0) { + // switched columns, lets do something.x + resetBuffer(buffer, offset, length); + if (ignoreCount) return ScanQueryMatcher.MatchCode.INCLUDE; + return checkVersion(type, timestamp); + } + + // new col < oldcol + // WARNING: This means that very likely an edit for some other family + // was incorrectly stored into the store for this one. Throw an exception, + // because this might lead to data corruption. + throw new IOException( + "ScanWildcardColumnTracker.checkColumn ran into a column actually " + + "smaller than the previous column: " + + Bytes.toStringBinary(buffer.array(), offset, length)); + + } + + @Override + public MatchCode getNextRowOrNextColumn(ByteBuffer buffer, int offset, int qualLength) { + return MatchCode.SEEK_NEXT_COL; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java index 3e139a5..41338e7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java @@ -33,7 +33,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -44,6 +43,7 @@ import org.apache.hadoop.hbase.HDFSBlocksDistribution; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.hfile.BlockType; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java index 0a4e1ed..053d389 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreScanner.java @@ -21,6 +21,7 @@ package org.apache.hadoop.hbase.regionserver; import java.io.IOException; import java.io.InterruptedIOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; import java.util.NavigableSet; @@ -29,7 +30,6 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.DoNotRetryIOException; @@ -37,12 +37,13 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValueUtil; +import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.IsolationLevel; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.executor.ExecutorService; import org.apache.hadoop.hbase.filter.Filter; import org.apache.hadoop.hbase.regionserver.handler.ParallelSeekHandler; -import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; /** @@ -464,10 +465,15 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner // only call setRow if the row changes; avoids confusing the query matcher // if scanning intra-row - byte[] row = peeked.getRowArray(); + ByteBuffer row; + if(peeked.getRowArray() != null) { + row = ByteBuffer.wrap(peeked.getRowArray()); + } else { + row = peeked.getRowBuffer(); + } int offset = peeked.getRowOffset(); short length = peeked.getRowLength(); - if (limit < 0 || matcher.row == null || !Bytes.equals(row, offset, length, matcher.row, + if (limit < 0 || matcher.row == null || !ByteBufferUtils.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) { this.countPerRow = 0; matcher.setRow(row, offset, length); @@ -668,10 +674,15 @@ public class StoreScanner extends NonReversedNonLazyKeyValueScanner if (kv == null) { kv = lastTopKey; } - byte[] row = kv.getRowArray(); + ByteBuffer row; + if(kv.getRowArray() != null) { + row = ByteBuffer.wrap(kv.getRowArray()); + } else { + row = kv.getRowBuffer(); + } int offset = kv.getRowOffset(); short length = kv.getRowLength(); - if ((matcher.row == null) || !Bytes.equals(row, offset, length, matcher.row, + if ((matcher.row == null) || !ByteBufferUtils.equals(row, offset, length, matcher.row, matcher.rowOffset, matcher.rowLength)) { this.countPerRow = 0; matcher.reset();