From 057b49260c71782b0ba20f4e963485be10fdf739 Mon Sep 17 00:00:00 2001 From: Michael Stack Date: Thu, 10 May 2018 01:25:16 -0700 Subject: [PATCH] BBKV comparator --- .../apache/hadoop/hbase/ByteBufferKeyValue.java | 60 +++++--- .../apache/hadoop/hbase/CellComparatorImpl.java | 158 ++++++++++++++++++--- 2 files changed, 177 insertions(+), 41 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java index 760d02c956..3152e95038 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/ByteBufferKeyValue.java @@ -96,15 +96,18 @@ public class ByteBufferKeyValue extends ByteBufferExtendedCell { @Override public byte getFamilyLength() { - return getFamilyLength(getFamilyLengthPosition()); + return getFamilyLength(getRowLen()); } - private int getFamilyLengthPosition() { - return this.offset + KeyValue.ROW_KEY_OFFSET - + getRowLen(); + byte getFamilyLength(int rowLength) { + return getFamilyLengthAtPosition(getFamilyLengthPosition(rowLength)); } - private byte getFamilyLength(int famLenPos) { + int getFamilyLengthPosition(int rowLength) { + return this.offset + KeyValue.ROW_KEY_OFFSET + rowLength; + } + + byte getFamilyLengthAtPosition(int famLenPos) { return ByteBufferUtils.toByte(this.buf, famLenPos); } @@ -120,32 +123,38 @@ public class ByteBufferKeyValue extends ByteBufferExtendedCell { @Override public int getQualifierLength() { - return getQualifierLength(getRowLength(), getFamilyLength()); + return getQualifierLength(getKeyLength(), getRowLength(), getFamilyLength()); } - private int getQualifierLength(int rlength, int flength) { - return getKeyLen() - - (int) KeyValue.getKeyDataStructureSize(rlength, flength, 0); + int getQualifierLength(int keyLength, int rlength, int flength) { + return keyLength - (int) KeyValue.getKeyDataStructureSize(rlength, flength, 0); } @Override public long getTimestamp() { - int offset = getTimestampOffset(getKeyLen()); + return getTimestamp(getKeyLength()); + } + + long getTimestamp(int keyLength) { + int offset = getTimestampOffset(keyLength); return ByteBufferUtils.toLong(this.buf, offset); } - private int getKeyLen() { + int getKeyLength() { return ByteBufferUtils.toInt(this.buf, this.offset); } - private int getTimestampOffset(int keyLen) { - return this.offset + KeyValue.ROW_OFFSET + keyLen - KeyValue.TIMESTAMP_TYPE_SIZE; + int getTimestampOffset(int keyLength) { + return this.offset + KeyValue.ROW_OFFSET + keyLength - KeyValue.TIMESTAMP_TYPE_SIZE; } @Override public byte getTypeByte() { - return ByteBufferUtils.toByte(this.buf, - this.offset + getKeyLen() - 1 + KeyValue.ROW_OFFSET); + return getTypeByte(getKeyLength()); + } + + byte getTypeByte(int keyLength) { + return ByteBufferUtils.toByte(this.buf, this.offset + keyLength - 1 + KeyValue.ROW_OFFSET); } @Override @@ -185,7 +194,7 @@ public class ByteBufferKeyValue extends ByteBufferExtendedCell { @Override public int getTagsLength() { - int tagsLen = this.length - (getKeyLen() + getValueLength() + int tagsLen = this.length - (getKeyLength() + getValueLength() + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE); if (tagsLen > 0) { // There are some Tag bytes in the byte[]. So reduce 2 bytes which is @@ -213,7 +222,11 @@ public class ByteBufferKeyValue extends ByteBufferExtendedCell { @Override public int getFamilyPosition() { - return getFamilyLengthPosition() + Bytes.SIZEOF_BYTE; + return getFamilyPosition(getRowLen()); + } + + int getFamilyPosition(int rowLength) { + return getFamilyLengthPosition(rowLength) + Bytes.SIZEOF_BYTE; } @Override @@ -223,7 +236,11 @@ public class ByteBufferKeyValue extends ByteBufferExtendedCell { @Override public int getQualifierPosition() { - return getFamilyPosition() + getFamilyLength(); + return getQualifierPosition(getFamilyPosition(), getFamilyLength()); + } + + int getQualifierPosition(int familyPosition, int familyLength) { + return familyPosition + familyLength; } @Override @@ -233,7 +250,7 @@ public class ByteBufferKeyValue extends ByteBufferExtendedCell { @Override public int getValuePosition() { - return this.offset + KeyValue.ROW_OFFSET + getKeyLen(); + return this.offset + KeyValue.ROW_OFFSET + getKeyLength(); } @Override @@ -270,7 +287,7 @@ public class ByteBufferKeyValue extends ByteBufferExtendedCell { if (withTags) { return this.length; } - return getKeyLen() + this.getValueLength() + return getKeyLength() + this.getValueLength() + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE; } @@ -291,8 +308,7 @@ public class ByteBufferKeyValue extends ByteBufferExtendedCell { } private int getTimestampOffset() { - return this.offset + KeyValue.KEYVALUE_INFRASTRUCTURE_SIZE - + getKeyLen() - KeyValue.TIMESTAMP_TYPE_SIZE; + return getTimestampOffset(getKeyLength()); } @Override diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java index b1af716614..785d62d0c0 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparatorImpl.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase; import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.nio.ByteBuff; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.Bytes; import org.apache.yetus.audience.InterfaceAudience; @@ -70,22 +71,139 @@ public class CellComparatorImpl implements CellComparator { * @return 0 if equal, -1 if a < b, and +1 if a > b. */ public final int compare(final Cell a, final Cell b, boolean ignoreSequenceid) { - // row - int c = compareRows(a, b); - if (c != 0) return c; + if (a instanceof ByteBufferKeyValue && b instanceof ByteBufferKeyValue) { + return compareByteBufferKeyValue((ByteBufferKeyValue)a, (ByteBufferKeyValue)b); + } else { + // row + int c = compareRows(a, b); + if (c != 0) return c; - c = compareWithoutRow(a, b); - if(c != 0) return c; + c = compareWithoutRow(a, b); + if (c != 0) return c; - if (!ignoreSequenceid) { - // Negate following comparisons so later edits show up first - // mvccVersion: later sorts first - return Longs.compare(b.getSequenceId(), a.getSequenceId()); - } else { - return c; + if (!ignoreSequenceid) { + // Negate following comparisons so later edits show up first + // mvccVersion: later sorts first + return Longs.compare(b.getSequenceId(), a.getSequenceId()); + } else { + return c; + } + } + } + + /** + * Specialized comparator for the ByteBufferKeyValue type exclusivesly. + * Caches deserialized lengths of rows and families, etc., and reuses them where it can + * (ByteBufferKeyValue has been changed to be amenable to our providing pre-made lengths, etc.) + */ + private final int compareByteBufferKeyValue(ByteBufferKeyValue left, ByteBufferKeyValue right) { + // left and right can be exactly the same at the beginning of a row + if (left == right) { + return 0; + } + int leftRowLength = left.getRowLength(); + int rightRowLength = right.getRowLength(); + // Compare Rows + int diff = ByteBufferUtils.compareTo( + left.getRowByteBuffer(), left.getRowPosition(), leftRowLength, + right.getRowByteBuffer(), right.getRowPosition(), rightRowLength); + if (diff != 0) { + return diff; + } + // If the column is not specified, the "minimum" key type appears the + // latest in the sorted order, regardless of the timestamp. This is used + // for specifying the last key/value in a given row, because there is no + // "lexicographically last column" (it would be infinitely long). The + // "maximum" key type does not need this behavior. + // Copied from KeyValue. This is bad in that we can't do memcmp w/ special rules like this. + // I tried to get rid of the above but scanners depend on it. TODO. + + int leftFamilyPosition = left.getFamilyLengthPosition(leftRowLength); + int leftFamilyLength = left.getFamilyLengthAtPosition(leftFamilyPosition); + int rightFamilyPosition = right.getFamilyLengthPosition(rightRowLength); + int rightFamilyLength = right.getFamilyLengthAtPosition(rightFamilyPosition); + + int leftKeyLength = left.getKeyLength(); + int leftQualifierLength = left.getQualifierLength(leftKeyLength, leftKeyLength, + leftFamilyLength); + byte leftType = left.getTypeByte(leftKeyLength); + if (leftFamilyLength + leftQualifierLength == 0 && leftType == Type.Minimum.getCode()) { + // left is "bigger", i.e. it appears later in the sorted order + return 1; + } + int rightKeyLength = right.getKeyLength(); + int rightQualifierLength = right.getQualifierLength(rightKeyLength, rightKeyLength, + rightFamilyLength); + byte rightType = right.getTypeByte(rightKeyLength); + if (rightFamilyLength + rightQualifierLength == 0 && rightType == Type.Minimum.getCode()) { + return -1; + } + // Compare families. + diff = ByteBufferUtils.compareTo( + left.getFamilyByteBuffer(), leftFamilyPosition, leftFamilyLength, + right.getFamilyByteBuffer(), rightFamilyPosition, rightFamilyLength); + if (diff != 0) { + return diff; + } + // Compare qualifiers + diff = ByteBufferUtils.compareTo(left.getQualifierByteBuffer(), + left.getQualifierPosition(leftFamilyPosition, leftFamilyLength), leftQualifierLength, + right.getQualifierByteBuffer(), + right.getQualifierPosition(rightFamilyPosition, rightFamilyLength), + rightQualifierLength); + if (diff != 0) { + return diff; + } + // Timestamps. + diff = compareTimestamps(left.getTimestamp(leftKeyLength), right.getTimestamp(rightKeyLength)); + if (diff != 0) { + return diff; + } + // Compare types. Let the delete types sort ahead of puts; i.e. types + // of higher numbers sort before those of lesser numbers. Maximum (255) + // appears ahead of everything, and minimum (0) appears after + // everything. + return (0xff & rightType) - (0xff & leftType); + } + + private final int compareWithoutRows(final ByteBufferExtendedCell left, int leftRowLength, + final ByteBufferExtendedCell right, int rightRowLength) { + // If the column is not specified, the "minimum" key type appears the + // latest in the sorted order, regardless of the timestamp. This is used + // for specifying the last key/value in a given row, because there is no + // "lexicographically last column" (it would be infinitely long). The + // "maximum" key type does not need this behavior. + // Copied from KeyValue. This is bad in that we can't do memcmp w/ special rules like this. + int lFamLength = left.getFamilyLength(); + int rFamLength = right.getFamilyLength(); + int lQualLength = left.getQualifierLength(); + int rQualLength = right.getQualifierLength(); + if (lFamLength + lQualLength == 0 && left.getTypeByte() == Type.Minimum.getCode()) { + // left is "bigger", i.e. it appears later in the sorted order + return 1; + } + if (rFamLength + rQualLength == 0 && right.getTypeByte() == Type.Minimum.getCode()) { + return -1; + } + if (lFamLength != rFamLength) { + // comparing column family is enough. + return compareFamilies(left, right); } + // Compare cf:qualifier + int diff = compareColumns(left, right); + if (diff != 0) return diff; + + diff = compareTimestamps(left, right); + if (diff != 0) return diff; + + // Compare types. Let the delete types sort ahead of puts; i.e. types + // of higher numbers sort before those of lesser numbers. Maximum (255) + // appears ahead of everything, and minimum (0) appears after + // everything. + return (0xff & right.getTypeByte()) - (0xff & left.getTypeByte()); } + /** * Compares the family and qualifier part of the cell * @param left the left cell @@ -173,35 +291,37 @@ public class CellComparatorImpl implements CellComparator { * Compares the rows of the left and right cell. * For the hbase:meta case this method is overridden such that it can handle hbase:meta cells. * The caller should ensure using the appropriate comparator for hbase:meta. - * @param left - * @param right * @return 0 if both cells are equal, 1 if left cell is bigger than right, -1 otherwise */ @Override public int compareRows(final Cell left, final Cell right) { + return compareRows(left, left.getRowLength(), right, right.getRowLength()); + } + + int compareRows(final Cell left, int leftRowLength, final Cell right, int rightRowLength) { // left and right can be exactly the same at the beginning of a row if (left == right) { return 0; } if (left instanceof ByteBufferExtendedCell && right instanceof ByteBufferExtendedCell) { return ByteBufferUtils.compareTo(((ByteBufferExtendedCell) left).getRowByteBuffer(), - ((ByteBufferExtendedCell) left).getRowPosition(), left.getRowLength(), + ((ByteBufferExtendedCell) left).getRowPosition(), leftRowLength, ((ByteBufferExtendedCell) right).getRowByteBuffer(), - ((ByteBufferExtendedCell) right).getRowPosition(), right.getRowLength()); + ((ByteBufferExtendedCell) right).getRowPosition(), rightRowLength); } if (left instanceof ByteBufferExtendedCell) { return ByteBufferUtils.compareTo(((ByteBufferExtendedCell) left).getRowByteBuffer(), - ((ByteBufferExtendedCell) left).getRowPosition(), left.getRowLength(), - right.getRowArray(), right.getRowOffset(), right.getRowLength()); + ((ByteBufferExtendedCell) left).getRowPosition(), leftRowLength, + right.getRowArray(), right.getRowOffset(), rightRowLength); } if (right instanceof ByteBufferExtendedCell) { // Notice how we flip the order of the compare here. We used to negate the return value but // see what FindBugs says // http://findbugs.sourceforge.net/bugDescriptions.html#RV_NEGATING_RESULT_OF_COMPARETO // It suggest flipping the order to get same effect and 'safer'. - return ByteBufferUtils.compareTo(left.getRowArray(), left.getRowOffset(), left.getRowLength(), + return ByteBufferUtils.compareTo(left.getRowArray(), left.getRowOffset(), leftRowLength, ((ByteBufferExtendedCell)right).getRowByteBuffer(), - ((ByteBufferExtendedCell)right).getRowPosition(), right.getRowLength()); + ((ByteBufferExtendedCell)right).getRowPosition(), rightRowLength); } return Bytes.compareTo(left.getRowArray(), left.getRowOffset(), left.getRowLength(), right.getRowArray(), right.getRowOffset(), right.getRowLength()); -- 2.16.3