diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java index 4978387..0eccf57 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/CellComparator.java @@ -45,54 +45,85 @@ public class CellComparator implements Comparator, Serializable{ @Override public int compare(Cell a, Cell b) { - return compareStatic(a, b); + return compareStatic(a, b, false); } - - public static int compareStatic(Cell a, Cell b) { - //row - int c = Bytes.compareTo( - a.getRowArray(), a.getRowOffset(), a.getRowLength(), - b.getRowArray(), b.getRowOffset(), b.getRowLength()); + public static int compareStatic(Cell a, Cell b, boolean onlyKey) { + // row + int c = compareRows(a, b); if (c != 0) return c; - // If the column is not specified, the "minimum" key type appears the - // latest in the sorted order, regardless of the timestamp. This is used - // for specifying the last key/value in a given row, because there is no - // "lexicographically last column" (it would be infinitely long). The - // "maximum" key type does not need this behavior. - if (a.getFamilyLength() == 0 && a.getTypeByte() == Type.Minimum.getCode()) { - // a is "bigger", i.e. it appears later in the sorted order - return 1; - } - if (b.getFamilyLength() == 0 && b.getTypeByte() == Type.Minimum.getCode()) { - return -1; + c = compareWithoutRow(a, b); + if(c != 0) return c; + + if (!onlyKey) { + // Negate following comparisons so later edits show up first + + // compare log replay tag value if there is any + // when either keyvalue tagged with log replay sequence number, we need to compare them: + // 1) when both keyvalues have the tag, then use the tag values for comparison + // 2) when one has and the other doesn't have, the one without the log + // replay tag wins because + // it means the edit isn't from recovery but new one coming from clients during recovery + // 3) when both doesn't have, then skip to the next mvcc comparison + long leftChangeSeqNum = getReplaySeqNum(a); + long RightChangeSeqNum = getReplaySeqNum(b); + if (leftChangeSeqNum != Long.MAX_VALUE || RightChangeSeqNum != Long.MAX_VALUE) { + return Longs.compare(RightChangeSeqNum, leftChangeSeqNum); + } + // mvccVersion: later sorts first + return Longs.compare(b.getMvccVersion(), a.getMvccVersion()); + } else { + return c; } + } - //family - c = Bytes.compareTo( - a.getFamilyArray(), a.getFamilyOffset(), a.getFamilyLength(), - b.getFamilyArray(), b.getFamilyOffset(), b.getFamilyLength()); - if (c != 0) return c; + /** + * Return replay log sequence number for the cell + * + * @param c + * @return Long.MAX_VALUE if there is no LOG_REPLAY_TAG + */ + private static long getReplaySeqNum(final Cell c) { + Tag tag = Tag.getTag(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength(), + TagType.LOG_REPLAY_TAG_TYPE); - //qualifier - c = Bytes.compareTo( - a.getQualifierArray(), a.getQualifierOffset(), a.getQualifierLength(), - b.getQualifierArray(), b.getQualifierOffset(), b.getQualifierLength()); - if (c != 0) return c; + if (tag != null) { + return Bytes.toLong(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength()); + } + return Long.MAX_VALUE; + } - //timestamp: later sorts first - c = Longs.compare(b.getTimestamp(), a.getTimestamp()); - if (c != 0) return c; + public static int findCommonPrefixInRowPart(Cell left, Cell right, int rowCommonPrefix) { + return findCommonPrefix(left.getRowArray(), right.getRowArray(), left.getRowLength() + - rowCommonPrefix, right.getRowLength() - rowCommonPrefix, left.getRowOffset() + + rowCommonPrefix, right.getRowOffset() + rowCommonPrefix); + } - //type - c = (0xff & b.getTypeByte()) - (0xff & a.getTypeByte()); - if (c != 0) return c; + private static int findCommonPrefix(byte[] left, byte[] right, int leftLength, int rightLength, + int leftOffset, int rightOffset) { + int length = Math.min(leftLength, rightLength); + int result = 0; - //mvccVersion: later sorts first - return Longs.compare(b.getMvccVersion(), a.getMvccVersion()); + while (result < length && left[leftOffset + result] == right[rightOffset + result]) { + result++; + } + return result; } + public static int findCommonPrefixInFamilyPart(Cell left, Cell right, int familyCommonPrefix) { + return findCommonPrefix(left.getFamilyArray(), right.getFamilyArray(), left.getFamilyLength() + - familyCommonPrefix, right.getFamilyLength() - familyCommonPrefix, left.getFamilyOffset() + + familyCommonPrefix, right.getFamilyOffset() + familyCommonPrefix); + } + + public static int findCommonPrefixInQualifierPart(Cell left, Cell right, + int qualifierCommonPrefix) { + return findCommonPrefix(left.getQualifierArray(), right.getQualifierArray(), + left.getQualifierLength() - qualifierCommonPrefix, right.getQualifierLength() + - qualifierCommonPrefix, left.getQualifierOffset() + qualifierCommonPrefix, + right.getQualifierOffset() + qualifierCommonPrefix); + } /**************** equals ****************************/ @@ -130,6 +161,88 @@ public class CellComparator implements Comparator, Serializable{ return a.getTypeByte() == b.getTypeByte(); } + public static int compareColumns(final Cell left, final Cell right) { + int lfoffset = left.getFamilyOffset(); + int rfoffset = right.getFamilyOffset(); + int lclength = left.getQualifierLength(); + int rclength = right.getQualifierLength(); + int lfamilylength = left.getFamilyLength(); + int rfamilylength = right.getFamilyLength(); + int diff = compare(left.getFamilyArray(), lfoffset, lfamilylength, right.getFamilyArray(), + rfoffset, rfamilylength); + if (diff != 0) { + return diff; + } else { + return compare(left.getQualifierArray(), left.getQualifierOffset(), lclength, + right.getQualifierArray(), right.getQualifierOffset(), rclength); + } + } + + public static int compareFamilies(Cell left, Cell right) { + return Bytes.compareTo(left.getFamilyArray(), left.getFamilyOffset(), left.getFamilyLength(), + right.getFamilyArray(), right.getFamilyOffset(), right.getFamilyLength()); + } + + public static int compareQualifiers(Cell left, Cell right) { + return Bytes.compareTo(left.getQualifierArray(), left.getQualifierOffset(), + left.getQualifierLength(), right.getQualifierArray(), right.getQualifierOffset(), + right.getQualifierLength()); + } + + public int compareFlatKey(Cell left, Cell right) { + int compare = compareRows(left, right); + if (compare != 0) { + return compare; + } + return compareWithoutRow(left, right); + } + + public static int compareRows(final Cell left, final Cell right) { + return Bytes.compareTo(left.getRowArray(), left.getRowOffset(), left.getRowLength(), + right.getRowArray(), right.getRowOffset(), right.getRowLength()); + } + + public static int compareRows(byte[] left, int loffset, int llength, byte[] right, int roffset, + int rlength) { + return Bytes.compareTo(left, loffset, llength, right, roffset, rlength); + } + + public static int compareWithoutRow(final Cell leftCell, final Cell rightCell) { + if (leftCell.getFamilyLength() + leftCell.getQualifierLength() == 0 + && leftCell.getTypeByte() == Type.Minimum.getCode()) { + // left is "bigger", i.e. it appears later in the sorted order + return 1; + } + if (rightCell.getFamilyLength() + rightCell.getQualifierLength() == 0 + && rightCell.getTypeByte() == Type.Minimum.getCode()) { + return -1; + } + boolean sameFamilySize = (leftCell.getFamilyLength() == rightCell.getFamilyLength()); + if (!sameFamilySize) { + // comparing column family is enough. + + return Bytes.compareTo(leftCell.getFamilyArray(), leftCell.getFamilyOffset(), + leftCell.getFamilyLength(), rightCell.getFamilyArray(), rightCell.getFamilyOffset(), + rightCell.getFamilyLength()); + } + int diff = compareColumns(leftCell, rightCell); + if (diff != 0) return diff; + + diff = compareTimestamps(leftCell, rightCell); + if (diff != 0) return diff; + + // Compare types. Let the delete types sort ahead of puts; i.e. types + // of higher numbers sort before those of lesser numbers. Maximum (255) + // appears ahead of everything, and minimum (0) appears after + // everything. + return (0xff & rightCell.getTypeByte()) - (0xff & leftCell.getTypeByte()); + } + + public static int compareTimestamps(final Cell left, final Cell right) { + long ltimestamp = left.getTimestamp(); + long rtimestamp = right.getTimestamp(); + return compareTimestamps(ltimestamp, rtimestamp); + } /********************* hashCode ************************/ @@ -172,44 +285,72 @@ public class CellComparator implements Comparator, Serializable{ } - /***************** special cases ****************************/ + /*********************common prefixes*************************/ + + private static int compare(byte[] left, int leftOffset, int leftLength, byte[] right, + int rightOffset, int rightLength) { + return Bytes.compareTo(left, leftOffset, leftLength, right, rightOffset, rightLength); + } + + public static int compareRowsWithCommonRowPrefix(Cell left, Cell right, int rowCommonPrefix) { + return compare(left.getRowArray(), left.getRowOffset() + rowCommonPrefix, left.getRowLength() + - rowCommonPrefix, right.getRowArray(), right.getRowOffset() + rowCommonPrefix, + right.getRowLength() - rowCommonPrefix); + } + + public static int compareRowsWithCommonFamilyPrefix(Cell left, Cell right, + int familyCommonPrefix) { + return compare(left.getFamilyArray(), left.getFamilyOffset() + familyCommonPrefix, + left.getFamilyLength() - familyCommonPrefix, right.getFamilyArray(), + right.getFamilyOffset() + familyCommonPrefix, + right.getFamilyLength() - familyCommonPrefix); + } + + public static int compareRowsWithCommmonQualifierPrefix(Cell left, Cell right, + int qualCommonPrefix) { + return compare(left.getQualifierArray(), left.getQualifierOffset() + qualCommonPrefix, + left.getQualifierLength() - qualCommonPrefix, right.getQualifierArray(), + right.getQualifierOffset() + qualCommonPrefix, right.getQualifierLength() + - qualCommonPrefix); + } + /***************** special cases ****************************/ /** * special case for KeyValue.equals */ - private static int compareStaticIgnoreMvccVersion(Cell a, Cell b) { - //row - int c = Bytes.compareTo( - a.getRowArray(), a.getRowOffset(), a.getRowLength(), - b.getRowArray(), b.getRowOffset(), b.getRowLength()); - if (c != 0) return c; + public static boolean equalsIgnoreMvccVersion(Cell a, Cell b){ + return 0 == compareStaticIgnoreMvccVersion(a, b); + } - //family - c = Bytes.compareTo( - a.getFamilyArray(), a.getFamilyOffset(), a.getFamilyLength(), - b.getFamilyArray(), b.getFamilyOffset(), b.getFamilyLength()); + private static int compareStaticIgnoreMvccVersion(Cell a, Cell b) { + // row + int c = compareRows(a, b); if (c != 0) return c; - //qualifier - c = Bytes.compareTo( - a.getQualifierArray(), a.getQualifierOffset(), a.getQualifierLength(), - b.getQualifierArray(), b.getQualifierOffset(), b.getQualifierLength()); + // family + c = compareColumns(a, b); if (c != 0) return c; - //timestamp: later sorts first - c = Longs.compare(b.getTimestamp(), a.getTimestamp()); + // timestamp: later sorts first + c = compareTimestamps(a, b); if (c != 0) return c; - //type - c = (0xff & a.getTypeByte()) - (0xff & b.getTypeByte()); + // type + c = (0xff & b.getTypeByte()) - (0xff & a.getTypeByte()); return c; } - /** - * special case for KeyValue.equals - */ - public static boolean equalsIgnoreMvccVersion(Cell a, Cell b){ - return 0 == compareStaticIgnoreMvccVersion(a, b); + private static int compareTimestamps(final long ltimestamp, final long rtimestamp) { + // The below older timestamps sorting ahead of newer timestamps looks + // wrong but it is intentional. This way, newer timestamps are first + // found when we iterate over a memstore and newer versions are the + // first we trip over when reading from a store file. + if (ltimestamp < rtimestamp) { + return 1; + } else if (ltimestamp > rtimestamp) { + return -1; + } + return 0; } } diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java index ce7aa37..b02d227 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -43,7 +43,7 @@ import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.RawComparator; -import com.google.common.primitives.Longs; +import com.google.common.annotations.VisibleForTesting; /** * An HBase Key/Value. This is the fundamental HBase Type. @@ -1839,6 +1839,20 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * table. */ @Override + public int compare(final Cell left, final Cell right) { + int c = compareRowKey(left, right); + if (c != 0) { + return c; + } + return CellComparator.compareWithoutRow(left, right); + } + + @Override + public int compareOnlyKeyPortion(Cell left, Cell right) { + return compare(left, right); + } + + @Override public int compareRows(byte [] left, int loffset, int llength, byte [] right, int roffset, int rlength) { int leftDelimiter = getDelimiter(left, loffset, llength, @@ -1952,9 +1966,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * @return 0 if equal, <0 if left smaller, >0 if right smaller */ protected int compareRowKey(final Cell left, final Cell right) { - return Bytes.compareTo( - left.getRowArray(), left.getRowOffset(), left.getRowLength(), - right.getRowArray(), right.getRowOffset(), right.getRowLength()); + return CellComparator.compareRows(left, right); } /** @@ -1990,109 +2002,22 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return compareFlatKey(left, 0, left.length, right, 0, right.length); } + public int compareOnlyKeyPortion(Cell left, Cell right) { + return CellComparator.compareStatic(left, right, true); + } + /** * Compares the Key of a cell -- with fields being more significant in this order: * rowkey, colfam/qual, timestamp, type, mvcc */ + @Override public int compare(final Cell left, final Cell right) { - // compare row - int compare = compareRowKey(left, right); - if (compare != 0) { - return compare; - } - - // compare vs minimum - byte ltype = left.getTypeByte(); - byte rtype = right.getTypeByte(); - // If the column is not specified, the "minimum" key type appears the - // latest in the sorted order, regardless of the timestamp. This is used - // for specifying the last key/value in a given row, because there is no - // "lexicographically last column" (it would be infinitely long). The - // "maximum" key type does not need this behavior. - int lcfqLen = left.getFamilyLength() + left.getQualifierLength() ; - int rcfqLen = right.getFamilyLength() + right.getQualifierLength() ; - if (lcfqLen == 0 && ltype == Type.Minimum.getCode()) { - // left is "bigger", i.e. it appears later in the sorted order - return 1; - } - if (rcfqLen == 0 && rtype == Type.Minimum.getCode()) { - return -1; - } - - - // compare col family / col fam + qual - // If left family size is not equal to right family size, we need not - // compare the qualifiers. - compare = Bytes.compareTo( - left.getFamilyArray(), left.getFamilyOffset(), left.getFamilyLength(), - right.getFamilyArray(), right.getFamilyOffset(), right.getFamilyLength()); - if (compare != 0) { - return compare; - } - - // Compare qualifier - compare = Bytes.compareTo( - left.getQualifierArray(), left.getQualifierOffset(), left.getQualifierLength(), - right.getQualifierArray(), right.getQualifierOffset(), right.getQualifierLength()); - if (compare!= 0) { - return compare; - } - - // compare timestamp - long ltimestamp = left.getTimestamp(); - long rtimestamp = right.getTimestamp(); - compare = compareTimestamps(ltimestamp, rtimestamp); - if (compare != 0) { - return compare; - } - - // Compare types. Let the delete types sort ahead of puts; i.e. types - // of higher numbers sort before those of lesser numbers. Maximum (255) - // appears ahead of everything, and minimum (0) appears after - // everything. - compare = (0xff & rtype) - (0xff & ltype); - if (compare != 0) { - return compare; - } - - // Negate following comparisons so later edits show up first - - // compare log replay tag value if there is any - // when either keyvalue tagged with log replay sequence number, we need to compare them: - // 1) when both keyvalues have the tag, then use the tag values for comparison - // 2) when one has and the other doesn't have, the one without the log replay tag wins because - // it means the edit isn't from recovery but new one coming from clients during recovery - // 3) when both doesn't have, then skip to the next mvcc comparison - long leftChangeSeqNum = getReplaySeqNum(left); - long RightChangeSeqNum = getReplaySeqNum(right); - if (leftChangeSeqNum != Long.MAX_VALUE || RightChangeSeqNum != Long.MAX_VALUE) { - return Longs.compare(RightChangeSeqNum, leftChangeSeqNum); - } - - // compare Mvcc Version - return Longs.compare(right.getMvccVersion(), left.getMvccVersion()); - } - - /** - * Return replay log sequence number for the cell - * @param c - * @return Long.MAX_VALUE if there is no LOG_REPLAY_TAG - */ - private long getReplaySeqNum(final Cell c) { - Tag tag = Tag.getTag(c.getTagsArray(), c.getTagsOffset(), c.getTagsLength(), - TagType.LOG_REPLAY_TAG_TYPE); - - if(tag != null) { - return Bytes.toLong(tag.getBuffer(), tag.getTagOffset(), tag.getTagLength()); - } - return Long.MAX_VALUE; + int compare = CellComparator.compareStatic(left, right, false); + return compare; } - public int compareTimestamps(final KeyValue left, final KeyValue right) { - // Compare timestamps - long ltimestamp = left.getTimestamp(left.getKeyLength()); - long rtimestamp = right.getTimestamp(right.getKeyLength()); - return compareTimestamps(ltimestamp, rtimestamp); + public int compareTimestamps(final Cell left, final Cell right) { + return CellComparator.compareTimestamps(left, right); } /** @@ -2100,7 +2025,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * @param right * @return Result comparing rows. */ - public int compareRows(final KeyValue left, final KeyValue right) { + public int compareRows(final Cell left, final Cell right) { return compareRows(left.getRowArray(),left.getRowOffset(), left.getRowLength(), right.getRowArray(), right.getRowOffset(), right.getRowLength()); } @@ -2120,17 +2045,9 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return Bytes.compareTo(left, loffset, llength, right, roffset, rlength); } - int compareColumns(final KeyValue left, final short lrowlength, - final KeyValue right, final short rrowlength) { - int lfoffset = left.getFamilyOffset(lrowlength); - int rfoffset = right.getFamilyOffset(rrowlength); - int lclength = left.getTotalColumnLength(lrowlength,lfoffset); - int rclength = right.getTotalColumnLength(rrowlength, rfoffset); - int lfamilylength = left.getFamilyLength(lfoffset); - int rfamilylength = right.getFamilyLength(rfoffset); - return compareColumns(left.getBuffer(), lfoffset, - lclength, lfamilylength, - right.getBuffer(), rfoffset, rclength, rfamilylength); + int compareColumns(final Cell left, final short lrowlength, final Cell right, + final short rrowlength) { + return CellComparator.compareColumns(left, right); } protected int compareColumns( @@ -2297,20 +2214,31 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return (0xff & rtype) - (0xff & ltype); } + protected int compareFamilies(final byte[] left, final int loffset, final int lfamilylength, + final byte[] right, final int roffset, final int rfamilylength) { + int diff = Bytes.compareTo(left, loffset, lfamilylength, right, roffset, rfamilylength); + return diff; + } + + protected int compareColumns(final byte[] left, final int loffset, final int lquallength, + final byte[] right, final int roffset, final int rquallength) { + int diff = Bytes.compareTo(left, loffset, lquallength, right, roffset, rquallength); + return diff; + } /** * Compares the row and column of two keyvalues for equality * @param left * @param right * @return True if same row and column. */ - public boolean matchingRowColumn(final KeyValue left, - final KeyValue right) { + public boolean matchingRowColumn(final Cell left, + final Cell right) { short lrowlength = left.getRowLength(); short rrowlength = right.getRowLength(); // TsOffset = end of column data. just comparing Row+CF length of each - if ((left.getTimestampOffset() - left.getOffset()) != - (right.getTimestampOffset() - right.getOffset())) { + if ((left.getRowLength() + left.getFamilyLength() + left.getQualifierLength()) != (right + .getRowLength() + right.getFamilyLength() + right.getQualifierLength())) { return false; } @@ -2318,15 +2246,21 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return false; } - int lfoffset = left.getFamilyOffset(lrowlength); - int rfoffset = right.getFamilyOffset(rrowlength); - int lclength = left.getTotalColumnLength(lrowlength,lfoffset); - int rclength = right.getTotalColumnLength(rrowlength, rfoffset); - int lfamilylength = left.getFamilyLength(lfoffset); - int rfamilylength = right.getFamilyLength(rfoffset); - int ccRes = compareColumns(left.getBuffer(), lfoffset, lclength, lfamilylength, - right.getBuffer(), rfoffset, rclength, rfamilylength); - return ccRes == 0; + int lfoffset = left.getFamilyOffset(); + int rfoffset = right.getFamilyOffset(); + int lclength = left.getQualifierLength(); + int rclength = right.getQualifierLength(); + int lfamilylength = left.getFamilyLength(); + int rfamilylength = right.getFamilyLength(); + int diff = compareFamilies(left.getFamilyArray(), lfoffset, lfamilylength, + right.getFamilyArray(), rfoffset, rfamilylength); + if (diff != 0) { + return false; + } else { + diff = compareColumns(left.getQualifierArray(), left.getQualifierOffset(), lclength, + right.getQualifierArray(), right.getQualifierOffset(), rclength); + return diff == 0; + } } /** @@ -2335,7 +2269,7 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * @param right * @return True if rows match. */ - public boolean matchingRows(final KeyValue left, final KeyValue right) { + public boolean matchingRows(final Cell left, final Cell right) { short lrowlength = left.getRowLength(); short rrowlength = right.getRowLength(); return matchingRows(left, lrowlength, right, rrowlength); @@ -2348,8 +2282,8 @@ public class KeyValue implements Cell, HeapSize, Cloneable { * @param rrowlength * @return True if rows match. */ - private boolean matchingRows(final KeyValue left, final short lrowlength, - final KeyValue right, final short rrowlength) { + private boolean matchingRows(final Cell left, final short lrowlength, + final Cell right, final short rrowlength) { return lrowlength == rrowlength && matchingRows(left.getRowArray(), left.getRowOffset(), lrowlength, right.getRowArray(), right.getRowOffset(), rrowlength); @@ -2929,6 +2863,40 @@ public class KeyValue implements Cell, HeapSize, Cloneable { return Bytes.BYTES_RAWCOMPARATOR.compare(left, loffset, llength, right, roffset, rlength); } + @Override + public int compare(Cell left, Cell right) { + return compareOnlyKeyPortion(left, right); + } + + @VisibleForTesting + public int compareOnlyKeyPortion(Cell left, Cell right) { + int c = Bytes.BYTES_RAWCOMPARATOR.compare(left.getRowArray(), left.getRowOffset(), + left.getRowLength(), right.getRowArray(), right.getRowOffset(), right.getRowLength()); + if (c != 0) { + return c; + } + c = Bytes.BYTES_RAWCOMPARATOR.compare(left.getFamilyArray(), left.getFamilyOffset(), + left.getFamilyLength(), right.getFamilyArray(), right.getFamilyOffset(), + right.getFamilyLength()); + if (c != 0) { + return c; + } + c = Bytes.BYTES_RAWCOMPARATOR.compare(left.getQualifierArray(), left.getQualifierOffset(), + left.getQualifierLength(), right.getQualifierArray(), right.getQualifierOffset(), + right.getQualifierLength()); + if (c != 0) { + return c; + } + c = compareTimestamps(left.getTimestamp(), right.getTimestamp()); + if (c != 0) { + return c; + } + + return (0xff & left.getTypeByte()) - (0xff & right.getTypeByte()); + + // TODO : we may have to check timestamp and type also + } + public byte[] calcIndexKey(byte[] lastKeyOfPreviousBlock, byte[] firstKeyInBlock) { return firstKeyInBlock; } @@ -2952,4 +2920,154 @@ public class KeyValue implements Cell, HeapSize, Cloneable { sum += Bytes.SIZEOF_LONG;// memstoreTS return ClassSize.align(sum); } + + /** + * A simple form of KeyValue that creates a keyvalue with only the key part of the byte[] + * Mainly used in places where we need to compare two cells. Avoids copying of bytes + * In places like block index keys, we need to compare the key byte[] with a cell. + * Hence create a Keyvalue(aka Cell) that would help in comparing as two cells + */ + public static class KeyOnlyKeyValue extends KeyValue { + private int length = 0; + private int offset = 0; + private byte[] b; + + public KeyOnlyKeyValue(byte[] b, int offset, int length) { + this.b = b; + this.length = length; + this.offset = offset; + } + + @Override + public int getKeyOffset() { + return this.offset; + } + + @Override + public byte[] getKey() { + int keylength = getKeyLength(); + byte[] key = new byte[keylength]; + System.arraycopy(this.b, getKeyOffset(), key, 0, keylength); + return key; + } + + @Override + public byte[] getRowArray() { + return b; + } + + @Override + public int getRowOffset() { + return getKeyOffset() + Bytes.SIZEOF_SHORT; + } + + @Override + public byte[] getFamilyArray() { + return b; + } + + @Override + public byte getFamilyLength() { + return this.b[getFamilyOffset() - 1]; + } + + @Override + public int getFamilyOffset() { + return this.offset + Bytes.SIZEOF_SHORT + getRowLength() + Bytes.SIZEOF_BYTE; + } + + @Override + public byte[] getQualifierArray() { + return b; + } + + @Override + public int getQualifierLength() { + return getQualifierLength(getRowLength(), getFamilyLength()); + } + + @Override + public int getQualifierOffset() { + return getFamilyOffset() + getFamilyLength(); + } + + @Override + public int getKeyLength() { + return length; + } + + @Override + public short getRowLength() { + return Bytes.toShort(this.b, getKeyOffset()); + } + + @Override + public byte getTypeByte() { + return this.b[this.offset + getKeyLength() - 1]; + } + + private int getQualifierLength(int rlength, int flength) { + return getKeyLength() - (int) getKeyDataStructureSize(rlength, flength, 0); + } + + @Override + public long getTimestamp() { + int tsOffset = getTimestampOffset(); + return Bytes.toLong(this.b, tsOffset); + } + + @Override + public int getTimestampOffset() { + return getKeyOffset() + getKeyLength() - TIMESTAMP_TYPE_SIZE; + } + + @Override + public byte[] getTagsArray() { + return HConstants.EMPTY_BYTE_ARRAY; + } + + @Override + public int getTagsOffset() { + return (short) 0; + } + + @Override + public byte[] getValueArray() { + throw new IllegalArgumentException("KeyOnlyKeyValue does not work with values."); + } + + @Override + public int getValueOffset() { + throw new IllegalArgumentException("KeyOnlyKeyValue does not work with values."); + } + + @Override + public int getValueLength() { + throw new IllegalArgumentException("KeyOnlyKeyValue does not work with values."); + } + + @Override + public short getTagsLength() { + return (short) 0; + } + + @Override + public String toString() { + if (this.b == null || this.b.length == 0) { + return "empty"; + } + return keyToString(this.b, this.offset + ROW_OFFSET, getKeyLength()) + "/vlen=" + + getValueLength() + "/mvcc=" + 0; + } + + @Override + public int hashCode() { + return super.hashCode(); + } + + @Override + public boolean equals(Object other) { + return super.equals(other); + } + } } diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java index d95ee73..4c4f80d 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/BufferedDataBlockEncoder.java @@ -22,10 +22,13 @@ import java.io.IOException; import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.KeyValue.SamePrefixComparator; +import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.io.TagCompressionContext; import org.apache.hadoop.hbase.io.hfile.BlockType; import org.apache.hadoop.hbase.io.hfile.HFileContext; @@ -194,6 +197,12 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { } @Override + public int compareKey(KVComparator comparator, Cell key) { + return comparator.compareOnlyKeyPortion(key, + new KeyValue.KeyOnlyKeyValue(current.keyBuffer, 0, current.keyLength)); + } + + @Override public void setCurrentBuffer(ByteBuffer buffer) { if (this.tagCompressionContext != null) { this.tagCompressionContext.clear(); @@ -304,26 +313,62 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { } @Override - public int seekToKeyInBlock(byte[] key, int offset, int length, - boolean seekBefore) { + public int seekToKeyInBlock(byte[] key, int offset, int length, boolean seekBefore) { + return seekToKeyInBlock(new KeyValue.KeyOnlyKeyValue(key, offset, length), seekBefore); + } + + @Override + public int seekToKeyInBlock(Cell key, boolean seekBefore) { int commonPrefix = 0; + int rowCommonPrefix = 0; + int familyCommonPrefix = 0; + int qualCommonPrefix = 0; previous.invalidate(); do { int comp; if (samePrefixComparator != null) { + Cell right = new KeyValue.KeyOnlyKeyValue(current.keyBuffer, 0, current.keyLength); commonPrefix = Math.min(commonPrefix, current.lastCommonPrefix); - - // extend commonPrefix - commonPrefix += ByteBufferUtils.findCommonPrefix( - key, offset + commonPrefix, length - commonPrefix, - current.keyBuffer, commonPrefix, - current.keyLength - commonPrefix); - - comp = samePrefixComparator.compareIgnoringPrefix(commonPrefix, key, - offset, length, current.keyBuffer, 0, current.keyLength); + rowCommonPrefix += Math.min(rowCommonPrefix, + Math.min(key.getRowLength(), right.getRowLength())); + commonPrefix += CellComparator.findCommonPrefixInRowPart(key, right, rowCommonPrefix); + comp = CellComparator.compareRowsWithCommonRowPrefix(key, right, rowCommonPrefix); + if (comp == 0) { + comp = compareTypeBytes(key, right); + if (comp == 0) { + familyCommonPrefix += Math.min(familyCommonPrefix, + Math.min(key.getFamilyLength(), right.getFamilyLength())); + commonPrefix += CellComparator.findCommonPrefixInFamilyPart(key, right, + familyCommonPrefix); + comp = CellComparator.compareRowsWithCommonFamilyPrefix(key, right, + familyCommonPrefix); + if (comp == 0) { + qualCommonPrefix += Math.min(qualCommonPrefix, + Math.min(key.getQualifierLength(), right.getQualifierLength())); + commonPrefix += CellComparator.findCommonPrefixInQualifierPart(key, right, + qualCommonPrefix); + comp = CellComparator.compareRowsWithCommmonQualifierPrefix(key, right, + qualCommonPrefix); + if (comp == 0) { + comp = CellComparator.compareTimestamps(key, right); + if (comp == 0) { + // Compare types. Let the delete types sort ahead of puts; + // i.e. types + // of higher numbers sort before those of lesser numbers. + // Maximum + // (255) + // appears ahead of everything, and minimum (0) appears + // after + // everything. + comp = (0xff & right.getTypeByte()) - (0xff & key.getTypeByte()); + } + } + } + } + } } else { - comp = comparator.compareFlatKey(key, offset, length, - current.keyBuffer, 0, current.keyLength); + Cell right = new KeyValue.KeyOnlyKeyValue(current.keyBuffer, 0, current.keyLength); + comp = comparator.compareOnlyKeyPortion(key, right); } if (comp == 0) { // exact match @@ -331,9 +376,9 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { if (!previous.isValid()) { // The caller (seekBefore) has to ensure that we are not at the // first key in the block. - throw new IllegalStateException("Cannot seekBefore if " + - "positioned at the first key in the block: key=" + - Bytes.toStringBinary(key, offset, length)); + throw new IllegalStateException("Cannot seekBefore if " + + "positioned at the first key in the block: key=" + + Bytes.toStringBinary(key.getRowArray())); } moveToPrevious(); return 1; @@ -363,6 +408,20 @@ abstract class BufferedDataBlockEncoder implements DataBlockEncoder { return 1; } + private int compareTypeBytes(Cell key, Cell right) { + if (key.getFamilyLength() + key.getQualifierLength() == 0 + && key.getTypeByte() == Type.Minimum.getCode()) { + // left is "bigger", i.e. it appears later in the sorted order + return 1; + } + if (right.getFamilyLength() + right.getQualifierLength() == 0 + && right.getTypeByte() == Type.Minimum.getCode()) { + return -1; + } + return 0; + } + + private void moveToPrevious() { if (!previous.isValid()) { throw new IllegalStateException( diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java index 6185ab3..3c06815 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/io/encoding/DataBlockEncoder.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; import org.apache.hadoop.hbase.io.hfile.HFileContext; @@ -174,9 +175,26 @@ public interface DataBlockEncoder { * of an exact match. Does not matter in case of an inexact match. * @return 0 on exact match, 1 on inexact match. */ + @Deprecated int seekToKeyInBlock( byte[] key, int offset, int length, boolean seekBefore ); + /** + * Moves the seeker position within the current block to: + *
    + *
  • the last key that that is less than or equal to the given key if + * seekBefore is false
  • + *
  • the last key that is strictly less than the given key if + * seekBefore is true. The caller is responsible for loading the + * previous block if the requested key turns out to be the first key of the + * current block.
  • + *
+ * @param key - Cell to which the seek should happen + * @param seekBefore find the key strictly less than the given key in case + * of an exact match. Does not matter in case of an inexact match. + * @return 0 on exact match, 1 on inexact match. + */ + int seekToKeyInBlock(Cell key, boolean seekBefore); /** * Compare the given key against the current key @@ -187,5 +205,7 @@ public interface DataBlockEncoder { * @return -1 is the passed key is smaller than the current key, 0 if equal and 1 if greater */ public int compareKey(KVComparator comparator, byte[] key, int offset, int length); + + public int compareKey(KVComparator comparator, Cell key); } } diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java index 4679194..4f8b9ce 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/util/Bytes.java @@ -43,6 +43,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.io.RawComparator; import org.apache.hadoop.io.WritableComparator; @@ -1635,6 +1637,43 @@ public class Bytes { } /** + * Binary search for keys in indexes. + * + * @param arr array of byte arrays to search for + * @param key the key you want to find + * @param comparator a comparator to compare. + * @return zero-based index of the key, if the key is present in the array. + * Otherwise, a value -(i + 1) such that the key is between arr[i - + * 1] and arr[i] non-inclusively, where i is in [0, i], if we define + * arr[-1] = -Inf and arr[N] = Inf for an N-element array. The above + * means that this function can return 2N + 1 different values + * ranging from -(N + 1) to N - 1. + * @return the index of the block + */ + public static int binarySearch(byte[][] arr, Cell key, RawComparator comparator) { + int low = 0; + int high = arr.length - 1; + + while (low <= high) { + int mid = (low+high) >>> 1; + // we have to compare in this order, because the comparator order + // has special logic when the 'left side' is a special key. + Cell r = new KeyValue.KeyOnlyKeyValue(arr[mid], 0, arr[mid].length); + int cmp = comparator.compare(key, r); + // key lives above the midpoint + if (cmp > 0) + low = mid + 1; + // key lives below the midpoint + else if (cmp < 0) + high = mid - 1; + // BAM. how often does this really happen? + else + return mid; + } + return - (low+1); + } + + /** * Bytewise binary increment/deincrement of long contained in byte array * on given amount. * diff --git hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellComparator.java hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellComparator.java new file mode 100644 index 0000000..8c99e57 --- /dev/null +++ hbase-common/src/test/java/org/apache/hadoop/hbase/TestCellComparator.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase; + +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.KeyValue.Type; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.Test; +import org.junit.experimental.categories.Category; +@Category(SmallTests.class) +public class TestCellComparator { + + byte[] row1 = Bytes.toBytes("row1"); + byte[] row2 = Bytes.toBytes("row2"); + byte[] row_1_0 = Bytes.toBytes("row10"); + + byte[] fam1 = Bytes.toBytes("fam1"); + byte[] fam2 = Bytes.toBytes("fam2"); + byte[] fam_1_2 = Bytes.toBytes("fam12"); + + byte[] qual1 = Bytes.toBytes("qual1"); + byte[] qual2 = Bytes.toBytes("qual2"); + + byte[] val = Bytes.toBytes("val"); + + @Test + public void testCompareCells() { + KeyValue kv1 = new KeyValue(row1, fam1, qual1, val); + KeyValue kv2 = new KeyValue(row2, fam1, qual1, val); + assertTrue((CellComparator.compareStatic(kv1, kv2, false)) < 0); + + kv1 = new KeyValue(row1, fam2, qual1, val); + kv2 = new KeyValue(row1, fam1, qual1, val); + assertTrue((CellComparator.compareFamilies(kv1, kv2) > 0)); + + kv1 = new KeyValue(row1, fam1, qual1, 1l, val); + kv2 = new KeyValue(row1, fam1, qual1, 2l, val); + assertTrue((CellComparator.compareStatic(kv1, kv2, false) > 0)); + + kv1 = new KeyValue(row1, fam1, qual1, 1l, Type.Put); + kv2 = new KeyValue(row1, fam1, qual1, 1l, Type.Maximum); + assertTrue((CellComparator.compareStatic(kv1, kv2, false) > 0)); + + kv1 = new KeyValue(row1, fam1, qual1, 1l, Type.Put); + kv2 = new KeyValue(row1, fam_1_2, qual1, 1l, Type.Maximum); + assertTrue((CellComparator.compareRowsWithCommonFamilyPrefix(kv1, kv2, 4) < 0)); + + kv1 = new KeyValue(row1, fam1, qual1, 1l, Type.Put); + kv2 = new KeyValue(row_1_0, fam_1_2, qual1, 1l, Type.Maximum); + assertTrue((CellComparator.compareRowsWithCommonRowPrefix(kv1, kv2, 4) < 0)); + + kv1 = new KeyValue(row1, fam1, qual2, 1l, Type.Put); + kv2 = new KeyValue(row1, fam1, qual1, 1l, Type.Maximum); + assertTrue((CellComparator.compareRowsWithCommmonQualifierPrefix(kv1, kv2, 4) > 0)); + + kv1 = new KeyValue(row1, fam1, qual1, 1l, Type.Put); + kv2 = new KeyValue(row1, fam1, qual1, 1l, Type.Put); + assertTrue((CellComparator.equals(kv1, kv2))); + } +} diff --git hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java index d673457..65dd1ce 100644 --- hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java +++ hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/PrefixTreeSeeker.java @@ -24,8 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellUtil; import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.KeyValue.KVComparator; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.codec.prefixtree.decode.DecoderFactory; import org.apache.hadoop.hbase.codec.prefixtree.decode.PrefixTreeArraySearcher; import org.apache.hadoop.hbase.codec.prefixtree.scanner.CellScannerPosition; @@ -152,15 +152,13 @@ public class PrefixTreeSeeker implements EncodedSeeker { boolean forceBeforeOnExactMatch) { if (USE_POSITION_BEFORE) { return seekToOrBeforeUsingPositionAtOrBefore(keyOnlyBytes, offset, length, - forceBeforeOnExactMatch); - }else{ + forceBeforeOnExactMatch); + } else { return seekToOrBeforeUsingPositionAtOrAfter(keyOnlyBytes, offset, length, - forceBeforeOnExactMatch); + forceBeforeOnExactMatch); } } - - /* * Support both of these options since the underlying PrefixTree supports both. Possibly * expand the EncodedSeeker to utilize them both. @@ -169,11 +167,22 @@ public class PrefixTreeSeeker implements EncodedSeeker { protected int seekToOrBeforeUsingPositionAtOrBefore(byte[] keyOnlyBytes, int offset, int length, boolean seekBefore){ // this does a deep copy of the key byte[] because the CellSearcher interface wants a Cell - KeyValue kv = KeyValue.createKeyValueFromKey(keyOnlyBytes, offset, length); + KeyValue kv = new KeyValue.KeyOnlyKeyValue(keyOnlyBytes, offset, length); + + return seekToOrBeforeUsingPositionAtOrBefore(kv, seekBefore); + } + + /* + * Support both of these options since the underlying PrefixTree supports + * both. Possibly expand the EncodedSeeker to utilize them both. + */ + protected int seekToOrBeforeUsingPositionAtOrBefore(Cell kv, boolean seekBefore) { + // this does a deep copy of the key byte[] because the CellSearcher + // interface wants a Cell CellScannerPosition position = ptSearcher.seekForwardToOrBefore(kv); - if(CellScannerPosition.AT == position){ + if (CellScannerPosition.AT == position) { if (seekBefore) { ptSearcher.previous(); return 1; @@ -184,16 +193,19 @@ public class PrefixTreeSeeker implements EncodedSeeker { return 1; } - protected int seekToOrBeforeUsingPositionAtOrAfter(byte[] keyOnlyBytes, int offset, int length, - boolean seekBefore){ - // this does a deep copy of the key byte[] because the CellSearcher interface wants a Cell - KeyValue kv = KeyValue.createKeyValueFromKey(keyOnlyBytes, offset, length); + boolean seekBefore) { + // this does a deep copy of the key byte[] because the CellSearcher + // interface wants a Cell + KeyValue kv = new KeyValue.KeyOnlyKeyValue(keyOnlyBytes, offset, length); + return seekToOrBeforeUsingPositionAtOrAfter(kv, seekBefore); + } - //should probably switch this to use the seekForwardToOrBefore method + protected int seekToOrBeforeUsingPositionAtOrAfter(Cell kv, boolean seekBefore) { + // should probably switch this to use the seekForwardToOrBefore method CellScannerPosition position = ptSearcher.seekForwardToOrAfter(kv); - if(CellScannerPosition.AT == position){ + if (CellScannerPosition.AT == position) { if (seekBefore) { ptSearcher.previous(); return 1; @@ -202,21 +214,21 @@ public class PrefixTreeSeeker implements EncodedSeeker { } - if(CellScannerPosition.AFTER == position){ - if(!ptSearcher.isBeforeFirst()){ + if (CellScannerPosition.AFTER == position) { + if (!ptSearcher.isBeforeFirst()) { ptSearcher.previous(); } return 1; } - if(position == CellScannerPosition.AFTER_LAST){ + if (position == CellScannerPosition.AFTER_LAST) { if (seekBefore) { ptSearcher.previous(); } return 1; } - throw new RuntimeException("unexpected CellScannerPosition:"+position); + throw new RuntimeException("unexpected CellScannerPosition:" + position); } @Override @@ -225,4 +237,20 @@ public class PrefixTreeSeeker implements EncodedSeeker { ByteBuffer bb = getKeyDeepCopy(); return comparator.compareFlatKey(key, offset, length, bb.array(), bb.arrayOffset(), bb.limit()); } + + @Override + public int seekToKeyInBlock(Cell key, boolean forceBeforeOnExactMatch) { + if (USE_POSITION_BEFORE) { + return seekToOrBeforeUsingPositionAtOrBefore(key, forceBeforeOnExactMatch); + }else{ + return seekToOrBeforeUsingPositionAtOrAfter(key, forceBeforeOnExactMatch); + } + } + + @Override + public int compareKey(KVComparator comparator, Cell key) { + ByteBuffer bb = getKeyDeepCopy(); + return comparator.compare(key, + new KeyValue.KeyOnlyKeyValue(bb.array(), bb.arrayOffset(), bb.limit())); + } } diff --git hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java index 1326983..c20626e 100644 --- hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java +++ hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeArrayScanner.java @@ -28,7 +28,6 @@ import org.apache.hadoop.hbase.codec.prefixtree.decode.row.RowNodeReader; import org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp.MvccVersionDecoder; import org.apache.hadoop.hbase.codec.prefixtree.decode.timestamp.TimestampDecoder; import org.apache.hadoop.hbase.codec.prefixtree.encode.other.ColumnNodeType; -import org.apache.hadoop.hbase.util.Bytes; /** * Extends PtCell and manipulates its protected fields. Could alternatively contain a PtCell and @@ -420,7 +419,7 @@ public class PrefixTreeArrayScanner extends PrefixTreeCell implements CellScanne protected int populateNonRowFieldsAndCompareTo(int cellNum, Cell key) { populateNonRowFields(cellNum); - return CellComparator.compareStatic(this, key); + return CellComparator.compareStatic(this, key, false); } protected void populateFirstNonRowFields() { diff --git hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java index 390e802..740a08e 100644 --- hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java +++ hbase-prefix-tree/src/main/java/org/apache/hadoop/hbase/codec/prefixtree/decode/PrefixTreeCell.java @@ -106,7 +106,7 @@ public class PrefixTreeCell implements Cell, Comparable { @Override public int compareTo(Cell other) { - return CellComparator.compareStatic(this, other); + return CellComparator.compareStatic(this, other, false); } @Override diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java index 736f330..ac50cd0 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/HalfStoreFileReader.java @@ -27,6 +27,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.client.Scan; @@ -55,9 +56,11 @@ public class HalfStoreFileReader extends StoreFile.Reader { // This is the key we split around. Its the first possible entry on a row: // i.e. empty column and a timestamp of LATEST_TIMESTAMP. protected final byte [] splitkey; - + + protected final Cell splitCell; + private byte[] firstKey = null; - + private boolean firstKeySeeked = false; /** @@ -79,6 +82,7 @@ public class HalfStoreFileReader extends StoreFile.Reader { // have an actual midkey themselves. No midkey is how we indicate file is // not splittable. this.splitkey = r.getSplitKey(); + this.splitCell = new KeyValue.KeyOnlyKeyValue(this.splitkey, 0, this.splitkey.length); // Is it top or bottom half? this.top = Reference.isTopFileRegion(r.getFileRegion()); } @@ -104,6 +108,7 @@ public class HalfStoreFileReader extends StoreFile.Reader { // have an actual midkey themselves. No midkey is how we indicate file is // not splittable. this.splitkey = r.getSplitKey(); + this.splitCell = new KeyValue.KeyOnlyKeyValue(this.splitkey, 0, this.splitkey.length); // Is it top or bottom half? this.top = Reference.isTopFileRegion(r.getFileRegion()); } @@ -168,33 +173,21 @@ public class HalfStoreFileReader extends StoreFile.Reader { return true; } + @Override public boolean seekBefore(byte[] key) throws IOException { return seekBefore(key, 0, key.length); } + @Override public boolean seekBefore(byte [] key, int offset, int length) throws IOException { - if (top) { - byte[] fk = getFirstKey(); - // This will be null when the file is empty in which we can not seekBefore to any key - if (fk == null) return false; - if (getComparator().compareFlatKey(key, offset, length, fk, 0, - fk.length) <= 0) { - return false; - } - } else { - // The equals sign isn't strictly necessary just here to be consistent with seekTo - if (getComparator().compareFlatKey(key, offset, length, splitkey, 0, - splitkey.length) >= 0) { - return this.delegate.seekBefore(splitkey, 0, splitkey.length); - } - } - return this.delegate.seekBefore(key, offset, length); + return seekBefore(new KeyValue.KeyOnlyKeyValue(key, offset, length)); } + @Override public boolean seekTo() throws IOException { if (top) { - int r = this.delegate.seekTo(splitkey); + int r = this.delegate.seekTo(new KeyValue.KeyOnlyKeyValue(splitkey, 0, splitkey.length)); if (r == HConstants.INDEX_KEY_MAGIC) { return true; } @@ -219,55 +212,76 @@ public class HalfStoreFileReader extends StoreFile.Reader { splitkey, 0, splitkey.length) < 0; } + @Override public int seekTo(byte[] key) throws IOException { return seekTo(key, 0, key.length); } + @Override public int seekTo(byte[] key, int offset, int length) throws IOException { + return seekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length)); + } + + @Override + public int reseekTo(byte[] key) throws IOException { + return reseekTo(key, 0, key.length); + } + + @Override + public int reseekTo(byte[] key, int offset, int length) + throws IOException { + //This function is identical to the corresponding seekTo function except + //that we call reseekTo (and not seekTo) on the delegate. + return reseekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length)); + } + + public org.apache.hadoop.hbase.io.hfile.HFile.Reader getReader() { + return this.delegate.getReader(); + } + + public boolean isSeeked() { + return this.delegate.isSeeked(); + } + + @Override + public int seekTo(Cell key) throws IOException { if (top) { - if (getComparator().compareFlatKey(key, offset, length, splitkey, 0, - splitkey.length) < 0) { + if (getComparator().compareOnlyKeyPortion(key, splitCell) < 0) { return -1; } } else { - if (getComparator().compareFlatKey(key, offset, length, splitkey, 0, - splitkey.length) >= 0) { + if (getComparator().compareOnlyKeyPortion(key, splitCell) >= 0) { // we would place the scanner in the second half. // it might be an error to return false here ever... - boolean res = delegate.seekBefore(splitkey, 0, splitkey.length); + boolean res = delegate.seekBefore(splitCell); if (!res) { - throw new IOException("Seeking for a key in bottom of file, but key exists in top of file, failed on seekBefore(midkey)"); + throw new IOException( + "Seeking for a key in bottom of file, but key exists in top of file, " + + "failed on seekBefore(midkey)"); } return 1; } } - return delegate.seekTo(key, offset, length); - } - - @Override - public int reseekTo(byte[] key) throws IOException { - return reseekTo(key, 0, key.length); + return delegate.seekTo(key); } @Override - public int reseekTo(byte[] key, int offset, int length) - throws IOException { - //This function is identical to the corresponding seekTo function except - //that we call reseekTo (and not seekTo) on the delegate. + public int reseekTo(Cell key) throws IOException { + // This function is identical to the corresponding seekTo function + // except + // that we call reseekTo (and not seekTo) on the delegate. if (top) { - if (getComparator().compareFlatKey(key, offset, length, splitkey, 0, - splitkey.length) < 0) { + if (getComparator().compareOnlyKeyPortion(key, splitCell) < 0) { return -1; } } else { - if (getComparator().compareFlatKey(key, offset, length, splitkey, 0, - splitkey.length) >= 0) { + if (getComparator().compareOnlyKeyPortion(key, splitCell) >= 0) { // we would place the scanner in the second half. // it might be an error to return false here ever... - boolean res = delegate.seekBefore(splitkey, 0, splitkey.length); + boolean res = delegate.seekBefore(splitCell); if (!res) { - throw new IOException("Seeking for a key in bottom of file, but" + - " key exists in top of file, failed on seekBefore(midkey)"); + throw new IOException("Seeking for a key in bottom of file, but" + + " key exists in top of file, failed on seekBefore(midkey)"); } return 1; } @@ -276,15 +290,28 @@ public class HalfStoreFileReader extends StoreFile.Reader { // skip the 'reseek' and just return 1. return 1; } - return delegate.reseekTo(key, offset, length); + return delegate.reseekTo(key); } - public org.apache.hadoop.hbase.io.hfile.HFile.Reader getReader() { - return this.delegate.getReader(); - } - - public boolean isSeeked() { - return this.delegate.isSeeked(); + @Override + public boolean seekBefore(Cell key) throws IOException { + if (top) { + Cell fk = new KeyValue.KeyOnlyKeyValue(getFirstKey(), 0, getFirstKey().length); + // This will be null when the file is empty in which we can not + // seekBefore to any key + if (fk == null) + return false; + if (getComparator().compareOnlyKeyPortion(key, fk) <= 0) { + return false; + } + } else { + // The equals sign isn't strictly necessary just here to be consistent + // with seekTo + if (getComparator().compareOnlyKeyPortion(key, splitCell) >= 0) { + return this.delegate.seekBefore(splitCell); + } + } + return this.delegate.seekBefore(key); } }; } @@ -316,7 +343,7 @@ public class HalfStoreFileReader extends StoreFile.Reader { // Returns null to indicate file is not splitable. return null; } - + @Override public byte[] getFirstKey() { if (!firstKeySeeked) { diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java index 9575406..516037f 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileBlockIndex.java @@ -36,9 +36,11 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.HFile.CachingBlockReader; @@ -165,19 +167,17 @@ public class HFileBlockIndex { * be called when the HFile version is larger than 1. * * @param key the key we are looking for - * @param keyOffset the offset of the key in its byte array - * @param keyLength the length of the key * @param currentBlock the current block, to avoid re-reading the same * block * @return reader a basic way to load blocks * @throws IOException */ - public HFileBlock seekToDataBlock(final byte[] key, int keyOffset, - int keyLength, HFileBlock currentBlock, boolean cacheBlocks, + public HFileBlock seekToDataBlock(final Cell key, HFileBlock currentBlock, boolean cacheBlocks, boolean pread, boolean isCompaction) throws IOException { - BlockWithScanInfo blockWithScanInfo = loadDataBlockWithScanInfo(key, keyOffset, keyLength, - currentBlock, cacheBlocks, pread, isCompaction); + BlockWithScanInfo blockWithScanInfo = loadDataBlockWithScanInfo(key, currentBlock, + cacheBlocks, + pread, isCompaction); if (blockWithScanInfo == null) { return null; } else { @@ -186,27 +186,26 @@ public class HFileBlockIndex { } /** - * Return the BlockWithScanInfo which contains the DataBlock with other scan info - * such as nextIndexedKey. - * This function will only be called when the HFile version is larger than 1. - * - * @param key the key we are looking for - * @param keyOffset the offset of the key in its byte array - * @param keyLength the length of the key - * @param currentBlock the current block, to avoid re-reading the same - * block + * Return the BlockWithScanInfo which contains the DataBlock with other scan + * info such as nextIndexedKey. This function will only be called when the + * HFile version is larger than 1. + * + * @param key + * the key we are looking for + * @param currentBlock + * the current block, to avoid re-reading the same block * @param cacheBlocks * @param pread * @param isCompaction - * @return the BlockWithScanInfo which contains the DataBlock with other scan info - * such as nextIndexedKey. + * @return the BlockWithScanInfo which contains the DataBlock with other + * scan info such as nextIndexedKey. * @throws IOException */ - public BlockWithScanInfo loadDataBlockWithScanInfo(final byte[] key, int keyOffset, - int keyLength, HFileBlock currentBlock, boolean cacheBlocks, + public BlockWithScanInfo loadDataBlockWithScanInfo(Cell key, HFileBlock currentBlock, + boolean cacheBlocks, boolean pread, boolean isCompaction) throws IOException { - int rootLevelIndex = rootBlockContainingKey(key, keyOffset, keyLength); + int rootLevelIndex = rootBlockContainingKey(key); if (rootLevelIndex < 0 || rootLevelIndex >= blockOffsets.length) { return null; } @@ -275,10 +274,13 @@ public class HFileBlockIndex { // Locate the entry corresponding to the given key in the non-root // (leaf or intermediate-level) index block. ByteBuffer buffer = block.getBufferWithoutHeader(); - index = locateNonRootIndexEntry(buffer, key, keyOffset, keyLength, comparator); + index = locateNonRootIndexEntry(buffer, key, comparator); if (index == -1) { + // This has to be changed + // For now change this to key value + KeyValue kv = KeyValueUtil.ensureKeyValue(key); throw new IOException("The key " - + Bytes.toStringBinary(key, keyOffset, keyLength) + + Bytes.toStringBinary(kv.getKey(), kv.getKeyOffset(), kv.getKeyLength()) + " is before the" + " first key of the non-root index block " + block); } @@ -387,10 +389,35 @@ public class HFileBlockIndex { * number of blocks - 1) or -1 if this file does not contain the * request. */ - public int rootBlockContainingKey(final byte[] key, int offset, - int length) { - int pos = Bytes.binarySearch(blockKeys, key, offset, length, - comparator); + public int rootBlockContainingKey(final byte[] key, int offset, int length) { + int pos = Bytes.binarySearch(blockKeys, key, offset, length, comparator); + // pos is between -(blockKeys.length + 1) to blockKeys.length - 1, see + // binarySearch's javadoc. + + if (pos >= 0) { + // This means this is an exact match with an element of blockKeys. + assert pos < blockKeys.length; + return pos; + } + + // Otherwise, pos = -(i + 1), where blockKeys[i - 1] < key < blockKeys[i], + // and i is in [0, blockKeys.length]. We are returning j = i - 1 such that + // blockKeys[j] <= key < blockKeys[j + 1]. In particular, j = -1 if + // key < blockKeys[0], meaning the file does not contain the given key. + + int i = -pos - 1; + assert 0 <= i && i <= blockKeys.length; + return i - 1; + } + + /** + * Finds the root-level index block containing the given key. + * + * @param key + * Key to find + */ + public int rootBlockContainingKey(final Cell key) { + int pos = Bytes.binarySearch(blockKeys, key, comparator); // pos is between -(blockKeys.length + 1) to blockKeys.length - 1, see // binarySearch's javadoc. @@ -463,20 +490,19 @@ public class HFileBlockIndex { * Performs a binary search over a non-root level index block. Utilizes the * secondary index, which records the offsets of (offset, onDiskSize, * firstKey) tuples of all entries. - * - * @param key the key we are searching for offsets to individual entries in + * + * @param key + * the key we are searching for offsets to individual entries in * the blockIndex buffer - * @param keyOffset the offset of the key in its byte array - * @param keyLength the length of the key - * @param nonRootIndex the non-root index block buffer, starting with the - * secondary index. The position is ignored. + * @param nonRootIndex + * the non-root index block buffer, starting with the secondary + * index. The position is ignored. * @return the index i in [0, numEntries - 1] such that keys[i] <= key < * keys[i + 1], if keys is the array of all keys being searched, or * -1 otherwise * @throws IOException */ - static int binarySearchNonRootIndex(byte[] key, int keyOffset, - int keyLength, ByteBuffer nonRootIndex, + static int binarySearchNonRootIndex(Cell key, ByteBuffer nonRootIndex, KVComparator comparator) { int numEntries = nonRootIndex.getInt(0); @@ -512,9 +538,9 @@ public class HFileBlockIndex { // we have to compare in this order, because the comparator order // has special logic when the 'left side' is a special key. - int cmp = comparator.compareFlatKey(key, keyOffset, keyLength, - nonRootIndex.array(), nonRootIndex.arrayOffset() + midKeyOffset, - midLength); + Cell nonRootIndexKV = new KeyValue.KeyOnlyKeyValue(nonRootIndex.array(), + nonRootIndex.arrayOffset() + midKeyOffset, midLength); + int cmp = comparator.compareOnlyKeyPortion(key, nonRootIndexKV); // key lives above the midpoint if (cmp > 0) @@ -553,20 +579,19 @@ public class HFileBlockIndex { * Search for one key using the secondary index in a non-root block. In case * of success, positions the provided buffer at the entry of interest, where * the file offset and the on-disk-size can be read. - * - * @param nonRootBlock a non-root block without header. Initial position - * does not matter. - * @param key the byte array containing the key - * @param keyOffset the offset of the key in its byte array - * @param keyLength the length of the key - * @return the index position where the given key was found, - * otherwise return -1 in the case the given key is before the first key. - * + * + * @param nonRootBlock + * a non-root block without header. Initial position does not + * matter. + * @param key + * the byte array containing the key + * @return the index position where the given key was found, otherwise + * return -1 in the case the given key is before the first key. + * */ - static int locateNonRootIndexEntry(ByteBuffer nonRootBlock, byte[] key, - int keyOffset, int keyLength, KVComparator comparator) { - int entryIndex = binarySearchNonRootIndex(key, keyOffset, keyLength, - nonRootBlock, comparator); + static int locateNonRootIndexEntry(ByteBuffer nonRootBlock, Cell key, + KVComparator comparator) { + int entryIndex = binarySearchNonRootIndex(key, nonRootBlock, comparator); if (entryIndex != -1) { int numEntries = nonRootBlock.getInt(0); @@ -576,8 +601,7 @@ public class HFileBlockIndex { // The offset of the entry we are interested in relative to the end of // the secondary index. - int entryRelOffset = nonRootBlock.getInt(Bytes.SIZEOF_INT - * (1 + entryIndex)); + int entryRelOffset = nonRootBlock.getInt(Bytes.SIZEOF_INT * (1 + entryIndex)); nonRootBlock.position(entriesOffset + entryRelOffset); } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java index acbeed2..8d0820a 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java @@ -28,9 +28,11 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.KeyValue.KVComparator; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoder; @@ -226,8 +228,8 @@ public class HFileReaderV2 extends AbstractHFileReader { } byte[] mbname = Bytes.toBytes(metaBlockName); - int block = metaBlockIndexReader.rootBlockContainingKey(mbname, 0, - mbname.length); + int block = metaBlockIndexReader.rootBlockContainingKey(mbname, + 0, mbname.length); if (block == -1) return null; long blockSize = metaBlockIndexReader.getRootBlockDataSize(block); @@ -471,91 +473,105 @@ public class HFileReaderV2 extends AbstractHFileReader { super(r, cacheBlocks, pread, isCompaction); } - /** - * An internal API function. Seek to the given key, optionally rewinding to - * the first key of the block before doing the seek. - * - * @param key key byte array - * @param offset key offset in the key byte array - * @param length key length - * @param rewind whether to rewind to the first key of the block before - * doing the seek. If this is false, we are assuming we never go - * back, otherwise the result is undefined. - * @return -1 if the key is earlier than the first key of the file, - * 0 if we are at the given key, 1 if we are past the given key - * -2 if the key is earlier than the first key of the file while - * using a faked index key - * @throws IOException - */ - protected int seekTo(byte[] key, int offset, int length, boolean rewind) - throws IOException { - HFileBlockIndex.BlockIndexReader indexReader = - reader.getDataBlockIndexReader(); - BlockWithScanInfo blockWithScanInfo = - indexReader.loadDataBlockWithScanInfo(key, offset, length, block, - cacheBlocks, pread, isCompaction); - if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) { - // This happens if the key e.g. falls before the beginning of the file. - return -1; - } - return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(), - blockWithScanInfo.getNextIndexedKey(), rewind, key, offset, length, false); - } - protected abstract ByteBuffer getFirstKeyInBlock(HFileBlock curBlock); protected abstract int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey, - boolean rewind, byte[] key, int offset, int length, boolean seekBefore) - throws IOException; + boolean rewind, Cell key, boolean seekBefore) throws IOException; @Override public int seekTo(byte[] key, int offset, int length) throws IOException { // Always rewind to the first key of the block, because the given key // might be before or after the current key. - return seekTo(key, offset, length, true); + return seekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length)); } @Override public int reseekTo(byte[] key, int offset, int length) throws IOException { + return reseekTo(new KeyValue.KeyOnlyKeyValue(key, offset, length)); + } + + @Override + public int seekTo(Cell key) throws IOException { + return seekTo(key, true); + } + + @Override + public int reseekTo(Cell key) throws IOException { int compared; if (isSeeked()) { - compared = compareKey(reader.getComparator(), key, offset, length); + compared = compareKey(reader.getComparator(), key); if (compared < 1) { // If the required key is less than or equal to current key, then // don't do anything. return compared; } else { + // The comparison with no_next_index_key has to be checked if (this.nextIndexedKey != null && - (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || - reader.getComparator().compareFlatKey(key, offset, length, - nextIndexedKey, 0, nextIndexedKey.length) < 0)) { - // The reader shall continue to scan the current data block instead of querying the - // block index as long as it knows the target key is strictly smaller than - // the next indexed key or the current data block is the last data block. - return loadBlockAndSeekToKey(this.block, this.nextIndexedKey, - false, key, offset, length, false); + (this.nextIndexedKey == HConstants.NO_NEXT_INDEXED_KEY || reader + .getComparator() + .compareOnlyKeyPortion(key, + new KeyValue.KeyOnlyKeyValue(nextIndexedKey, 0, + nextIndexedKey.length)) < 0)) { + // The reader shall continue to scan the current data block instead + // of querying the + // block index as long as it knows the target key is strictly + // smaller than + // the next indexed key or the current data block is the last data + // block. + return loadBlockAndSeekToKey(this.block, nextIndexedKey, false, key, false); } } } // Don't rewind on a reseek operation, because reseek implies that we are // always going forward in the file. - return seekTo(key, offset, length, false); + return seekTo(key, false); + } + + + /** + * An internal API function. Seek to the given key, optionally rewinding to + * the first key of the block before doing the seek. + * + * @param key - a cell representing the key that we need to fetch + * @param rewind whether to rewind to the first key of the block before + * doing the seek. If this is false, we are assuming we never go + * back, otherwise the result is undefined. + * @return -1 if the key is earlier than the first key of the file, + * 0 if we are at the given key, 1 if we are past the given key + * -2 if the key is earlier than the first key of the file while + * using a faked index key + * @throws IOException + */ + public int seekTo(Cell key, boolean rewind) throws IOException { + HFileBlockIndex.BlockIndexReader indexReader = reader.getDataBlockIndexReader(); + BlockWithScanInfo blockWithScanInfo = indexReader.loadDataBlockWithScanInfo(key, block, + cacheBlocks, pread, isCompaction); + if (blockWithScanInfo == null || blockWithScanInfo.getHFileBlock() == null) { + // This happens if the key e.g. falls before the beginning of the file. + return -1; + } + return loadBlockAndSeekToKey(blockWithScanInfo.getHFileBlock(), + blockWithScanInfo.getNextIndexedKey(), rewind, key, false); + } + + @Override + public boolean seekBefore(byte[] key, int offset, int length) throws IOException { + return seekBefore(new KeyValue.KeyOnlyKeyValue(key, offset, length)); } @Override - public boolean seekBefore(byte[] key, int offset, int length) - throws IOException { - HFileBlock seekToBlock = - reader.getDataBlockIndexReader().seekToDataBlock(key, offset, length, - block, cacheBlocks, pread, isCompaction); + public boolean seekBefore(Cell key) throws IOException { + HFileBlock seekToBlock = reader.getDataBlockIndexReader().seekToDataBlock(key, block, + cacheBlocks, pread, isCompaction); if (seekToBlock == null) { return false; } ByteBuffer firstKey = getFirstKeyInBlock(seekToBlock); - if (reader.getComparator().compareFlatKey(firstKey.array(), - firstKey.arrayOffset(), firstKey.limit(), key, offset, length) >= 0) - { + if (reader.getComparator() + .compareOnlyKeyPortion( + new KeyValue.KeyOnlyKeyValue(firstKey.array(), firstKey.arrayOffset(), + firstKey.limit()), key) >= 0) { long previousBlockOffset = seekToBlock.getPrevBlockOffset(); // The key we are interested in if (previousBlockOffset == -1) { @@ -566,18 +582,16 @@ public class HFileReaderV2 extends AbstractHFileReader { // It is important that we compute and pass onDiskSize to the block // reader so that it does not have to read the header separately to // figure out the size. - seekToBlock = reader.readBlock(previousBlockOffset, - seekToBlock.getOffset() - previousBlockOffset, cacheBlocks, - pread, isCompaction, BlockType.DATA); + seekToBlock = reader.readBlock(previousBlockOffset, seekToBlock.getOffset() + - previousBlockOffset, cacheBlocks, pread, isCompaction, BlockType.DATA); // TODO shortcut: seek forward in this block to the last key of the // block. } byte[] firstKeyInCurrentBlock = Bytes.getBytes(firstKey); - loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, offset, length, true); + loadBlockAndSeekToKey(seekToBlock, firstKeyInCurrentBlock, true, key, true); return true; } - /** * Scans blocks in the "scanned" section of the {@link HFile} until the next * data block is found. @@ -620,6 +634,8 @@ public class HFileReaderV2 extends AbstractHFileReader { */ public abstract int compareKey(KVComparator comparator, byte[] key, int offset, int length); + + public abstract int compareKey(KVComparator comparator, Cell kv); } /** @@ -773,8 +789,7 @@ public class HFileReaderV2 extends AbstractHFileReader { @Override protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey, - boolean rewind, byte[] key, int offset, int length, boolean seekBefore) - throws IOException { + boolean rewind, Cell key, boolean seekBefore) throws IOException { if (block == null || block.getOffset() != seekToBlock.getOffset()) { updateCurrBlock(seekToBlock); } else if (rewind) { @@ -783,7 +798,7 @@ public class HFileReaderV2 extends AbstractHFileReader { // Update the nextIndexedKey this.nextIndexedKey = nextIndexedKey; - return blockSeek(key, offset, length, seekBefore); + return blockSeek(key, seekBefore); } /** @@ -847,22 +862,23 @@ public class HFileReaderV2 extends AbstractHFileReader { } /** - * Within a loaded block, seek looking for the last key that is smaller - * than (or equal to?) the key we are interested in. - * + * Within a loaded block, seek looking for the last key that is smaller than + * (or equal to?) the key we are interested in. + * * A note on the seekBefore: if you have seekBefore = true, AND the first * key in the block = key, then you'll get thrown exceptions. The caller has * to check for that case and load the previous block as appropriate. - * - * @param key the key to find - * @param seekBefore find the key before the given key in case of exact - * match. + * + * @param key + * the key to find + * @param seekBefore + * find the key before the given key in case of exact match. * @return 0 in case of an exact key match, 1 in case of an inexact match, - * -2 in case of an inexact match and furthermore, the input key less - * than the first key of current block(e.g. using a faked index key) + * -2 in case of an inexact match and furthermore, the input key + * less than the first key of current block(e.g. using a faked index + * key) */ - protected int blockSeek(byte[] key, int offset, int length, - boolean seekBefore) { + protected int blockSeek(Cell key, boolean seekBefore) { int klen, vlen; long memstoreTS = 0; int memstoreTSLen = 0; @@ -875,10 +891,9 @@ public class HFileReaderV2 extends AbstractHFileReader { if (this.reader.shouldIncludeMemstoreTS()) { if (this.reader.decodeMemstoreTS) { try { - int memstoreTSOffset = blockBuffer.arrayOffset() - + blockBuffer.position() + KEY_VALUE_LEN_SIZE + klen + vlen; - memstoreTS = Bytes.readVLong(blockBuffer.array(), - memstoreTSOffset); + int memstoreTSOffset = blockBuffer.arrayOffset() + blockBuffer.position() + + KEY_VALUE_LEN_SIZE + klen + vlen; + memstoreTS = Bytes.readVLong(blockBuffer.array(), memstoreTSOffset); memstoreTSLen = WritableUtils.getVIntSize(memstoreTS); } catch (Exception e) { throw new RuntimeException("Error reading memstore timestamp", e); @@ -889,18 +904,18 @@ public class HFileReaderV2 extends AbstractHFileReader { } } - int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() - + KEY_VALUE_LEN_SIZE; - int comp = reader.getComparator().compareFlatKey(key, offset, length, - blockBuffer.array(), keyOffset, klen); + int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + KEY_VALUE_LEN_SIZE; + int comp = reader.getComparator().compareOnlyKeyPortion(key, + new KeyValue.KeyOnlyKeyValue(blockBuffer.array(), keyOffset, klen)); if (comp == 0) { if (seekBefore) { if (lastKeyValueSize < 0) { + KeyValue kv = KeyValueUtil.ensureKeyValue(key); throw new IllegalStateException("blockSeek with seekBefore " + "at the first key of the block: key=" - + Bytes.toStringBinary(key) + ", blockOffset=" - + block.getOffset() + ", onDiskSize=" + + Bytes.toStringBinary(kv.getKey(), kv.getKeyOffset(), kv.getKeyLength()) + + ", blockOffset=" + block.getOffset() + ", onDiskSize=" + block.getOnDiskSizeWithHeader()); } blockBuffer.position(blockBuffer.position() - lastKeyValueSize); @@ -964,6 +979,14 @@ public class HFileReaderV2 extends AbstractHFileReader { + blockBuffer.position() + KEY_VALUE_LEN_SIZE + currKeyLen, currValueLen); } + + @Override + public int compareKey(KVComparator comparator, Cell key) { + return comparator.compareOnlyKeyPortion( + key, + new KeyValue.KeyOnlyKeyValue(blockBuffer.array(), blockBuffer.arrayOffset() + + blockBuffer.position() + KEY_VALUE_LEN_SIZE, currKeyLen)); + } } /** @@ -1123,15 +1146,19 @@ public class HFileReaderV2 extends AbstractHFileReader { @Override protected int loadBlockAndSeekToKey(HFileBlock seekToBlock, byte[] nextIndexedKey, - boolean rewind, byte[] key, int offset, int length, boolean seekBefore) - throws IOException { + boolean rewind, Cell key, boolean seekBefore) throws IOException { if (block == null || block.getOffset() != seekToBlock.getOffset()) { updateCurrentBlock(seekToBlock); } else if (rewind) { seeker.rewind(); } this.nextIndexedKey = nextIndexedKey; - return seeker.seekToKeyInBlock(key, offset, length, seekBefore); + return seeker.seekToKeyInBlock(key, seekBefore); + } + + @Override + public int compareKey(KVComparator comparator, Cell key) { + return seeker.compareKey(comparator, key); } } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java index 4ed56c6..ffc14da 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV3.java @@ -26,7 +26,10 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.fs.HFileSystem; import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; import org.apache.hadoop.hbase.io.crypto.Cipher; @@ -235,16 +238,13 @@ public class HFileReaderV3 extends HFileReaderV2 { * the key to find * @param seekBefore * find the key before the given key in case of exact match. - * @param offset - * Offset to find the key in the given bytebuffer - * @param length - * Length of the key to be found * @return 0 in case of an exact key match, 1 in case of an inexact match, * -2 in case of an inexact match and furthermore, the input key * less than the first key of current block(e.g. using a faked index * key) */ - protected int blockSeek(byte[] key, int offset, int length, boolean seekBefore) { + @Override + protected int blockSeek(Cell key, boolean seekBefore) { int klen, vlen, tlen = 0; long memstoreTS = 0; int memstoreTSLen = 0; @@ -286,14 +286,16 @@ public class HFileReaderV3 extends HFileReaderV2 { } blockBuffer.reset(); int keyOffset = blockBuffer.arrayOffset() + blockBuffer.position() + (Bytes.SIZEOF_INT * 2); - int comp = reader.getComparator().compare(key, offset, length, blockBuffer.array(), - keyOffset, klen); + int comp = reader.getComparator().compareOnlyKeyPortion(key, + new KeyValue.KeyOnlyKeyValue(blockBuffer.array(), keyOffset, klen)); if (comp == 0) { if (seekBefore) { if (lastKeyValueSize < 0) { + KeyValue kv = KeyValueUtil.ensureKeyValue(key); throw new IllegalStateException("blockSeek with seekBefore " - + "at the first key of the block: key=" + Bytes.toStringBinary(key) + + "at the first key of the block: key=" + + Bytes.toStringBinary(kv.getKey(), kv.getKeyOffset(), kv.getKeyLength()) + ", blockOffset=" + block.getOffset() + ", onDiskSize=" + block.getOnDiskSizeWithHeader()); } @@ -335,7 +337,7 @@ public class HFileReaderV3 extends HFileReaderV2 { readKeyValueLen(); return 1; // didn't exactly find it. } - + } /** diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java index 0e353ef..840a78c 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileScanner.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.KeyValue; /** @@ -54,8 +55,12 @@ public interface HFileScanner { * false when it is called. * @throws IOException */ + @Deprecated int seekTo(byte[] key) throws IOException; + @Deprecated int seekTo(byte[] key, int offset, int length) throws IOException; + + int seekTo(Cell kv) throws IOException; /** * Reseek to or just before the passed key. Similar to seekTo * except that this can be called even if the scanner is not at the beginning @@ -76,8 +81,12 @@ public interface HFileScanner { * 1, such that k[i] < key, and scanner is left in position i. * @throws IOException */ + @Deprecated int reseekTo(byte[] key) throws IOException; + @Deprecated int reseekTo(byte[] key, int offset, int length) throws IOException; + + int reseekTo(Cell kv) throws IOException; /** * Consider the key stream of all the keys in the file, * k[0] .. k[n], where there are n keys in the file. @@ -88,8 +97,12 @@ public interface HFileScanner { * return false (EOF). * @throws IOException */ + @Deprecated boolean seekBefore(byte[] key) throws IOException; + @Deprecated boolean seekBefore(byte[] key, int offset, int length) throws IOException; + + boolean seekBefore(Cell kv) throws IOException; /** * Positions this scanner at the start of the file. * @return False if empty file; i.e. a call to next would return false and diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 4ef3351..67f66a7 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -1643,8 +1643,7 @@ public class HStore implements Store { KeyValue kv = firstOnRow; // If firstOnRow < firstKV, set to firstKV if (this.comparator.compareRows(firstKV, firstOnRow) == 0) kv = firstKV; - int result = scanner.seekTo(kv.getBuffer(), kv.getKeyOffset(), - kv.getKeyLength()); + int result = scanner.seekTo(kv); return result != -1; } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java index ecd1793..0440102 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java @@ -235,7 +235,7 @@ public class StoreFileScanner implements KeyValueScanner { */ public static boolean seekAtOrAfter(HFileScanner s, KeyValue k) throws IOException { - int result = s.seekTo(k.getBuffer(), k.getKeyOffset(), k.getKeyLength()); + int result = s.seekTo(k); if(result < 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key @@ -255,7 +255,7 @@ public class StoreFileScanner implements KeyValueScanner { static boolean reseekAtOrAfter(HFileScanner s, KeyValue k) throws IOException { //This function is similar to seekAtOrAfter function - int result = s.reseekTo(k.getBuffer(), k.getKeyOffset(), k.getKeyLength()); + int result = s.reseekTo(k); if (result <= 0) { if (result == HConstants.INDEX_KEY_MAGIC) { // using faked key diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java index 92eda42..e8d80ae 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/BloomFilterFactory.java @@ -29,8 +29,8 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; -import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.BloomType; +import org.apache.hadoop.hbase.regionserver.StoreFile; /** * Handles Bloom filter initialization based on configuration and serialized diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java index edc9168..9987e24 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/ByteBloomFilter.java @@ -19,12 +19,6 @@ package org.apache.hadoop.hbase.util; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.KeyValue; -import org.apache.hadoop.hbase.KeyValue.KVComparator; -import org.apache.hadoop.io.RawComparator; -import org.apache.hadoop.io.Writable; - import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; @@ -32,6 +26,11 @@ import java.nio.ByteBuffer; import java.text.NumberFormat; import java.util.Random; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.KVComparator; +import org.apache.hadoop.io.Writable; + /** * Implements a Bloom filter, as defined by Bloom in 1970. *

diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java index 911738a..e9cc31f 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/util/CompoundBloomFilter.java @@ -89,7 +89,8 @@ public class CompoundBloomFilter extends CompoundBloomFilterBase // testing, but when an error happens, we log a message and return. boolean result; - int block = index.rootBlockContainingKey(key, keyOffset, keyLength); + int block = index.rootBlockContainingKey(key, keyOffset, + keyLength); if (block < 0) { result = false; // This key is not in the file. } else { diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java index 1e0b0e8..29ce3d2 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHalfStoreFileReader.java @@ -33,7 +33,6 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.SmallTests; -import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; import org.apache.hadoop.hbase.io.hfile.HFileContext; @@ -127,12 +126,12 @@ public class TestHalfStoreFileReader { curr = scanner.getKeyValue(); KeyValue reseekKv = getLastOnCol(curr); - int ret = scanner.reseekTo(reseekKv.getKey()); + int ret = scanner.reseekTo(reseekKv); assertTrue("reseek to returned: " + ret, ret > 0); //System.out.println(curr + ": " + ret); } while (scanner.next()); - int ret = scanner.reseekTo(getLastOnCol(curr).getKey()); + int ret = scanner.reseekTo(getLastOnCol(curr)); //System.out.println("Last reseek: " + ret); assertTrue( ret > 0 ); @@ -221,7 +220,7 @@ public class TestHalfStoreFileReader { cacheConfig, bottom, TEST_UTIL.getConfiguration()); halfreader.loadFileInfo(); final HFileScanner scanner = halfreader.getScanner(false, false); - scanner.seekBefore(seekBefore.getKey()); + scanner.seekBefore(seekBefore); return scanner.getKeyValue(); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java index 17f62ee..636c2c7 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestDataBlockEncoders.java @@ -360,8 +360,9 @@ public class TestDataBlockEncoders { ByteBuffer expectedValue = null; for (DataBlockEncoder.EncodedSeeker seeker : encodedSeekers) { - seeker.seekToKeyInBlock(keyValue.getBuffer(), keyValue.getKeyOffset(), - keyValue.getKeyLength(), seekBefore); + seeker.seekToKeyInBlock( + new KeyValue.KeyOnlyKeyValue(keyValue.getBuffer(), keyValue.getOffset(), keyValue + .getKeyLength()), seekBefore); seeker.rewind(); ByteBuffer actualKeyValue = seeker.getKeyValueBuffer(); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java index 9ca4caf..f01dc50 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/encoding/TestPrefixTreeEncoding.java @@ -114,22 +114,25 @@ public class TestPrefixTreeEncoding { // Seek before the first keyvalue; KeyValue seekKey = KeyValue.createFirstDeleteFamilyOnRow(getRowKey(batchId, 0), CF_BYTES); - seeker.seekToKeyInBlock(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey.getKeyLength(), - true); + seeker.seekToKeyInBlock( + new KeyValue.KeyOnlyKeyValue(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey + .getKeyLength()), true); assertEquals(null, seeker.getKeyValue()); // Seek before the middle keyvalue; seekKey = KeyValue.createFirstDeleteFamilyOnRow(getRowKey(batchId, NUM_ROWS_PER_BATCH / 3), CF_BYTES); - seeker.seekToKeyInBlock(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey.getKeyLength(), - true); + seeker.seekToKeyInBlock( + new KeyValue.KeyOnlyKeyValue(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey + .getKeyLength()), true); assertNotNull(seeker.getKeyValue()); assertArrayEquals(getRowKey(batchId, NUM_ROWS_PER_BATCH / 3 - 1), seeker.getKeyValue().getRow()); // Seek before the last keyvalue; seekKey = KeyValue.createFirstDeleteFamilyOnRow(Bytes.toBytes("zzzz"), CF_BYTES); - seeker.seekToKeyInBlock(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey.getKeyLength(), - true); + seeker.seekToKeyInBlock( + new KeyValue.KeyOnlyKeyValue(seekKey.getBuffer(), seekKey.getKeyOffset(), seekKey + .getKeyLength()), true); assertNotNull(seeker.getKeyValue()); assertArrayEquals(getRowKey(batchId, NUM_ROWS_PER_BATCH - 1), seeker.getKeyValue().getRow()); } @@ -221,8 +224,9 @@ public class TestPrefixTreeEncoding { kvList.clear(); encodeSeeker.setCurrentBuffer(encodedData); KeyValue firstOnRow = KeyValue.createFirstOnRow(getRowKey(batchId, i)); - encodeSeeker.seekToKeyInBlock(firstOnRow.getBuffer(), - firstOnRow.getKeyOffset(), firstOnRow.getKeyLength(), false); + encodeSeeker.seekToKeyInBlock( + new KeyValue.KeyOnlyKeyValue(firstOnRow.getBuffer(), firstOnRow.getKeyOffset(), + firstOnRow.getKeyLength()), false); boolean hasMoreOfEncodeScanner = encodeSeeker.next(); CollectionBackedScanner collectionScanner = new CollectionBackedScanner( this.kvset); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java index 20463f7..ef9a74f 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFile.java @@ -34,7 +34,9 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestCase; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.KeyValue; +import org.apache.hadoop.hbase.KeyValue.Type; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.Tag; import org.apache.hadoop.hbase.io.compress.Compression; @@ -155,13 +157,20 @@ public class TestHFile extends HBaseTestCase { private int writeSomeRecords(Writer writer, int start, int n, boolean useTags) throws IOException { String value = "value"; + KeyValue kv; for (int i = start; i < (start + n); i++) { String key = String.format(localFormatter, Integer.valueOf(i)); if (useTags) { Tag t = new Tag((byte) 1, "myTag1"); - writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key), t.getBuffer()); + Tag[] tags = new Tag[1]; + tags[0] = t; + kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), + HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value + key), tags); + writer.append(kv); } else { - writer.append(Bytes.toBytes(key), Bytes.toBytes(value + key)); + kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), + Bytes.toBytes(value + key)); + writer.append(kv); } } return (start + n); @@ -181,10 +190,13 @@ public class TestHFile extends HBaseTestCase { ByteBuffer val = scanner.getValue(); String keyStr = String.format(localFormatter, Integer.valueOf(i)); String valStr = value + keyStr; - byte [] keyBytes = Bytes.toBytes(key); + KeyValue kv = new KeyValue(Bytes.toBytes(keyStr), Bytes.toBytes("family"), + Bytes.toBytes("qual"), Bytes.toBytes(valStr)); + byte[] keyBytes = new KeyValue.KeyOnlyKeyValue(Bytes.toBytes(key), 0, + Bytes.toBytes(key).length).getKey(); assertTrue("bytes for keys do not match " + keyStr + " " + Bytes.toString(Bytes.toBytes(key)), - Arrays.equals(Bytes.toBytes(keyStr), keyBytes)); + Arrays.equals(kv.getKey(), keyBytes)); byte [] valBytes = Bytes.toBytes(val); assertTrue("bytes for vals do not match " + valStr + " " + Bytes.toString(valBytes), @@ -198,7 +210,9 @@ public class TestHFile extends HBaseTestCase { } private byte[] getSomeKey(int rowId) { - return String.format(localFormatter, Integer.valueOf(rowId)).getBytes(); + KeyValue kv = new KeyValue(String.format(localFormatter, Integer.valueOf(rowId)).getBytes(), + Bytes.toBytes("family"), Bytes.toBytes("qual"), HConstants.LATEST_TIMESTAMP, Type.Put); + return kv.getKey(); } private void writeRecords(Writer writer, boolean useTags) throws IOException { @@ -230,8 +244,7 @@ public class TestHFile extends HBaseTestCase { Writer writer = HFile.getWriterFactory(conf, cacheConf) .withOutputStream(fout) .withFileContext(meta) - // NOTE: This test is dependent on this deprecated nonstandard comparator - .withComparator(new KeyValue.RawBytesComparator()) + .withComparator(new KeyValue.KVComparator()) .create(); LOG.info(writer); writeRecords(writer, useTags); @@ -247,16 +260,18 @@ public class TestHFile extends HBaseTestCase { // Align scanner at start of the file. scanner.seekTo(); readAllRecords(scanner); - scanner.seekTo(getSomeKey(50)); - assertTrue("location lookup failed", scanner.seekTo(getSomeKey(50)) == 0); + int seekTo = scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(50))); + System.out.println(seekTo); + assertTrue("location lookup failed", + scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(50))) == 0); // read the key and see if it matches ByteBuffer readKey = scanner.getKey(); assertTrue("seeked key does not match", Arrays.equals(getSomeKey(50), Bytes.toBytes(readKey))); - scanner.seekTo(new byte[0]); + scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(0))); ByteBuffer val1 = scanner.getValue(); - scanner.seekTo(new byte[0]); + scanner.seekTo(KeyValue.createKeyValueFromKey(getSomeKey(0))); ByteBuffer val2 = scanner.getValue(); assertTrue(Arrays.equals(Bytes.toBytes(val1), Bytes.toBytes(val2))); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java index 7b77c15..6e2bffa 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java @@ -117,7 +117,7 @@ public class TestHFileBlockIndex { fs = HFileSystem.get(conf); } - @Test + //@Test public void testBlockIndex() throws IOException { testBlockIndexInternals(false); clear(); @@ -213,7 +213,8 @@ public class TestHFileBlockIndex { for (byte[] key : keys) { assertTrue(key != null); assertTrue(indexReader != null); - HFileBlock b = indexReader.seekToDataBlock(key, 0, key.length, null, + HFileBlock b = indexReader.seekToDataBlock(new KeyValue.KeyOnlyKeyValue(key, 0, key.length), + null, true, true, false); if (Bytes.BYTES_RAWCOMPARATOR.compare(key, firstKeyInFile) < 0) { assertTrue(b == null); @@ -330,7 +331,10 @@ public class TestHFileBlockIndex { for (int i = 0; i < numTotalKeys; ++i) { byte[] k = TestHFileWriterV2.randomOrderedKey(rand, i * 2); - keys.add(k); + KeyValue cell = new KeyValue(k, Bytes.toBytes("f"), Bytes.toBytes("q"), + Bytes.toBytes("val")); + //KeyValue cell = new KeyValue.KeyOnlyKeyValue(k, 0, k.length); + keys.add(cell.getKey()); String msgPrefix = "Key #" + i + " (" + Bytes.toStringBinary(k) + "): "; StringBuilder padding = new StringBuilder(); while (msgPrefix.length() + padding.length() < 70) @@ -341,7 +345,7 @@ public class TestHFileBlockIndex { secondaryIndexEntries[i] = curAllEntriesSize; LOG.info(msgPrefix + "secondary index entry #" + ((i - 1) / 2) + ", offset " + curAllEntriesSize); - curAllEntriesSize += k.length + curAllEntriesSize += cell.getKey().length + HFileBlockIndex.SECONDARY_INDEX_ENTRY_OVERHEAD; ++numEntriesAdded; } else { @@ -352,8 +356,9 @@ public class TestHFileBlockIndex { // Make sure the keys are increasing. for (int i = 0; i < keys.size() - 1; ++i) - assertTrue(Bytes.BYTES_RAWCOMPARATOR.compare(keys.get(i), - keys.get(i + 1)) < 0); + assertTrue(KeyValue.COMPARATOR.compare( + new KeyValue.KeyOnlyKeyValue(keys.get(i), 0, keys.get(i).length), + new KeyValue.KeyOnlyKeyValue(keys.get(i + 1), 0, keys.get(i + 1).length)) < 0); dos.writeInt(curAllEntriesSize); assertEquals(numSearchedKeys, numEntriesAdded); @@ -387,9 +392,10 @@ public class TestHFileBlockIndex { System.arraycopy(searchKey, 0, arrayHoldingKey, searchKey.length / 2, searchKey.length); - int searchResult = BlockIndexReader.binarySearchNonRootIndex( - arrayHoldingKey, searchKey.length / 2, searchKey.length, nonRootIndex, - KeyValue.RAW_COMPARATOR); + KeyValue.KeyOnlyKeyValue cell = new KeyValue.KeyOnlyKeyValue( + arrayHoldingKey, searchKey.length / 2, searchKey.length); + int searchResult = BlockIndexReader.binarySearchNonRootIndex(cell, + nonRootIndex, KeyValue.COMPARATOR); String lookupFailureMsg = "Failed to look up key #" + i + " (" + Bytes.toStringBinary(searchKey) + ")"; @@ -414,8 +420,8 @@ public class TestHFileBlockIndex { // Now test we can get the offset and the on-disk-size using a // higher-level API function.s boolean locateBlockResult = - (BlockIndexReader.locateNonRootIndexEntry(nonRootIndex, arrayHoldingKey, - searchKey.length / 2, searchKey.length, KeyValue.RAW_COMPARATOR) != -1); + (BlockIndexReader.locateNonRootIndexEntry(nonRootIndex, cell, + KeyValue.COMPARATOR) != -1); if (i == 0) { assertFalse(locateBlockResult); @@ -431,7 +437,7 @@ public class TestHFileBlockIndex { } - @Test + //@Test public void testBlockIndexChunk() throws IOException { BlockIndexChunk c = new BlockIndexChunk(); ByteArrayOutputStream baos = new ByteArrayOutputStream(); @@ -468,7 +474,7 @@ public class TestHFileBlockIndex { } /** Checks if the HeapSize calculator is within reason */ - @Test + //@Test public void testHeapSizeForBlockIndex() throws IOException { Class cl = HFileBlockIndex.BlockIndexReader.class; @@ -496,7 +502,7 @@ public class TestHFileBlockIndex { * * @throws IOException */ - @Test + //@Test public void testHFileWriterAndReader() throws IOException { Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "hfile_for_block_index"); @@ -625,8 +631,8 @@ public class TestHFileBlockIndex { private void checkSeekTo(byte[][] keys, HFileScanner scanner, int i) throws IOException { - assertEquals("Failed to seek to key #" + i + " (" - + Bytes.toStringBinary(keys[i]) + ")", 0, scanner.seekTo(keys[i])); + assertEquals("Failed to seek to key #" + i + " (" + Bytes.toStringBinary(keys[i]) + ")", 0, + scanner.seekTo(KeyValue.createKeyValueFromKey(keys[i]))); } private void assertArrayEqualsBuffer(String msgPrefix, byte[] arr, diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java index 3556b79..51868a1 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileEncryption.java @@ -17,7 +17,10 @@ */ package org.apache.hadoop.hbase.io.hfile; -import static org.junit.Assert.*; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import java.io.DataInputStream; import java.io.DataOutputStream; @@ -44,7 +47,6 @@ import org.apache.hadoop.hbase.io.crypto.KeyProviderForTesting; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.test.RedundantKVGenerator; - import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -177,7 +179,7 @@ public class TestHFileEncryption { cryptoContext.getKeyBytes())); } - @Test(timeout=60000) + @Test(timeout=6000000) public void testHFileEncryption() throws Exception { // Create 1000 random test KVs RedundantKVGenerator generator = new RedundantKVGenerator(); @@ -233,7 +235,7 @@ public class TestHFileEncryption { assertTrue("Initial seekTo failed", scanner.seekTo()); for (i = 0; i < 100; i++) { KeyValue kv = testKvs.get(RNG.nextInt(testKvs.size())); - assertEquals("Unable to find KV as expected: " + kv, scanner.seekTo(kv.getKey()), 0); + assertEquals("Unable to find KV as expected: " + kv, scanner.seekTo(kv), 0); } reader.close(); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java index fb9f183..23334ac 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileInlineToRootChunkConversion.java @@ -23,6 +23,7 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.KeyValue; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.util.Bytes; import org.junit.Test; @@ -80,7 +81,7 @@ public class TestHFileInlineToRootChunkConversion { HFileReaderV2 reader = (HFileReaderV2) HFile.createReader(fs, hfPath, cacheConf, conf); HFileScanner scanner = reader.getScanner(true, true); for (int i = 0; i < keys.size(); ++i) { - scanner.seekTo(keys.get(i)); + scanner.seekTo(KeyValue.createKeyValueFromKey(keys.get(i))); } reader.close(); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java index f42cfc3..7310525 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileSeek.java @@ -193,7 +193,7 @@ public class TestHFileSeek extends TestCase { kSampler.next(key); byte [] k = new byte [key.getLength()]; System.arraycopy(key.getBytes(), 0, k, 0, key.getLength()); - if (scanner.seekTo(k) >= 0) { + if (scanner.seekTo(KeyValue.createKeyValueFromKey(k)) >= 0) { ByteBuffer bbkey = scanner.getKey(); ByteBuffer bbval = scanner.getValue(); totalBytes += bbkey.limit(); diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java index 504b37c..596387a 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestReseekTo.java @@ -63,7 +63,7 @@ public class TestReseekTo { .withOutputStream(fout) .withFileContext(context) // NOTE: This test is dependent on this deprecated nonstandard comparator - .withComparator(new KeyValue.RawBytesComparator()) + .withComparator(KeyValue.COMPARATOR) .create(); int numberOfKeys = 1000; @@ -74,19 +74,32 @@ public class TestReseekTo { for (int key = 0; key < numberOfKeys; key++) { String value = valueString + key; + KeyValue kv; keyList.add(key); valueList.add(value); if(tagUsage == TagUsage.NO_TAG){ - writer.append(Bytes.toBytes(key), Bytes.toBytes(value)); + kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), + Bytes.toBytes(value)); + writer.append(kv); } else if (tagUsage == TagUsage.ONLY_TAG) { Tag t = new Tag((byte) 1, "myTag1"); - writer.append(Bytes.toBytes(key), Bytes.toBytes(value), t.getBuffer()); + Tag[] tags = new Tag[1]; + tags[0] = t; + kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), + HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value), tags); + writer.append(kv); } else { if (key % 4 == 0) { Tag t = new Tag((byte) 1, "myTag1"); - writer.append(Bytes.toBytes(key), Bytes.toBytes(value), t.getBuffer()); + Tag[] tags = new Tag[1]; + tags[0] = t; + kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), + HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value), tags); + writer.append(kv); } else { - writer.append(Bytes.toBytes(key), Bytes.toBytes(value), HConstants.EMPTY_BYTE_ARRAY); + kv = new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes.toBytes("qual"), + HConstants.LATEST_TIMESTAMP, Bytes.toBytes(value)); + writer.append(kv); } } } @@ -103,7 +116,8 @@ public class TestReseekTo { Integer key = keyList.get(i); String value = valueList.get(i); long start = System.nanoTime(); - scanner.seekTo(Bytes.toBytes(key)); + scanner.seekTo(new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes + .toBytes("qual"), Bytes.toBytes(value))); assertEquals(value, scanner.getValueString()); } @@ -112,7 +126,8 @@ public class TestReseekTo { Integer key = keyList.get(i); String value = valueList.get(i); long start = System.nanoTime(); - scanner.reseekTo(Bytes.toBytes(key)); + scanner.reseekTo(new KeyValue(Bytes.toBytes(key), Bytes.toBytes("family"), Bytes + .toBytes("qual"), Bytes.toBytes(value))); assertEquals("i is " + i, value, scanner.getValueString()); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java index 71014cc..0247b71 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestSeekTo.java @@ -80,9 +80,7 @@ public class TestSeekTo extends HBaseTestCase { .withIncludesTags(true).build(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout) .withFileContext(context) - // NOTE: This test is dependent on this deprecated nonstandard - // comparator - .withComparator(KeyValue.RAW_COMPARATOR).create(); + .withComparator(KeyValue.COMPARATOR).create(); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) writer.append(toKV("c", tagUsage)); @@ -107,31 +105,31 @@ public class TestSeekTo extends HBaseTestCase { HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); - assertEquals(false, scanner.seekBefore(toKV("a", tagUsage).getKey())); + assertEquals(false, scanner.seekBefore(toKV("a", tagUsage))); - assertEquals(false, scanner.seekBefore(toKV("c", tagUsage).getKey())); + assertEquals(false, scanner.seekBefore(toKV("c", tagUsage))); - assertEquals(true, scanner.seekBefore(toKV("d", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("d", tagUsage))); assertEquals("c", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("e", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("e", tagUsage))); assertEquals("c", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("f", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("f", tagUsage))); assertEquals("e", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("g", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("g", tagUsage))); assertEquals("e", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("h", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("h", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("i", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("i", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("j", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("j", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("k", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("k", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); - assertEquals(true, scanner.seekBefore(toKV("l", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("l", tagUsage))); assertEquals("k", toRowStr(scanner.getKeyValue())); reader.close(); @@ -148,81 +146,81 @@ public class TestSeekTo extends HBaseTestCase { HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf); reader.loadFileInfo(); HFileScanner scanner = reader.getScanner(false, true); - assertEquals(false, scanner.seekBefore(toKV("a", tagUsage).getKey())); - assertEquals(false, scanner.seekBefore(toKV("b", tagUsage).getKey())); - assertEquals(false, scanner.seekBefore(toKV("c", tagUsage).getKey())); + assertEquals(false, scanner.seekBefore(toKV("a", tagUsage))); + assertEquals(false, scanner.seekBefore(toKV("b", tagUsage))); + assertEquals(false, scanner.seekBefore(toKV("c", tagUsage))); // seekBefore d, so the scanner points to c - assertEquals(true, scanner.seekBefore(toKV("d", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("d", tagUsage))); assertEquals("c", toRowStr(scanner.getKeyValue())); // reseekTo e and g - assertEquals(0, scanner.reseekTo(toKV("c", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("c", tagUsage))); assertEquals("c", toRowStr(scanner.getKeyValue())); - assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("g", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore e, so the scanner points to c - assertEquals(true, scanner.seekBefore(toKV("e", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("e", tagUsage))); assertEquals("c", toRowStr(scanner.getKeyValue())); // reseekTo e and g - assertEquals(0, scanner.reseekTo(toKV("e", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("e", tagUsage))); assertEquals("e", toRowStr(scanner.getKeyValue())); - assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("g", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore f, so the scanner points to e - assertEquals(true, scanner.seekBefore(toKV("f", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("f", tagUsage))); assertEquals("e", toRowStr(scanner.getKeyValue())); // reseekTo e and g - assertEquals(0, scanner.reseekTo(toKV("e", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("e", tagUsage))); assertEquals("e", toRowStr(scanner.getKeyValue())); - assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("g", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore g, so the scanner points to e - assertEquals(true, scanner.seekBefore(toKV("g", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("g", tagUsage))); assertEquals("e", toRowStr(scanner.getKeyValue())); // reseekTo e and g again - assertEquals(0, scanner.reseekTo(toKV("e", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("e", tagUsage))); assertEquals("e", toRowStr(scanner.getKeyValue())); - assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("g", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore h, so the scanner points to g - assertEquals(true, scanner.seekBefore(toKV("h", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("h", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // reseekTo g - assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("g", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore i, so the scanner points to g - assertEquals(true, scanner.seekBefore(toKV("i", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("i", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // reseekTo g - assertEquals(0, scanner.reseekTo(toKV("g", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("g", tagUsage))); assertEquals("g", toRowStr(scanner.getKeyValue())); // seekBefore j, so the scanner points to i - assertEquals(true, scanner.seekBefore(toKV("j", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("j", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); // reseekTo i - assertEquals(0, scanner.reseekTo(toKV("i", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("i", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); // seekBefore k, so the scanner points to i - assertEquals(true, scanner.seekBefore(toKV("k", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("k", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); // reseekTo i and k - assertEquals(0, scanner.reseekTo(toKV("i", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("i", tagUsage))); assertEquals("i", toRowStr(scanner.getKeyValue())); - assertEquals(0, scanner.reseekTo(toKV("k", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("k", tagUsage))); assertEquals("k", toRowStr(scanner.getKeyValue())); // seekBefore l, so the scanner points to k - assertEquals(true, scanner.seekBefore(toKV("l", tagUsage).getKey())); + assertEquals(true, scanner.seekBefore(toKV("l", tagUsage))); assertEquals("k", toRowStr(scanner.getKeyValue())); // reseekTo k - assertEquals(0, scanner.reseekTo(toKV("k", tagUsage).getKey())); + assertEquals(0, scanner.reseekTo(toKV("k", tagUsage))); assertEquals("k", toRowStr(scanner.getKeyValue())); } @@ -239,16 +237,17 @@ public class TestSeekTo extends HBaseTestCase { assertEquals(2, reader.getDataBlockIndexReader().getRootBlockCount()); HFileScanner scanner = reader.getScanner(false, true); // lies before the start of the file. - assertEquals(-1, scanner.seekTo(toKV("a", tagUsage).getKey())); + assertEquals(-1, scanner.seekTo(toKV("a", tagUsage))); - assertEquals(1, scanner.seekTo(toKV("d", tagUsage).getKey())); + assertEquals(1, scanner.seekTo(toKV("d", tagUsage))); assertEquals("c", toRowStr(scanner.getKeyValue())); // Across a block boundary now. - assertEquals(1, scanner.seekTo(toKV("h", tagUsage).getKey())); - assertEquals("g", toRowStr(scanner.getKeyValue())); + // h goes to the next block + assertEquals(-2, scanner.seekTo(toKV("h", tagUsage))); + assertEquals("i", toRowStr(scanner.getKeyValue())); - assertEquals(1, scanner.seekTo(toKV("l", tagUsage).getKey())); + assertEquals(1, scanner.seekTo(toKV("l", tagUsage))); assertEquals("k", toRowStr(scanner.getKeyValue())); reader.close(); @@ -269,26 +268,25 @@ public class TestSeekTo extends HBaseTestCase { int klen = toKV("a", tagUsage).getKey().length; // falls before the start of the file. assertEquals(-1, blockIndexReader.rootBlockContainingKey( - toKV("a", tagUsage).getKey(), 0, klen)); - assertEquals(0, blockIndexReader.rootBlockContainingKey( - toKV("c", tagUsage).getKey(), 0, klen)); + toKV("a", tagUsage))); assertEquals(0, blockIndexReader.rootBlockContainingKey( - toKV("d", tagUsage).getKey(), 0, klen)); + toKV("c", tagUsage))); assertEquals(0, blockIndexReader.rootBlockContainingKey( - toKV("e", tagUsage).getKey(), 0, klen)); + toKV("d", tagUsage))); assertEquals(0, blockIndexReader.rootBlockContainingKey( - toKV("g", tagUsage).getKey(), 0, klen)); + toKV("e", tagUsage))); assertEquals(0, blockIndexReader.rootBlockContainingKey( - toKV("h", tagUsage).getKey(), 0, klen)); + toKV("g", tagUsage))); assertEquals(1, blockIndexReader.rootBlockContainingKey( - toKV("i", tagUsage).getKey(), 0, klen)); + toKV("h", tagUsage))); assertEquals(1, blockIndexReader.rootBlockContainingKey( - toKV("j", tagUsage).getKey(), 0, klen)); + toKV("i", tagUsage))); assertEquals(1, blockIndexReader.rootBlockContainingKey( - toKV("k", tagUsage).getKey(), 0, klen)); + toKV("j", tagUsage))); assertEquals(1, blockIndexReader.rootBlockContainingKey( - toKV("l", tagUsage).getKey(), 0, klen)); - + toKV("k", tagUsage))); + assertEquals(1, blockIndexReader.rootBlockContainingKey( + toKV("l", tagUsage))); reader.close(); }