Index: src/test/java/org/apache/hadoop/hbase/KeyValueTestUtil.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/KeyValueTestUtil.java (revision 1043216) +++ src/test/java/org/apache/hadoop/hbase/KeyValueTestUtil.java (working copy) @@ -22,33 +22,20 @@ import org.apache.hadoop.hbase.util.Bytes; +/** + * Utility for creating {@link KeyValue} + */ public class KeyValueTestUtil { - public static KeyValue create( - String row, - String family, - String qualifier, - long timestamp, - String value) - { + public static KeyValue create(String row, String family, String qualifier, + long timestamp, String value) { return create(row, family, qualifier, timestamp, KeyValue.Type.Put, value); } - public static KeyValue create( - String row, - String family, - String qualifier, - long timestamp, - KeyValue.Type type, - String value) - { - return new KeyValue( - Bytes.toBytes(row), - Bytes.toBytes(family), - Bytes.toBytes(qualifier), - timestamp, - type, - Bytes.toBytes(value) + public static KeyValue create(String row, String family, String qualifier, + long timestamp, KeyValue.Type type, String value) { + return new KeyValue(Bytes.toBytes(row), Bytes.toBytes(family), + Bytes.toBytes(qualifier), timestamp, type, Bytes.toBytes(value) ); } -} +} \ No newline at end of file Index: src/test/java/org/apache/hadoop/hbase/TestKeyValue.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/TestKeyValue.java (revision 1043216) +++ src/test/java/org/apache/hadoop/hbase/TestKeyValue.java (working copy) @@ -34,6 +34,14 @@ public class TestKeyValue extends TestCase { private final Log LOG = LogFactory.getLog(this.getClass().getName()); + public void testVersion() { + int expected = KeyValue.getVersion(KeyValue.VERSION_BITS); + KeyValue kv = new KeyValue(Bytes.toBytes("test"), + System.currentTimeMillis(), Type.Put); + int found = kv.getVersion(); + assertEquals(expected, found); + } + public void testColumnCompare() throws Exception { final byte [] a = Bytes.toBytes("aaa"); byte [] family1 = Bytes.toBytes("abc"); @@ -42,6 +50,7 @@ byte [] qualifier2 = Bytes.toBytes("ef"); KeyValue aaa = new KeyValue(a, family1, qualifier1, 0L, Type.Put, a); + LOG.info("aaa=" + aaa.toString()); assertFalse(aaa.matchingColumn(family2, qualifier2)); assertTrue(aaa.matchingColumn(family1, qualifier1)); aaa = new KeyValue(a, family2, qualifier2, 0L, Type.Put, a); Index: src/main/java/org/apache/hadoop/hbase/KeyValue.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/KeyValue.java (revision 1043218) +++ src/main/java/org/apache/hadoop/hbase/KeyValue.java (working copy) @@ -25,7 +25,6 @@ import java.nio.ByteBuffer; import java.util.Comparator; -import com.google.common.primitives.Longs; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.io.HeapSize; @@ -36,52 +35,59 @@ import org.apache.hadoop.io.Writable; /** - * An HBase Key/Value. This is the fundamental HBase Type. + * An HBase Key/Value. This is the fundamental HBase Type. Its persisted into + * StoreFiles/HFiles in the FileSystem and is what a + * {@link org.hadoop.hbase.client.Result} carries from server to HBase client. * - *

If being used client-side, the primary methods to access individual fields + *

If being used client-side, the primary methods to access KeyValue particles * are {@link #getRow()}, {@link #getFamily()}, {@link #getQualifier()}, * {@link #getTimestamp()}, and {@link #getValue()}. These methods allocate new * byte arrays and return copies. Avoid their use server-side. * - *

Instances of this class are immutable. They do not implement Comparable - * but Comparators are provided. Comparators change with context, - * whether user table or a catalog table comparison. Its critical you use the - * appropriate comparator. There are Comparators for KeyValue instances and - * then for just the Key portion of a KeyValue used mostly by {@link HFile}. + *

Instances of this class are immutable. It does not implement Comparable + * but Comparators are provided. Context dictates which Comparator to use. If + * a user table, use one type. If a catalog table comparison, you'd use anotther. + * Its critical you use the appropriate comparator. There are also Comparators + * for KeyValue instances and then for just the Key portion of a KeyValue used + * mostly by {@link HFile}. * - *

KeyValue wraps a byte array and takes offsets and lengths into passed + *

KeyValue takes a byte array and offsets and lengths into passed * array at where to start interpreting the content as KeyValue. The KeyValue * format inside a byte array is: * <keylength> <valuelength> <key> <value> * Key is further decomposed as: - * <rowlength> <row> <columnfamilylength> <columnfamily> <columnqualifier> <timestamp> <keytype> + * <rowlength> <row> <columnfamilylength> <columnfamily> <columnqualifier> <timestamp> <sequencenumber> <keytype> * The rowlength maximum is Short.MAX_SIZE, * column family length maximum is * Byte.MAX_SIZE, and column qualifier + key length must - * be < Integer.MAX_SIZE. - * The column does not contain the family/qualifier delimiter, {@link #COLUMN_FAMILY_DELIMITER} + * be < Integer.MAX_SIZE. A KeyValue does not contain the + * family/qualifier delimiter, {@link #COLUMN_FAMILY_DELIMITER}. The + * sequencenumber is an internally maintained edit insertion order; + * its used breaking ties where all else in the key portion of two keys are + * the same; we'll favor the KeyValue that has a higher sequencenumber. */ public class KeyValue implements Writable, HeapSize { static final Log LOG = LogFactory.getLog(KeyValue.class); - // TODO: Group Key-only comparators and operations into a Key class, just - // for neatness sake, if can figure what to call it. /** * Colon character in UTF-8 */ public static final char COLUMN_FAMILY_DELIMITER = ':'; - public static final byte[] COLUMN_FAMILY_DELIM_ARRAY = - new byte[]{COLUMN_FAMILY_DELIMITER}; + /** + * {@link #COLUMN_FAMILY_DELIMITER} as a byte array. + */ + public static final byte [] COLUMN_FAMILY_DELIM_ARRAY = + new byte[] {COLUMN_FAMILY_DELIMITER}; /** - * Comparator for plain key/values; i.e. non-catalog table key/values. + * Comparator for plain {@link KeyValue}s.; i.e. non-catalog table KeyValues. */ public static KVComparator COMPARATOR = new KVComparator(); /** - * Comparator for plain key; i.e. non-catalog table key. Works on Key portion - * of KeyValue only. + * Comparator for plain key portion of a {@link KeyValue}s; i.e. non-catalog + * table key. Works on Key portion of KeyValue only. */ public static KeyComparator KEY_COMPARATOR = new KeyComparator(); @@ -111,12 +117,8 @@ /** * Get the appropriate row comparator for the specified table. - * - * Hopefully we can get rid of this, I added this here because it's replacing - * something in HSK. We should move completely off of that. - * * @param tableName The table name. - * @return The comparator. + * @return The comparator to use on this table. */ public static KeyComparator getRowComparator(byte [] tableName) { if(Bytes.equals(HTableDescriptor.ROOT_TABLEDESC.getName(),tableName)) { @@ -128,30 +130,47 @@ return COMPARATOR.getRawComparator(); } - // Size of the timestamp and type byte on end of a key -- a long + a byte. - public static final int TIMESTAMP_TYPE_SIZE = + // Size of the timestamp, insertion seqid long and the type byte on the end + // of the key portion of a KeyValue -- two longs and a byte. + public static final int TIMESTAMP_SEQID_TYPE_SIZE = Bytes.SIZEOF_LONG /* timestamp */ + + Bytes.SIZEOF_LONG /* insertion seqid */ + Bytes.SIZEOF_BYTE /*keytype*/; - // Size of the length shorts and bytes in key. + // Size of the length shorts and bytes in a key portion of a KeyValue. public static final int KEY_INFRASTRUCTURE_SIZE = Bytes.SIZEOF_SHORT /*rowlength*/ + Bytes.SIZEOF_BYTE /*columnfamilylength*/ + - TIMESTAMP_TYPE_SIZE; + TIMESTAMP_SEQID_TYPE_SIZE; // How far into the key the row starts at. First thing to read is the short - // that says how long the row is. + // that says how long the key is, then the data length... then row bytes start. public static final int ROW_OFFSET = Bytes.SIZEOF_INT /*keylength*/ + Bytes.SIZEOF_INT /*valuelength*/; - // Size of the length ints in a KeyValue datastructure. + // Size of the length ints in a KeyValue datastructure; the data and key shorts. public static final int KEYVALUE_INFRASTRUCTURE_SIZE = ROW_OFFSET; /** + * Mask used comparing KeyValue Types without regard to KeyValue version. + * KeyValue version is kept in the upper two bits of the Type byte. + */ + private static final byte VERSION_MASK = (byte)192; + private static final byte VERSION_MASK_INVERSE = ~VERSION_MASK; + + /** + * This KeyValues version in bits that can be OR'd into a Type. + * This define will change as KeyValue version evolves. This is what you + * change changing the KeyValue type. + */ + static final byte VERSION_BITS = (byte)64; + + /** * Key type. - * Has space for other key types to be added later. Cannot rely on - * enum ordinals . They change if item is removed or moved. Do our own codes. + * Has space for other key types to be added later. Does not rely on + * enum ordinals . They change if item is removed or moved. Do our own + * explicit codes. Keys are versioned using two most significant bytes in Type. */ public static enum Type { Minimum((byte)0), @@ -161,9 +180,17 @@ DeleteColumn((byte)12), DeleteFamily((byte)14), - // Maximum is used when searching; you look from maximum on down. - Maximum((byte)255); + // Maximum is used when searching; you look from maximum on down. Shouldn't + // be written out to files or into memstore. + Maximum((byte)63); + // Bit 64 and 128 are reserved used specifying KeyValue version. If top + // two bits zero, then version is 0. If 7th bit is set -- 64 -- then we're + // version 1. Version is ignored when KV is compared. We define Version + // below just to show that the bits are occupied. The version bit twiddling + // and compares are done elsewhere, outside of this enum. See VERSION_MASK + // and VERSION_BITS defines. + private final byte code; Type(final byte c) { @@ -181,12 +208,14 @@ * @return Type associated with passed code. */ public static Type codeToType(final byte b) { + byte bWithVersionStripped = stripVersionFromType(b); for (Type t : Type.values()) { - if (t.getCode() == b) { + // When comparing, do not compare on version. + if (t.getCode() == bWithVersionStripped) { return t; } } - throw new RuntimeException("Unknown code " + b); + throw new RuntimeException("Unknown code " + stripVersionFromType(b)); } } @@ -196,49 +225,31 @@ * key can be equal or lower than this one in memstore or in store file. */ public static final KeyValue LOWESTKEY = - new KeyValue(HConstants.EMPTY_BYTE_ARRAY, HConstants.LATEST_TIMESTAMP); + new KeyValue(HConstants.EMPTY_BYTE_ARRAY, null, null, + HConstants.LATEST_TIMESTAMP, Long.MAX_VALUE, Type.Maximum, null); private byte [] bytes = null; private int offset = 0; private int length = 0; - // the row cached - private byte [] rowCache = null; + private static final long DEFAULT_SEQUENCE_NUMBER = 0; + // Cache of the sequence number portion of this KV. + private long sequenceNumberCache = -1; - /** Here be dragons **/ + // Cache for timestmp long + private long timestampCache = -1; - // used to achieve atomic operations in the memstore. - public long getMemstoreTS() { - return memstoreTS; - } + // Cached reference to the byte array of row content + private byte [] rowCache = null; - public void setMemstoreTS(long memstoreTS) { - this.memstoreTS = memstoreTS; - } - - // default value is 0, aka DNC - private long memstoreTS = 0; - - /** Dragon time over, return to normal business */ - - /** Writable Constructor -- DO NOT USE */ public KeyValue() {} /** - * Creates a KeyValue from the start of the specified byte array. - * Presumes bytes content is formatted as a KeyValue blob. - * @param bytes byte array - */ - public KeyValue(final byte [] bytes) { - this(bytes, 0); - } - - /** * Creates a KeyValue from the specified byte array and offset. * Presumes bytes content starting at offset is - * formatted as a KeyValue blob. + * formatted as a KeyValue. * @param bytes byte array * @param offset offset to start of KeyValue */ @@ -248,7 +259,10 @@ /** * Creates a KeyValue from the specified byte array, starting at offset, and - * for length length. + * for length length. Does NOT make a copy of the passed + * array. + * Presumes bytes content starting at offset is + * formatted as a KeyValue. * @param bytes byte array * @param offset offset to start of the KeyValue * @param length length of the KeyValue @@ -259,29 +273,30 @@ this.length = length; } - /** Constructors that build a new backing byte array from fields */ - /** - * Constructs KeyValue structure filled with null value. - * Sets type to {@link KeyValue.Type#Maximum} - * @param row - row key (arbitrary byte array) + * Constructs KeyValue filled with a null family, qualifier and + * data. Sets type to {@link KeyValue.Type#Maximum}. + * @param row Row key * @param timestamp + * @param sequenceNumber */ public KeyValue(final byte [] row, final long timestamp) { this(row, timestamp, Type.Maximum); } /** - * Constructs KeyValue structure filled with null value. - * @param row - row key (arbitrary byte array) + * Constructs KeyValue filled with a null family, qualifier and + * data. + * @param row Row key + * @param type Key Type to use. * @param timestamp */ public KeyValue(final byte [] row, final long timestamp, Type type) { - this(row, null, null, timestamp, type, null); + this(row, null, null, timestamp, DEFAULT_SEQUENCE_NUMBER, type, null); } /** - * Constructs KeyValue structure filled with null value. + * Constructs KeyValue filled with null data. * Sets type to {@link KeyValue.Type#Maximum} * @param row - row key (arbitrary byte array) * @param family family name @@ -289,22 +304,25 @@ */ public KeyValue(final byte [] row, final byte [] family, final byte [] qualifier) { - this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum); + this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, + DEFAULT_SEQUENCE_NUMBER, Type.Maximum); } /** - * Constructs KeyValue structure filled with null value. + * Constructs KeyValue. + * Sets type to {@link KeyValue.Type#Put} * @param row - row key (arbitrary byte array) * @param family family name * @param qualifier column qualifier */ public KeyValue(final byte [] row, final byte [] family, final byte [] qualifier, final byte [] value) { - this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Put, value); + this(row, family, qualifier, HConstants.LATEST_TIMESTAMP, + DEFAULT_SEQUENCE_NUMBER, Type.Put, value); } /** - * Constructs KeyValue structure filled with specified values. + * Constructs KeyValue filled with null data. * @param row row key * @param family family name * @param qualifier column qualifier @@ -312,50 +330,102 @@ * @param type key type * @throws IllegalArgumentException */ - public KeyValue(final byte[] row, final byte[] family, + KeyValue(final byte[] row, final byte[] family, final byte[] qualifier, final long timestamp, Type type) { - this(row, family, qualifier, timestamp, type, null); + // Used by tests + this(row, family, qualifier, timestamp, DEFAULT_SEQUENCE_NUMBER, type); } /** - * Constructs KeyValue structure filled with specified values. + * Constructs KeyValue filled with null data. * @param row row key * @param family family name * @param qualifier column qualifier * @param timestamp version timestamp + * @param sequenceNumber + * @param type key type + * @throws IllegalArgumentException + */ + public KeyValue(final byte[] row, final byte[] family, + final byte[] qualifier, final long timestamp, + final long sequenceNumber, Type type) { + this(row, family, qualifier, timestamp, sequenceNumber, type, null); + } + + /** + * Constructs KeyValue + * @param row row key + * @param family family name + * @param qualifier column qualifier + * @param timestamp version timestamp * @param value column value * @throws IllegalArgumentException */ - public KeyValue(final byte[] row, final byte[] family, + KeyValue(final byte[] row, final byte[] family, final byte[] qualifier, final long timestamp, final byte[] value) { - this(row, family, qualifier, timestamp, Type.Put, value); + // Used by tests. + this(row, family, qualifier, timestamp, DEFAULT_SEQUENCE_NUMBER, Type.Put, value); } /** - * Constructs KeyValue structure filled with specified values. + * Constructs KeyValue * @param row row key * @param family family name * @param qualifier column qualifier * @param timestamp version timestamp - * @param type key type + * @param sequenceNumber * @param value column value * @throws IllegalArgumentException */ public KeyValue(final byte[] row, final byte[] family, + final byte[] qualifier, final long timestamp, final long sequenceNumber, + final byte[] value) { + this(row, family, qualifier, timestamp, sequenceNumber, Type.Put, value); + } + + /** + * Constructs KeyValue + * @param row row key + * @param family family name + * @param qualifier column qualifier + * @param timestamp version timestamp + * @param type key type + * @param value column value/data. + * @throws IllegalArgumentException + */ + KeyValue(final byte[] row, final byte[] family, final byte[] qualifier, final long timestamp, Type type, final byte[] value) { + // This constructor used by tests only. + this(row, family, qualifier, timestamp, DEFAULT_SEQUENCE_NUMBER, type, value); + } + + /** + * Constructs KeyValue + * @param row row key + * @param family family name + * @param qualifier column qualifier + * @param timestamp version timestamp + * @param type key type + * @param value column value/data. + * @throws IllegalArgumentException + */ + public KeyValue(final byte[] row, final byte[] family, + final byte[] qualifier, final long timestamp, final long sequenceNumber, + Type type, final byte[] value) { this(row, family, qualifier, 0, qualifier==null ? 0 : qualifier.length, - timestamp, type, value, 0, value==null ? 0 : value.length); + timestamp, sequenceNumber, type, value, 0, value==null ? 0 : value.length); } /** - * Constructs KeyValue structure filled with specified values. + * Constructs KeyValue * @param row row key * @param family family name * @param qualifier column qualifier * @param qoffset qualifier offset * @param qlength qualifier length * @param timestamp version timestamp + * @param sequenceNumber * @param type key type * @param value column value * @param voffset value offset @@ -363,16 +433,17 @@ * @throws IllegalArgumentException */ public KeyValue(byte [] row, byte [] family, - byte [] qualifier, int qoffset, int qlength, long timestamp, Type type, + byte [] qualifier, int qoffset, int qlength, long timestamp, + final long sequenceNumber, Type type, byte [] value, int voffset, int vlength) { this(row, 0, row==null ? 0 : row.length, family, 0, family==null ? 0 : family.length, - qualifier, qoffset, qlength, timestamp, type, + qualifier, qoffset, qlength, timestamp, sequenceNumber, type, value, voffset, vlength); } /** - * Constructs KeyValue structure filled with specified values. + * Constructs KeyValue. *

* Column is split into two fields, family and qualifier. * @param row row key @@ -385,6 +456,7 @@ * @param qoffset qualifier offset * @param qlength qualifier length * @param timestamp version timestamp + * @param sequenceNumber * @param type key type * @param value column value * @param voffset value offset @@ -394,11 +466,11 @@ public KeyValue(final byte [] row, final int roffset, final int rlength, final byte [] family, final int foffset, final int flength, final byte [] qualifier, final int qoffset, final int qlength, - final long timestamp, final Type type, + final long timestamp, final long sequenceNumber, final Type type, final byte [] value, final int voffset, final int vlength) { this.bytes = createByteArray(row, roffset, rlength, family, foffset, flength, qualifier, qoffset, qlength, - timestamp, type, value, voffset, vlength); + timestamp, sequenceNumber, type, value, voffset, vlength); this.length = bytes.length; this.offset = 0; } @@ -416,6 +488,7 @@ * @param qoffset qualifier offset * @param qlength qualifier length * @param timestamp version timestamp + * @param sequenceNumber * @param type key type * @param value column value * @param voffset value offset @@ -425,7 +498,7 @@ static byte [] createByteArray(final byte [] row, final int roffset, final int rlength, final byte [] family, final int foffset, int flength, final byte [] qualifier, final int qoffset, int qlength, - final long timestamp, final Type type, + final long timestamp, final long sequenceNumber, final Type type, final byte [] value, final int voffset, int vlength) { if (rlength > Short.MAX_VALUE) { throw new IllegalArgumentException("Row > " + Short.MAX_VALUE); @@ -472,8 +545,12 @@ if(qlength != 0) { pos = Bytes.putBytes(bytes, pos, qualifier, qoffset, qlength); } + // Timestamp pos = Bytes.putLong(bytes, pos, timestamp); - pos = Bytes.putByte(bytes, pos, type.getCode()); + // Sequencenumber + pos = Bytes.putLong(bytes, pos, sequenceNumber); + // Type with KV version added. + pos = Bytes.putByte(bytes, pos, addVersionToType(type.getCode())); if (value != null && value.length > 0) { pos = Bytes.putBytes(bytes, pos, value, voffset, vlength); } @@ -481,45 +558,20 @@ } /** - * Write KeyValue format into a byte array. - *

- * Takes column in the form family:qualifier - * @param row - row key (arbitrary byte array) - * @param roffset - * @param rlength - * @param column - * @param coffset - * @param clength - * @param timestamp + * Decorate passed type with this KVs version. * @param type - * @param value - * @param voffset - * @param vlength - * @return The newly created byte array. + * @return Type with Version added. */ - static byte [] createByteArray(final byte [] row, final int roffset, - final int rlength, - final byte [] column, final int coffset, int clength, - final long timestamp, final Type type, - final byte [] value, final int voffset, int vlength) { - // If column is non-null, figure where the delimiter is at. - int delimiteroffset = 0; - if (column != null && column.length > 0) { - delimiteroffset = getFamilyDelimiterIndex(column, coffset, clength); - if (delimiteroffset > Byte.MAX_VALUE) { - throw new IllegalArgumentException("Family > " + Byte.MAX_VALUE); - } - } else { - return createByteArray(row,roffset,rlength,null,0,0,null,0,0,timestamp, - type,value,voffset,vlength); - } - int flength = delimiteroffset-coffset; - int qlength = clength - flength - 1; - return createByteArray(row, roffset, rlength, column, coffset, - flength, column, delimiteroffset+1, qlength, timestamp, type, - value, voffset, vlength); + private static byte addVersionToType(final byte type) { + // First blank out any existing version before adding this KVs. + byte b = (byte)(((type & VERSION_MASK_INVERSE)) | VERSION_BITS); + return b; } + private static byte stripVersionFromType(final byte b) { + return (byte)(b & VERSION_MASK_INVERSE); + } + // Needed doing 'contains' on List. Only compares the key portion, not the // value. public boolean equals(Object other) { @@ -558,12 +610,7 @@ public KeyValue clone() { byte [] b = new byte[this.length]; System.arraycopy(this.bytes, this.offset, b, 0, this.length); - KeyValue ret = new KeyValue(b, 0, b.length); - // Important to clone the memstoreTS as well - otherwise memstore's - // update-in-place methods (eg increment) will end up creating - // new entries - ret.setMemstoreTS(memstoreTS); - return ret; + return new KeyValue(b, 0, b.length); } //--------------------------------------------------------------------------- @@ -601,20 +648,20 @@ String row = Bytes.toStringBinary(b, o + Bytes.SIZEOF_SHORT, rowlength); int columnoffset = o + Bytes.SIZEOF_SHORT + 1 + rowlength; int familylength = b[columnoffset - 1]; - int columnlength = l - ((columnoffset - o) + TIMESTAMP_TYPE_SIZE); + int columnlength = l - ((columnoffset - o) + TIMESTAMP_SEQID_TYPE_SIZE); String family = familylength == 0? "": Bytes.toStringBinary(b, columnoffset, familylength); String qualifier = columnlength == 0? "": Bytes.toStringBinary(b, columnoffset + familylength, columnlength - familylength); - long timestamp = Bytes.toLong(b, o + (l - TIMESTAMP_TYPE_SIZE)); + long timestamp = Bytes.toLong(b, o + (l - TIMESTAMP_SEQID_TYPE_SIZE)); + long sequenceNumber = + Bytes.toLong(b, o + (l - TIMESTAMP_SEQID_TYPE_SIZE + Bytes.SIZEOF_LONG)); byte type = b[o + l - 1]; -// return row + "/" + family + -// (family != null && family.length() > 0? COLUMN_FAMILY_DELIMITER: "") + -// qualifier + "/" + timestamp + "/" + Type.codeToType(type); return row + "/" + family + (family != null && family.length() > 0? ":" :"") + - qualifier + "/" + timestamp + "/" + Type.codeToType(type); + qualifier + "/" + timestamp + "/" + sequenceNumber + "/" + + Type.codeToType(type); } //--------------------------------------------------------------------------- @@ -644,6 +691,22 @@ return length; } + /** + * @return This instance's version. + */ + public int getVersion() { + return getVersion(getTypeByte(getKeyLength())); + } + + /** + * @param type + * @return Version that is in passed type + */ + static int getVersion(final byte type) { + // Not public. Version is an internal implementation detail. + return (type & VERSION_MASK) >>> 6; + } + //--------------------------------------------------------------------------- // // Length and Offset Calculators @@ -777,7 +840,7 @@ public int getTotalColumnLength() { int rlength = getRowLength(); int foffset = getFamilyOffset(rlength); - return getTotalColumnLength(rlength,foffset); + return getTotalColumnLength(rlength, foffset); } /** @@ -785,7 +848,7 @@ */ public int getTotalColumnLength(int rlength, int foffset) { int flength = getFamilyLength(foffset); - int qlength = getQualifierLength(rlength,flength); + int qlength = getQualifierLength(rlength, flength); return flength + qlength; } @@ -801,7 +864,7 @@ * @return Timestamp offset */ public int getTimestampOffset(final int keylength) { - return getKeyOffset() + keylength - TIMESTAMP_TYPE_SIZE; + return getKeyOffset() + keylength - TIMESTAMP_SEQID_TYPE_SIZE; } /** @@ -826,6 +889,22 @@ return false; } + /** + * @return Edit Sequence Number offset + */ + public int getSequenceNumberOffset() { + return getTimestampOffset(getKeyLength()); + } + + /** + * @param keylength Pass if you have it to save on a int creation. + * @return Edit Sequence Number offset + */ + public int getSequenceNumberOffset(final int keylength) { + return getKeyOffset() + keylength - TIMESTAMP_SEQID_TYPE_SIZE + + Bytes.SIZEOF_LONG; + } + //--------------------------------------------------------------------------- // // Methods that return copies of fields @@ -880,10 +959,8 @@ } /** - * * @return Timestamp */ - private long timestampCache = -1; public long getTimestamp() { if (timestampCache == -1) { timestampCache = getTimestamp(getKeyLength()); @@ -901,6 +978,25 @@ } /** + * @return Edit sequence number + */ + public long getSequenceNumber() { + if (this.sequenceNumberCache == -1) { + this.sequenceNumberCache = getSequenceNumber(getKeyLength()); + } + return sequenceNumberCache; + } + + /** + * @param keylength Pass if you have it to save on a int creation. + * @return Edit sequence number + */ + long getSequenceNumber(final int keylength) { + int offset = getSequenceNumberOffset(keylength); + return Bytes.toLong(this.bytes, offset); + } + + /** * @return Type of this KeyValue. */ public byte getType() { @@ -909,9 +1005,14 @@ /** * @param keylength Pass if you have it to save on a int creation. - * @return Type of this KeyValue. + * @return Type of this KeyValue */ byte getType(final int keylength) { + // Strip version from Type before returning. + return stripVersionFromType(getTypeByte(keylength)); + } + + byte getTypeByte(final int keylength) { return this.bytes[this.offset + keylength - 1 + ROW_OFFSET]; } @@ -995,20 +1096,22 @@ public static class SplitKeyValue { private byte [][] split; SplitKeyValue() { - this.split = new byte[6][]; + this.split = new byte[7][]; } public void setRow(byte [] value) { this.split[0] = value; } public void setFamily(byte [] value) { this.split[1] = value; } public void setQualifier(byte [] value) { this.split[2] = value; } public void setTimestamp(byte [] value) { this.split[3] = value; } - public void setType(byte [] value) { this.split[4] = value; } - public void setValue(byte [] value) { this.split[5] = value; } + public void setSequenceNumber(byte [] value) { this.split[4] = value; } + public void setType(byte [] value) { this.split[5] = value; } + public void setValue(byte [] value) { this.split[6] = value; } public byte [] getRow() { return this.split[0]; } public byte [] getFamily() { return this.split[1]; } public byte [] getQualifier() { return this.split[2]; } public byte [] getTimestamp() { return this.split[3]; } - public byte [] getType() { return this.split[4]; } - public byte [] getValue() { return this.split[5]; } + public byte [] getSequenceNumber() { return this.split[4]; } + public byte [] getType() { return this.split[5]; } + public byte [] getValue() { return this.split[6]; } } public SplitKeyValue split() { @@ -1037,14 +1140,22 @@ System.arraycopy(bytes, splitOffset, qualifier, 0, colLen); splitOffset += colLen; split.setQualifier(qualifier); + byte [] timestamp = new byte[Bytes.SIZEOF_LONG]; System.arraycopy(bytes, splitOffset, timestamp, 0, Bytes.SIZEOF_LONG); splitOffset += Bytes.SIZEOF_LONG; split.setTimestamp(timestamp); + + byte [] sequenceNumber = new byte[Bytes.SIZEOF_LONG]; + System.arraycopy(bytes, splitOffset, sequenceNumber, 0, Bytes.SIZEOF_LONG); + splitOffset += Bytes.SIZEOF_LONG; + split.setSequenceNumber(sequenceNumber); + byte [] type = new byte[1]; type[0] = bytes[splitOffset]; splitOffset += Bytes.SIZEOF_BYTE; split.setType(type); + byte [] value = new byte[valLen]; System.arraycopy(bytes, splitOffset, value, 0, valLen); split.setValue(value); @@ -1111,18 +1222,6 @@ } /** - * @param column Column minus its delimiter - * @return True if column matches. - */ - public boolean matchingColumnNoDelimiter(final byte [] column) { - int rl = getRowLength(); - int o = getFamilyOffset(rl); - int fl = getFamilyLength(o); - int l = fl + getQualifierLength(rl,fl); - return Bytes.compareTo(column, 0, column.length, this.bytes, o, l) == 0; - } - - /** * * @param family column family * @param qualifier column qualifier @@ -1378,13 +1477,10 @@ } public int compare(final KeyValue left, final KeyValue right) { - int ret = getRawComparator().compare(left.getBuffer(), - left.getOffset() + ROW_OFFSET, left.getKeyLength(), - right.getBuffer(), right.getOffset() + ROW_OFFSET, - right.getKeyLength()); - if (ret != 0) return ret; - // Negate this comparison so later edits show up first - return -Longs.compare(left.getMemstoreTS(), right.getMemstoreTS()); + return getRawComparator().compare(left.getBuffer(), + left.getOffset() + ROW_OFFSET, left.getKeyLength(), + right.getBuffer(), right.getOffset() + ROW_OFFSET, + right.getKeyLength()); } public int compareTimestamps(final KeyValue left, final KeyValue right) { @@ -1575,17 +1671,6 @@ } /** - * Creates a KeyValue that is last on the specified row id. That is, - * every other possible KeyValue for the given row would compareTo() - * less than the result of this call. - * @param row row key - * @return Last possible KeyValue on passed row - */ - public static KeyValue createLastOnRow(final byte[] row) { - return new KeyValue(row, null, null, HConstants.LATEST_TIMESTAMP, Type.Minimum); - } - - /** * Create a KeyValue that is smaller than all other possible KeyValues * for the given row. That is any (valid) KeyValue on 'row' would sort * _after_ the result. @@ -1604,26 +1689,11 @@ * @param ts - timestamp * @return First possible key on passed row and timestamp. */ - public static KeyValue createFirstOnRow(final byte [] row, - final long ts) { - return new KeyValue(row, null, null, ts, Type.Maximum); + public static KeyValue createFirstOnRow(final byte [] row, final long ts) { + return new KeyValue(row, ts); } /** - * @param row - row key (arbitrary byte array) - * @param c column - {@link #parseColumn(byte[])} is called to split - * the column. - * @param ts - timestamp - * @return First possible key on passed row, column and timestamp - * @deprecated - */ - public static KeyValue createFirstOnRow(final byte [] row, final byte [] c, - final long ts) { - byte [][] split = parseColumn(c); - return new KeyValue(row, split[0], split[1], ts, Type.Maximum); - } - - /** * Create a KeyValue for the specified row, family and qualifier that would be * smaller than all other possible KeyValues that have the same row,family,qualifier. * Used for seeking. @@ -1634,22 +1704,10 @@ */ public static KeyValue createFirstOnRow(final byte [] row, final byte [] family, final byte [] qualifier) { - return new KeyValue(row, family, qualifier, HConstants.LATEST_TIMESTAMP, Type.Maximum); + return new KeyValue(row, family, qualifier); } /** - * @param row - row key (arbitrary byte array) - * @param f - family name - * @param q - column qualifier - * @param ts - timestamp - * @return First possible key on passed row, column and timestamp - */ - public static KeyValue createFirstOnRow(final byte [] row, final byte [] f, - final byte [] q, final long ts) { - return new KeyValue(row, f, q, ts, Type.Maximum); - } - - /** * Create a KeyValue for the specified row, family and qualifier that would be * smaller than all other possible KeyValues that have the same row, * family, qualifier. @@ -1670,11 +1728,24 @@ final int foffset, final int flength, final byte [] qualifier, final int qoffset, final int qlength) { return new KeyValue(row, roffset, rlength, family, - foffset, flength, qualifier, qoffset, qlength, - HConstants.LATEST_TIMESTAMP, Type.Maximum, null, 0, 0); + foffset, flength, qualifier, qoffset, qlength, + HConstants.LATEST_TIMESTAMP, DEFAULT_SEQUENCE_NUMBER, Type.Maximum, null, + 0, 0); } + /** + * Creates a KeyValue that is last on the specified row id. That is, + * every other possible KeyValue for the given row would compareTo() + * less than the result of this call. + * @param row row key + * @return Last possible KeyValue on passed row + */ + public static KeyValue createLastOnRow(final byte[] row) { + return createLastOnRow(row, 0, row.length, null, 0, 0, null, 0, 0); + } + + /** * Create a KeyValue for the specified row, family and qualifier that would be * larger than or equal to all other possible KeyValues that have the same * row, family, qualifier. @@ -1696,7 +1767,7 @@ final int qoffset, final int qlength) { return new KeyValue(row, roffset, rlength, family, foffset, flength, qualifier, qoffset, qlength, - HConstants.OLDEST_TIMESTAMP, Type.Minimum, null, 0, 0); + HConstants.OLDEST_TIMESTAMP, Long.MIN_VALUE, Type.Minimum, null, 0, 0); } /** @@ -1730,7 +1801,7 @@ System.arraycopy(b, o, newb, ROW_OFFSET, l); Bytes.putInt(newb, 0, b.length); Bytes.putInt(newb, Bytes.SIZEOF_INT, 0); - return new KeyValue(newb); + return new KeyValue(newb, 0); } /** @@ -1855,6 +1926,7 @@ */ public static class KeyComparator implements RawComparator { volatile boolean ignoreTimestamp = false; + volatile boolean ignoreSequenceNumber = false; volatile boolean ignoreType = false; public int compare(byte[] left, int loffset, int llength, byte[] right, @@ -1872,9 +1944,9 @@ // Compare column family. Start compare past row and family length. int lcolumnoffset = Bytes.SIZEOF_SHORT + lrowlength + 1 + loffset; int rcolumnoffset = Bytes.SIZEOF_SHORT + rrowlength + 1 + roffset; - int lcolumnlength = llength - TIMESTAMP_TYPE_SIZE - + int lcolumnlength = llength - TIMESTAMP_SEQID_TYPE_SIZE - (lcolumnoffset - loffset); - int rcolumnlength = rlength - TIMESTAMP_TYPE_SIZE - + int rcolumnlength = rlength - TIMESTAMP_SEQID_TYPE_SIZE - (rcolumnoffset - roffset); // if row matches, and no column in the 'left' AND put type is 'minimum', @@ -1888,10 +1960,12 @@ byte ltype = left[loffset + (llength - 1)]; byte rtype = right[roffset + (rlength - 1)]; - if (lcolumnlength == 0 && ltype == Type.Minimum.getCode()) { + if (lcolumnlength == 0 && + stripVersionFromType(ltype) == Type.Minimum.getCode()) { return 1; // left is bigger. } - if (rcolumnlength == 0 && rtype == Type.Minimum.getCode()) { + if (rcolumnlength == 0 && + stripVersionFromType(rtype) == Type.Minimum.getCode()) { return -1; } @@ -1905,15 +1979,29 @@ if (!this.ignoreTimestamp) { // Get timestamps. long ltimestamp = Bytes.toLong(left, - loffset + (llength - TIMESTAMP_TYPE_SIZE)); + loffset + (llength - TIMESTAMP_SEQID_TYPE_SIZE)); long rtimestamp = Bytes.toLong(right, - roffset + (rlength - TIMESTAMP_TYPE_SIZE)); + roffset + (rlength - TIMESTAMP_SEQID_TYPE_SIZE)); compare = compareTimestamps(ltimestamp, rtimestamp); if (compare != 0) { return compare; } } + if (!this.ignoreSequenceNumber) { + // Get timestamps. + long lseqnum = Bytes.toLong(left, + loffset + (llength - TIMESTAMP_SEQID_TYPE_SIZE + Bytes.SIZEOF_LONG)); + long rseqnum = Bytes.toLong(right, + roffset + (rlength - TIMESTAMP_SEQID_TYPE_SIZE + Bytes.SIZEOF_LONG)); + // Sort same as we do for timestamps where the higher sequence number + // sorts before the lower. + compare = compareTimestamps(lseqnum, rseqnum); + if (compare != 0) { + return compare; + } + } + if (!this.ignoreType) { // Compare types. Let the delete types sort ahead of puts; i.e. types // of higher numbers sort before those of lesser numbers @@ -1958,7 +2046,7 @@ ClassSize.align(ClassSize.ARRAY) + ClassSize.align(length) + (3 * Bytes.SIZEOF_INT) + ClassSize.align(ClassSize.ARRAY) + - (2 * Bytes.SIZEOF_LONG)); + (3 * Bytes.SIZEOF_LONG)); } // this overload assumes that the length bytes have already been read, @@ -1981,4 +2069,4 @@ out.writeInt(this.length); out.write(this.bytes, this.offset, this.length); } -} +} \ No newline at end of file