Index: src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java =================================================================== --- src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java (revision 692598) +++ src/test/org/apache/hadoop/hbase/regionserver/TestHStoreFile.java (working copy) @@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.HBaseTestCase; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.HRegionInfo; import org.apache.hadoop.hbase.HStoreKey; /** * Test HStoreFile @@ -126,7 +127,7 @@ throws IOException { // Make a store file and write data to it. HStoreFile hsf = new HStoreFile(this.conf, this.fs, this.dir, - JenkinsHash.hash(Bytes.toBytes(getName())), + HRegionInfo.FIRST_META_REGIONINFO, Bytes.toBytes("colfamily"), 1234567890L, null); MapFile.Writer writer = hsf.getWriter(this.fs, SequenceFile.CompressionType.NONE, false, 0); @@ -145,7 +146,7 @@ midkey, HStoreFile.Range.top); HStoreFile refHsf = new HStoreFile(this.conf, this.fs, new Path(DIR, getName()), - JenkinsHash.hash(Bytes.toBytes(getName() + "_reference")), + HRegionInfo.FIRST_META_REGIONINFO, hsf.getColFamily(), 456, reference); // Assert that reference files are written and that we can write and // read the info reference file at least. Index: src/java/org/apache/hadoop/hbase/HStoreKey.java =================================================================== --- src/java/org/apache/hadoop/hbase/HStoreKey.java (revision 692598) +++ src/java/org/apache/hadoop/hbase/HStoreKey.java (working copy) @@ -26,6 +26,7 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.io.WritableComparable; +import org.apache.hadoop.io.WritableComparator; /** * A Key for a stored row. @@ -50,7 +51,7 @@ public HStoreKey() { super(); } - + /** * Create an HStoreKey specifying only the row * The column defaults to the empty string, the time stamp defaults to @@ -73,13 +74,37 @@ this(row, Long.MAX_VALUE); } + /** * Create an HStoreKey specifying the row and timestamp * The column and table names default to the empty string * * @param row row key + * @param hri + */ + public HStoreKey(final byte [] row, final HRegionInfo hri) { + this(row, HConstants.EMPTY_BYTE_ARRAY, hri); + } + + /** + * Create an HStoreKey specifying the row and timestamp + * The column and table names default to the empty string + * + * @param row row key * @param timestamp timestamp value + * @param hri HRegionInfo */ + public HStoreKey(final byte [] row, long timestamp, final HRegionInfo hri) { + this(row, HConstants.EMPTY_BYTE_ARRAY, timestamp, hri); + } + + /** + * Create an HStoreKey specifying the row and timestamp + * The column and table names default to the empty string + * + * @param row row key + * @param timestamp timestamp value + */ public HStoreKey(final byte [] row, long timestamp) { this(row, HConstants.EMPTY_BYTE_ARRAY, timestamp); } @@ -188,7 +213,7 @@ * @param other the source key */ public HStoreKey(HStoreKey other) { - this(other.row, other.column, other.timestamp); + this(other.row, other.column, other.timestamp, other.regionInfo); } /** @@ -317,15 +342,19 @@ /** {@inheritDoc} */ public int compareTo(Object o) { - HStoreKey other = (HStoreKey)o; - int result = compareTwoRowKeys(this.regionInfo, this.row, other.row); + return compareTo(this.regionInfo, this, (HStoreKey)o); + } + + static int compareTo(final HRegionInfo hri, final HStoreKey left, + final HStoreKey right) { + int result = compareTwoRowKeys(hri, left.getRow(), right.getRow()); if (result != 0) { return result; } - result = this.column == null && other.column == null? 0: - this.column == null && other.column != null? -1: - this.column != null && other.column == null? 1: - Bytes.compareTo(this.column, other.column); + result = left.getColumn() == null && right.getColumn() == null? 0: + left.getColumn() == null && right.getColumn() != null? -1: + left.getColumn() != null && right.getColumn() == null? 1: + Bytes.compareTo(left.getColumn(), right.getColumn()); if (result != 0) { return result; } @@ -333,9 +362,9 @@ // wrong but it is intentional. This way, newer timestamps are first // found when we iterate over a memcache and newer versions are the // first we trip over when reading from a store file. - if (this.timestamp < other.timestamp) { + if (left.getTimestamp() < right.getTimestamp()) { result = 1; - } else if (this.timestamp > other.timestamp) { + } else if (left.getTimestamp() > right.getTimestamp()) { result = -1; } return result; @@ -482,9 +511,8 @@ if(rowCompare == 0) rowCompare = Bytes.compareTo(keysA[1], KeysB[1]); return rowCompare; - } else { - return Bytes.compareTo(rowA, rowB); } + return Bytes.compareTo(rowA, rowB); } /** @@ -513,10 +541,14 @@ break; } } - byte [] row = new byte[offset]; - System.arraycopy(rowKey, 0, row, 0,offset); - byte [] timestamp = new byte[rowKey.length - offset - 1]; - System.arraycopy(rowKey, offset+1, timestamp, 0,rowKey.length - offset - 1); + byte [] row = rowKey; + byte [] timestamp = HConstants.EMPTY_BYTE_ARRAY; + if (offset != -1) { + row = new byte[offset]; + System.arraycopy(rowKey, 0, row, 0, offset); + timestamp = new byte[rowKey.length - offset - 1]; + System.arraycopy(rowKey, offset+1, timestamp, 0,rowKey.length - offset - 1); + } byte[][] elements = new byte[2][]; elements[0] = row; elements[1] = timestamp; @@ -538,4 +570,18 @@ this.column = Bytes.readByteArray(in); this.timestamp = in.readLong(); } + + // WritableComparator that takes account of meta keys. + public static class HStoreKeyWritableComparator extends WritableComparator { + private final HRegionInfo hri; + + public HStoreKeyWritableComparator(final HRegionInfo hri) { + super(HStoreKey.class); + this.hri = hri; + } + + public int compare(final HStoreKey left, final HStoreKey right) { + return compareTo(this.hri, left, right); + } + } } Index: src/java/org/apache/hadoop/hbase/regionserver/Memcache.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/Memcache.java (revision 692598) +++ src/java/org/apache/hadoop/hbase/regionserver/Memcache.java (working copy) @@ -259,7 +259,7 @@ synchronized (map) { // Make an HSK with maximum timestamp so we get past most of the current // rows cell entries. - HStoreKey hsk = new HStoreKey(row, HConstants.LATEST_TIMESTAMP); + HStoreKey hsk = new HStoreKey(row, HConstants.LATEST_TIMESTAMP, this.regionInfo); SortedMap tailMap = map.tailMap(hsk); // Iterate until we fall into the next row; i.e. move off current row for (Map.Entry es: tailMap.entrySet()) { @@ -377,8 +377,8 @@ final Set deletes) { // We want the earliest possible to start searching from. Start before // the candidate key in case it turns out a delete came in later. - HStoreKey search_key = candidateKeys.isEmpty()? new HStoreKey(row): - new HStoreKey(candidateKeys.firstKey().getRow()); + HStoreKey search_key = candidateKeys.isEmpty()? new HStoreKey(row, this.regionInfo): + new HStoreKey(candidateKeys.firstKey().getRow(), this.regionInfo); List victims = new ArrayList(); long now = System.currentTimeMillis(); @@ -496,7 +496,7 @@ // smaller acceptable candidate keys would have caused us to start // our search earlier in the list, and we wouldn't be searching here. SortedMap thisRowTailMap = - headMap.tailMap(new HStoreKey(headMap.lastKey().getRow())); + headMap.tailMap(new HStoreKey(headMap.lastKey().getRow(), this.regionInfo)); Iterator key_iterator = thisRowTailMap.keySet().iterator(); do { HStoreKey found_key = key_iterator.next(); @@ -521,7 +521,7 @@ } static HStoreKey stripTimestamp(HStoreKey key) { - return new HStoreKey(key.getRow(), key.getColumn()); + return new HStoreKey(key.getRow(), key.getColumn(), key.getHRegionInfo()); } /* Index: src/java/org/apache/hadoop/hbase/regionserver/HStore.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/HStore.java (revision 692598) +++ src/java/org/apache/hadoop/hbase/regionserver/HStore.java (working copy) @@ -319,7 +319,8 @@ || !HStoreKey.matchingFamily(family.getName(), column)) { continue; } - HStoreKey k = new HStoreKey(key.getRow(), column, val.getTimestamp()); + HStoreKey k = new HStoreKey(key.getRow(), column, val.getTimestamp(), + this.info); reconstructedCache.put(k, val.getVal()); editsCount++; // Every 2k edits, tell the reporter we're making progress. @@ -390,7 +391,7 @@ if (isReference) { reference = HStoreFile.readSplitInfo(p, fs); } - curfile = new HStoreFile(conf, fs, basedir, info.getEncodedName(), + curfile = new HStoreFile(conf, fs, basedir, this.info, family.getName(), fid, reference); long storeSeqId = -1; try { @@ -424,7 +425,9 @@ // Try fixing this file.. if we can. Use the hbase version of fix. // Need to remove the old index file first else fix won't go ahead. this.fs.delete(new Path(mapfile, MapFile.INDEX_FILE_NAME), false); - long count = MapFile.fix(this.fs, mapfile, HStoreFile.HbaseMapFile.KEY_CLASS, + // TODO: This is going to fail if we are to rebuild a file from + // meta because it won't have right comparator: HBASE-848. + long count = MapFile.fix(this.fs, mapfile, HStoreKey.class, HStoreFile.HbaseMapFile.VALUE_CLASS, false, this.conf); if (LOG.isDebugEnabled()) { LOG.debug("Fixed index on " + mapfile.toString() + "; had " + @@ -589,7 +592,7 @@ long now = System.currentTimeMillis(); // A. Write the Maps out to the disk HStoreFile flushedFile = new HStoreFile(conf, fs, basedir, - info.getEncodedName(), family.getName(), -1L, null); + this.info, family.getName(), -1L, null); MapFile.Writer out = flushedFile.getWriter(this.fs, this.compression, this.family.isBloomfilter(), cache.size()); out.setIndexInterval(family.getMapFileIndexInterval()); @@ -873,8 +876,7 @@ // Step through them, writing to the brand-new MapFile HStoreFile compactedOutputFile = new HStoreFile(conf, fs, - this.compactionDir, info.getEncodedName(), family.getName(), - -1L, null); + this.compactionDir, this.info, family.getName(), -1L, null); if (LOG.isDebugEnabled()) { LOG.debug("started compaction of " + rdrs.size() + " files into " + FSUtils.getPath(compactedOutputFile.getMapFilePath())); @@ -1045,7 +1047,7 @@ try { // 1. Moving the new MapFile into place. HStoreFile finalCompactedFile = new HStoreFile(conf, fs, basedir, - info.getEncodedName(), family.getName(), -1, null); + this.info, family.getName(), -1, null); if (LOG.isDebugEnabled()) { LOG.debug("moving " + FSUtils.getPath(compactedFile.getMapFilePath()) + " to " + FSUtils.getPath(finalCompactedFile.getMapFilePath())); @@ -1769,7 +1771,7 @@ } static HStoreKey stripTimestamp(HStoreKey key) { - return new HStoreKey(key.getRow(), key.getColumn()); + return new HStoreKey(key.getRow(), key.getColumn(), key.getHRegionInfo()); } /* @@ -1952,4 +1954,8 @@ return key; } } + + HRegionInfo getHRegionInfo() { + return this.info; + } } Index: src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java (revision 692598) +++ src/java/org/apache/hadoop/hbase/regionserver/HStoreFile.java (working copy) @@ -119,25 +119,28 @@ private final HBaseConfiguration conf; private final FileSystem fs; private final Reference reference; + private final HRegionInfo hri; /** * Constructor that fully initializes the object * @param conf Configuration object * @param basedir qualified path that is parent of region directory - * @param encodedRegionName file name friendly name of the region * @param colFamily name of the column family * @param fileId file identifier * @param ref Reference to another HStoreFile. + * @param hri The region info for this file (HACK HBASE-868). TODO: Fix. * @throws IOException */ HStoreFile(HBaseConfiguration conf, FileSystem fs, Path basedir, - int encodedRegionName, byte [] colFamily, long fileId, - final Reference ref) throws IOException { + final HRegionInfo hri, byte [] colFamily, long fileId, + final Reference ref) + throws IOException { this.conf = conf; this.fs = fs; this.basedir = basedir; - this.encodedRegionName = encodedRegionName; + this.encodedRegionName = hri.getEncodedName(); this.colFamily = colFamily; + this.hri = hri; long id = fileId; if (id == -1) { @@ -431,7 +434,7 @@ "HStoreFile reference"); } return new BloomFilterMapFile.Writer(conf, fs, - getMapFilePath().toString(), compression, bloomFilter, nrows); + getMapFilePath().toString(), compression, bloomFilter, nrows, this.hri); } /** @@ -584,7 +587,6 @@ * Hbase customizations of MapFile. */ static class HbaseMapFile extends MapFile { - static final Class KEY_CLASS = HStoreKey.class; static final Class VALUE_CLASS = ImmutableBytesWritable.class; @@ -672,9 +674,10 @@ * @throws IOException */ public HbaseWriter(Configuration conf, FileSystem fs, String dirName, - SequenceFile.CompressionType compression) + SequenceFile.CompressionType compression, final HRegionInfo hri) throws IOException { - super(conf, fs, dirName, KEY_CLASS, VALUE_CLASS, compression); + super(conf, fs, dirName, new HStoreKey.HStoreKeyWritableComparator(hri), + VALUE_CLASS, compression); // Default for mapfiles is 128. Makes random reads faster if we // have more keys indexed and we're not 'next'-ing around in the // mapfile. @@ -788,14 +791,15 @@ * @param compression * @param filter * @param nrows + * @param hri * @throws IOException */ @SuppressWarnings("unchecked") public Writer(Configuration conf, FileSystem fs, String dirName, SequenceFile.CompressionType compression, final boolean filter, - int nrows) + int nrows, final HRegionInfo hri) throws IOException { - super(conf, fs, dirName, compression); + super(conf, fs, dirName, compression, hri); this.dirName = dirName; this.fs = fs; if (filter) { Index: src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java (revision 692598) +++ src/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java (working copy) @@ -273,7 +273,7 @@ private boolean findFirstRow(int i, final byte [] firstRow) throws IOException { ImmutableBytesWritable ibw = new ImmutableBytesWritable(); HStoreKey firstKey - = (HStoreKey)readers[i].getClosest(new HStoreKey(firstRow), ibw); + = (HStoreKey)readers[i].getClosest(new HStoreKey(firstRow, this.store.getHRegionInfo()), ibw); if (firstKey == null) { // Didn't find it. Close the scanner and return TRUE closeSubScanner(i); Index: src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java (revision 692598) +++ src/java/org/apache/hadoop/hbase/regionserver/HStoreScanner.java (working copy) @@ -165,7 +165,7 @@ // values with older ones. So now we only insert // a result if the map does not contain the key. HStoreKey hsk = new HStoreKey(key.getRow(), HConstants.EMPTY_BYTE_ARRAY, - key.getTimestamp()); + key.getTimestamp(), this.store.getHRegionInfo()); for (Map.Entry e : resultSets[i].entrySet()) { hsk.setColumn(e.getKey()); if (HLogEdit.isDeleted(e.getValue().getValue())) { Index: src/java/org/apache/hadoop/hbase/regionserver/HRegion.java =================================================================== --- src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (revision 692598) +++ src/java/org/apache/hadoop/hbase/regionserver/HRegion.java (working copy) @@ -232,7 +232,7 @@ } for (HStoreFile hsf: srcFiles) { HStoreFile dst = new HStoreFile(conf, fs, basedir, - newRegionInfo.getEncodedName(), colFamily, -1, null); + newRegionInfo, colFamily, -1, null); if (LOG.isDebugEnabled()) { LOG.debug("Renaming " + hsf + " to " + dst); } @@ -789,15 +789,15 @@ // A reference to the bottom half of the hsf store file. HStoreFile.Reference aReference = new HStoreFile.Reference( this.regionInfo.getEncodedName(), h.getFileId(), - new HStoreKey(midKey), HStoreFile.Range.bottom); + new HStoreKey(midKey, this.regionInfo), HStoreFile.Range.bottom); HStoreFile a = new HStoreFile(this.conf, fs, splits, - regionAInfo.getEncodedName(), h.getColFamily(), -1, aReference); + regionAInfo, h.getColFamily(), -1, aReference); // Reference to top half of the hsf store file. HStoreFile.Reference bReference = new HStoreFile.Reference( this.regionInfo.getEncodedName(), h.getFileId(), new HStoreKey(midKey), HStoreFile.Range.top); HStoreFile b = new HStoreFile(this.conf, fs, splits, - regionBInfo.getEncodedName(), h.getColFamily(), -1, bReference); + regionBInfo, h.getColFamily(), -1, bReference); h.splitStoreFile(a, b, this.fs); } @@ -1163,7 +1163,7 @@ checkRow(row); checkColumn(column); // Don't need a row lock for a simple get - HStoreKey key = new HStoreKey(row, column, timestamp); + HStoreKey key = new HStoreKey(row, column, timestamp, this.regionInfo); return getStore(column).get(key, numVersions); } finally { splitsAndClosesLock.readLock().unlock(); @@ -1195,7 +1195,7 @@ checkColumn(column); } } - HStoreKey key = new HStoreKey(row, ts); + HStoreKey key = new HStoreKey(row, ts, this.regionInfo); Integer lid = obtainRowLock(row); HashSet storeSet = new HashSet(); try { @@ -1259,14 +1259,14 @@ byte [] closestKey = store.getRowKeyAtOrBefore(row); // if it happens to be an exact match, we can stop looping if (HStoreKey.equalsTwoRowKeys(regionInfo,row, closestKey)) { - key = new HStoreKey(closestKey); + key = new HStoreKey(closestKey, this.regionInfo); break; } // otherwise, we need to check if it's the max and move to the next if (closestKey != null && (key == null || HStoreKey.compareTwoRowKeys( regionInfo,closestKey, key.getRow()) > 0) ) { - key = new HStoreKey(closestKey); + key = new HStoreKey(closestKey, this.regionInfo); } } if (key == null) { @@ -1388,7 +1388,8 @@ try { List deletes = null; for (BatchOperation op: b) { - HStoreKey key = new HStoreKey(row, op.getColumn(), commitTime); + HStoreKey key = new HStoreKey(row, op.getColumn(), commitTime, + this.regionInfo); byte[] val = null; if (op.isPut()) { val = op.getValue(); @@ -1507,7 +1508,7 @@ long now = System.currentTimeMillis(); try { for (HStore store : stores.values()) { - List keys = store.getKeys(new HStoreKey(row, ts), + List keys = store.getKeys(new HStoreKey(row, ts, this.regionInfo), ALL_VERSIONS, now); TreeMap edits = new TreeMap(); for (HStoreKey key: keys) { @@ -1538,7 +1539,7 @@ // find the HStore for the column family HStore store = getStore(family); // find all the keys that match our criteria - List keys = store.getKeys(new HStoreKey(row, timestamp), + List keys = store.getKeys(new HStoreKey(row, timestamp, this.regionInfo), ALL_VERSIONS, now); // delete all the cells TreeMap edits = new TreeMap(); @@ -1566,7 +1567,7 @@ final long ts, final int versions) throws IOException { checkReadOnly(); - HStoreKey origin = new HStoreKey(row, column, ts); + HStoreKey origin = new HStoreKey(row, column, ts, this.regionInfo); Set keys = getKeys(origin, versions); if (keys.size() > 0) { TreeMap edits = new TreeMap(); @@ -2138,7 +2139,8 @@ byte [] row = r.getRegionName(); Integer lid = meta.obtainRowLock(row); try { - HStoreKey key = new HStoreKey(row, COL_REGIONINFO, System.currentTimeMillis()); + HStoreKey key = new HStoreKey(row, COL_REGIONINFO, + System.currentTimeMillis(), r.getRegionInfo()); TreeMap edits = new TreeMap(); edits.put(key, Writables.getBytes(r.getRegionInfo())); meta.update(edits);