Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (revision 1213570) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (working copy) @@ -47,7 +47,7 @@ // Add all the blocks for (CachedItem block : blocks) { - cache.cacheBlock(block.blockName, block); + cache.cacheBlock(block.cacheKey, block); } // Let the eviction run @@ -75,12 +75,12 @@ // Confirm empty for (CachedItem block : blocks) { - assertTrue(cache.getBlock(block.blockName, true) == null); + assertTrue(cache.getBlock(block.cacheKey, true) == null); } // Add blocks for (CachedItem block : blocks) { - cache.cacheBlock(block.blockName, block); + cache.cacheBlock(block.cacheKey, block); expectedCacheSize += block.cacheBlockHeapSize(); } @@ -89,7 +89,7 @@ // Check if all blocks are properly cached and retrieved for (CachedItem block : blocks) { - HeapSize buf = cache.getBlock(block.blockName, true); + HeapSize buf = cache.getBlock(block.cacheKey, true); assertTrue(buf != null); assertEquals(buf.heapSize(), block.heapSize()); } @@ -97,7 +97,7 @@ // Re-add same blocks and ensure nothing has changed for (CachedItem block : blocks) { try { - cache.cacheBlock(block.blockName, block); + cache.cacheBlock(block.cacheKey, block); assertTrue("Cache should not allow re-caching a block", false); } catch(RuntimeException re) { // expected @@ -109,7 +109,7 @@ // Check if all blocks are properly cached and retrieved for (CachedItem block : blocks) { - HeapSize buf = cache.getBlock(block.blockName, true); + HeapSize buf = cache.getBlock(block.cacheKey, true); assertTrue(buf != null); assertEquals(buf.heapSize(), block.heapSize()); } @@ -134,7 +134,7 @@ // Add all the blocks for (CachedItem block : blocks) { - cache.cacheBlock(block.blockName, block); + cache.cacheBlock(block.cacheKey, block); expectedCacheSize += block.cacheBlockHeapSize(); } @@ -153,10 +153,10 @@ (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); // All blocks except block 0 and 1 should be in the cache - assertTrue(cache.getBlock(blocks[0].blockName, true) == null); - assertTrue(cache.getBlock(blocks[1].blockName, true) == null); + assertTrue(cache.getBlock(blocks[0].cacheKey, true) == null); + assertTrue(cache.getBlock(blocks[1].cacheKey, true) == null); for(int i=2;i blocks = queue.get(); - assertEquals(blocks.poll().getName(), "cb1"); - assertEquals(blocks.poll().getName(), "cb2"); - assertEquals(blocks.poll().getName(), "cb3"); - assertEquals(blocks.poll().getName(), "cb4"); - assertEquals(blocks.poll().getName(), "cb5"); - assertEquals(blocks.poll().getName(), "cb6"); - assertEquals(blocks.poll().getName(), "cb7"); - assertEquals(blocks.poll().getName(), "cb8"); - + for (int i = 1; i <= 8; i++) { + assertEquals(blocks.poll().getCacheKey().getHfileName(), "cb"+i); + } } public void testQueueSmallBlockEdgeCase() throws Exception { @@ -114,22 +108,15 @@ assertEquals(queue.heapSize(), expectedSize); LinkedList blocks = queue.get(); - assertEquals(blocks.poll().getName(), "cb0"); - assertEquals(blocks.poll().getName(), "cb1"); - assertEquals(blocks.poll().getName(), "cb2"); - assertEquals(blocks.poll().getName(), "cb3"); - assertEquals(blocks.poll().getName(), "cb4"); - assertEquals(blocks.poll().getName(), "cb5"); - assertEquals(blocks.poll().getName(), "cb6"); - assertEquals(blocks.poll().getName(), "cb7"); - assertEquals(blocks.poll().getName(), "cb8"); - + for (int i = 0; i <= 8; i++) { + assertEquals(blocks.poll().getCacheKey().getHfileName(), "cb"+i); + } } private static class CachedBlock extends org.apache.hadoop.hbase.io.hfile.CachedBlock { public CachedBlock(final long heapSize, String name, long accessTime) { - super(name, + super(new BlockCacheKey(name, 0), new Cacheable(){ @Override public long heapSize() { Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java (revision 1213570) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileBlockIndex.java (working copy) @@ -467,8 +467,7 @@ LOG.info("Index block size: " + indexBlockSize + ", compression: " + compr); // Evict all blocks that were cached-on-write by the previous invocation. - blockCache.evictBlocksByPrefix(hfilePath.getName() - + HFile.CACHE_KEY_SEPARATOR); + blockCache.evictBlocksByHfileName(hfilePath.getName()); conf.setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, indexBlockSize); Set keyStrSet = new HashSet(); Index: src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java (revision 1213570) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java (working copy) @@ -30,6 +30,7 @@ import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; import org.apache.hadoop.hbase.io.hfile.CacheStats; import org.apache.hadoop.hbase.io.hfile.Cacheable; import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer; @@ -53,7 +54,7 @@ **/ public class SingleSizeCache implements BlockCache, HeapSize { private final Slab backingStore; - private final ConcurrentMap backingMap; + private final ConcurrentMap backingMap; private final int numBlocks; private final int blockSize; private final CacheStats stats; @@ -90,9 +91,9 @@ // This evictionListener is called whenever the cache automatically // evicts // something. - MapEvictionListener listener = new MapEvictionListener() { + MapEvictionListener listener = new MapEvictionListener() { @Override - public void onEviction(String key, CacheablePair value) { + public void onEviction(BlockCacheKey key, CacheablePair value) { timeSinceLastAccess.set(System.nanoTime() - value.recentlyAccessed.get()); stats.evict(); @@ -106,7 +107,7 @@ } @Override - public void cacheBlock(String blockName, Cacheable toBeCached) { + public void cacheBlock(BlockCacheKey blockName, Cacheable toBeCached) { ByteBuffer storedBlock; try { @@ -138,7 +139,7 @@ } @Override - public Cacheable getBlock(String key, boolean caching) { + public Cacheable getBlock(BlockCacheKey key, boolean caching) { CacheablePair contentBlock = backingMap.get(key); if (contentBlock == null) { stats.miss(caching); @@ -170,7 +171,7 @@ * @param key the key of the entry we are going to evict * @return the evicted ByteBuffer */ - public boolean evictBlock(String key) { + public boolean evictBlock(BlockCacheKey key) { stats.evict(); CacheablePair evictedBlock = backingMap.remove(key); @@ -181,7 +182,7 @@ } - private void doEviction(String key, CacheablePair evictedBlock) { + private void doEviction(BlockCacheKey key, CacheablePair evictedBlock) { long evictedHeap = 0; synchronized (evictedBlock) { if (evictedBlock.serializedData == null) { @@ -282,8 +283,8 @@ /* Since its offheap, it doesn't matter if its in memory or not */ @Override - public void cacheBlock(String blockName, Cacheable buf, boolean inMemory) { - this.cacheBlock(blockName, buf); + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { + this.cacheBlock(cacheKey, buf); } /* @@ -291,10 +292,10 @@ * implemented in the event we want to use this as a standalone cache. */ @Override - public int evictBlocksByPrefix(String prefix) { + public int evictBlocksByHfileName(String hfileName) { int evictedCount = 0; - for (String e : backingMap.keySet()) { - if (e.startsWith(prefix)) { + for (BlockCacheKey e : backingMap.keySet()) { + if (e.getHfileName().equals(hfileName)) { this.evictBlock(e); } } Index: src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java (revision 1213570) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java (working copy) @@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.io.HeapSize; import org.apache.hadoop.hbase.io.hfile.BlockCache; import org.apache.hadoop.hbase.io.hfile.BlockCacheColumnFamilySummary; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; import org.apache.hadoop.hbase.io.hfile.CacheStats; import org.apache.hadoop.hbase.io.hfile.Cacheable; import org.apache.hadoop.hbase.util.ClassSize; @@ -53,7 +54,7 @@ **/ public class SlabCache implements SlabItemActionWatcher, BlockCache, HeapSize { - private final ConcurrentHashMap backingStore; + private final ConcurrentHashMap backingStore; private final TreeMap sizer; static final Log LOG = LogFactory.getLog(SlabCache.class); static final int STAT_THREAD_PERIOD_SECS = 60 * 5; @@ -85,7 +86,7 @@ this.requestStats = new SlabStats(); this.successfullyCachedStats = new SlabStats(); - backingStore = new ConcurrentHashMap(); + backingStore = new ConcurrentHashMap(); sizer = new TreeMap(); this.scheduleThreadPool.scheduleAtFixedRate(new StatisticsThread(this), STAT_THREAD_PERIOD_SECS, STAT_THREAD_PERIOD_SECS, TimeUnit.SECONDS); @@ -184,7 +185,7 @@ } /** - * Cache the block with the specified name and buffer. First finds what size + * Cache the block with the specified key and buffer. First finds what size * SingleSlabCache it should fit in. If the block doesn't fit in any, it will * return without doing anything. *

@@ -192,10 +193,10 @@ * is done, it is assumed that you are reinserting the same exact block due to * a race condition, and will throw a runtime exception. * - * @param blockName block name + * @param cacheKey block cache key * @param cachedItem block buffer */ - public void cacheBlock(String blockName, Cacheable cachedItem) { + public void cacheBlock(BlockCacheKey cacheKey, Cacheable cachedItem) { Entry scacheEntry = getHigherBlock(cachedItem .getSerializedLength()); @@ -212,15 +213,15 @@ * This will throw a runtime exception if we try to cache the same value * twice */ - scache.cacheBlock(blockName, cachedItem); + scache.cacheBlock(cacheKey, cachedItem); } /** * We don't care about whether its in memory or not, so we just pass the call * through. */ - public void cacheBlock(String blockName, Cacheable buf, boolean inMemory) { - cacheBlock(blockName, buf); + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { + cacheBlock(cacheKey, buf); } public CacheStats getStats() { @@ -234,7 +235,7 @@ * @param caching * @return buffer of specified block name, or null if not in cache */ - public Cacheable getBlock(String key, boolean caching) { + public Cacheable getBlock(BlockCacheKey key, boolean caching) { SingleSizeCache cachedBlock = backingStore.get(key); if (cachedBlock == null) { stats.miss(caching); @@ -255,24 +256,24 @@ * Evicts a block from the cache. This is public, and thus contributes to the * the evict counter. */ - public boolean evictBlock(String key) { - SingleSizeCache cacheEntry = backingStore.get(key); + public boolean evictBlock(BlockCacheKey cacheKey) { + SingleSizeCache cacheEntry = backingStore.get(cacheKey); if (cacheEntry == null) { return false; } else { - cacheEntry.evictBlock(key); + cacheEntry.evictBlock(cacheKey); return true; } } @Override - public void onEviction(String key, SingleSizeCache notifier) { + public void onEviction(BlockCacheKey key, SingleSizeCache notifier) { stats.evicted(); backingStore.remove(key); } @Override - public void onInsertion(String key, SingleSizeCache notifier) { + public void onInsertion(BlockCacheKey key, SingleSizeCache notifier) { backingStore.put(key, notifier); } @@ -402,10 +403,10 @@ } } - public int evictBlocksByPrefix(String prefix) { + public int evictBlocksByHfileName(String hfileName) { int numEvicted = 0; - for (String key : backingStore.keySet()) { - if (key.startsWith(prefix)) { + for (BlockCacheKey key : backingStore.keySet()) { + if (key.getHfileName().equals(hfileName)) { if (evictBlock(key)) ++numEvicted; } Index: src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java (revision 1213570) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java (working copy) @@ -20,6 +20,8 @@ package org.apache.hadoop.hbase.io.hfile.slab; +import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; + /** * Interface for objects that want to know when actions occur in a SingleSizeCache. * */ @@ -31,7 +33,7 @@ * @param key the key of the item being evicted * @param notifier the object notifying the SlabCache of the eviction. */ - void onEviction(String key, SingleSizeCache notifier); + void onEviction(BlockCacheKey key, SingleSizeCache notifier); /** * This is called as a callback when an item is inserted into a SingleSizeCache. @@ -39,5 +41,5 @@ * @param key the key of the item being added * @param notifier the object notifying the SlabCache of the insertion.. */ - void onInsertion(String key, SingleSizeCache notifier); + void onInsertion(BlockCacheKey key, SingleSizeCache notifier); } Index: src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (revision 1213570) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (working copy) @@ -110,7 +110,7 @@ static final int statThreadPeriod = 60 * 5; /** Concurrent map (the cache) */ - private final ConcurrentHashMap map; + private final ConcurrentHashMap map; /** Eviction lock (locked when eviction in process) */ private final ReentrantLock evictionLock = new ReentrantLock(true); @@ -220,7 +220,7 @@ } this.maxSize = maxSize; this.blockSize = blockSize; - map = new ConcurrentHashMap(mapInitialSize, + map = new ConcurrentHashMap(mapInitialSize, mapLoadFactor, mapConcurrencyLevel); this.minFactor = minFactor; this.acceptableFactor = acceptableFactor; @@ -258,18 +258,18 @@ * that is done, it is assumed that you are reinserting the same exact * block due to a race condition and will update the buffer but not modify * the size of the cache. - * @param blockName block name + * @param cacheKey block's cache key * @param buf block buffer * @param inMemory if block is in-memory */ - public void cacheBlock(String blockName, Cacheable buf, boolean inMemory) { - CachedBlock cb = map.get(blockName); + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { + CachedBlock cb = map.get(cacheKey); if(cb != null) { throw new RuntimeException("Cached an already cached block"); } - cb = new CachedBlock(blockName, buf, count.incrementAndGet(), inMemory); + cb = new CachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory); long newSize = size.addAndGet(cb.heapSize()); - map.put(blockName, cb); + map.put(cacheKey, cb); elements.incrementAndGet(); if(newSize > acceptableSize() && !evictionInProgress) { runEviction(); @@ -283,20 +283,20 @@ * that is done, it is assumed that you are reinserting the same exact * block due to a race condition and will update the buffer but not modify * the size of the cache. - * @param blockName block name + * @param cacheKey block's cache key * @param buf block buffer */ - public void cacheBlock(String blockName, Cacheable buf) { - cacheBlock(blockName, buf, false); + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { + cacheBlock(cacheKey, buf, false); } /** * Get the buffer of the block with the specified name. - * @param blockName block name - * @return buffer of specified block name, or null if not in cache + * @param cacheKey block's cache key + * @return buffer of specified cache key, or null if not in cache */ - public Cacheable getBlock(String blockName, boolean caching) { - CachedBlock cb = map.get(blockName); + public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) { + CachedBlock cb = map.get(cacheKey); if(cb == null) { stats.miss(caching); return null; @@ -308,31 +308,28 @@ @Override - public boolean evictBlock(String blockName) { - CachedBlock cb = map.get(blockName); + public boolean evictBlock(BlockCacheKey cacheKey) { + CachedBlock cb = map.get(cacheKey); if (cb == null) return false; evictBlock(cb); return true; } /** - * Evicts all blocks whose name starts with the given prefix. This is an + * Evicts all blocks for a specific HFile. This is an * expensive operation implemented as a linear-time search through all blocks * in the cache. Ideally this should be a search in a log-access-time map. * *

* This is used for evict-on-close to remove all blocks of a specific HFile. - * The prefix would be the HFile/StoreFile name (a UUID) followed by an - * underscore, because HFile v2 block names in cache are of the form - * "<storeFileUUID>_<blockOffset>". * * @return the number of blocks evicted */ @Override - public int evictBlocksByPrefix(String prefix) { + public int evictBlocksByHfileName(String hfileName) { int numEvicted = 0; - for (String key : map.keySet()) { - if (key.startsWith(prefix)) { + for (BlockCacheKey key : map.keySet()) { + if (key.getHfileName().equals(hfileName)) { if (evictBlock(key)) ++numEvicted; } @@ -341,7 +338,7 @@ } protected long evictBlock(CachedBlock block) { - map.remove(block.getName()); + map.remove(block.getCacheKey()); size.addAndGet(-1 * block.heapSize()); elements.decrementAndGet(); stats.evicted(); @@ -661,26 +658,19 @@ Map bcs = new HashMap(); - final String pattern = "\\" + HFile.CACHE_KEY_SEPARATOR; - for (CachedBlock cb : map.values()) { - // split name and get the first part (e.g., "8351478435190657655_0") - // see HFile.getBlockCacheKey for structure of block cache key. - String s[] = cb.getName().split(pattern); - if (s.length > 0) { - String sf = s[0]; - Path path = sfMap.get(sf); - if ( path != null) { - BlockCacheColumnFamilySummary lookup = - BlockCacheColumnFamilySummary.createFromStoreFilePath(path); - BlockCacheColumnFamilySummary bcse = bcs.get(lookup); - if (bcse == null) { - bcse = BlockCacheColumnFamilySummary.create(lookup); - bcs.put(lookup,bcse); - } - bcse.incrementBlocks(); - bcse.incrementHeapSize(cb.heapSize()); + String sf = cb.getCacheKey().getHfileName(); + Path path = sfMap.get(sf); + if ( path != null) { + BlockCacheColumnFamilySummary lookup = + BlockCacheColumnFamilySummary.createFromStoreFilePath(path); + BlockCacheColumnFamilySummary bcse = bcs.get(lookup); + if (bcse == null) { + bcse = BlockCacheColumnFamilySummary.create(lookup); + bcs.put(lookup,bcse); } + bcse.incrementBlocks(); + bcse.incrementHeapSize(cb.heapSize()); } } List list = Index: src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java (revision 1213570) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java (working copy) @@ -78,28 +78,28 @@ } @Override - public void cacheBlock(String blockName, Cacheable buf, boolean inMemory) { - onHeapCache.cacheBlock(blockName, buf, inMemory); - offHeapCache.cacheBlock(blockName, buf); + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory) { + onHeapCache.cacheBlock(cacheKey, buf, inMemory); + offHeapCache.cacheBlock(cacheKey, buf); } @Override - public void cacheBlock(String blockName, Cacheable buf) { - onHeapCache.cacheBlock(blockName, buf); - offHeapCache.cacheBlock(blockName, buf); + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { + onHeapCache.cacheBlock(cacheKey, buf); + offHeapCache.cacheBlock(cacheKey, buf); } @Override - public Cacheable getBlock(String blockName, boolean caching) { + public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) { Cacheable cachedBlock; - if ((cachedBlock = onHeapCache.getBlock(blockName, caching)) != null) { + if ((cachedBlock = onHeapCache.getBlock(cacheKey, caching)) != null) { stats.hit(caching); return cachedBlock; - } else if ((cachedBlock = offHeapCache.getBlock(blockName, caching)) != null) { + } else if ((cachedBlock = offHeapCache.getBlock(cacheKey, caching)) != null) { if (caching) { - onHeapCache.cacheBlock(blockName, cachedBlock); + onHeapCache.cacheBlock(cacheKey, cachedBlock); } stats.hit(caching); return cachedBlock; @@ -110,10 +110,10 @@ } @Override - public boolean evictBlock(String blockName) { + public boolean evictBlock(BlockCacheKey cacheKey) { stats.evict(); - boolean cacheA = onHeapCache.evictBlock(blockName); - boolean cacheB = offHeapCache.evictBlock(blockName); + boolean cacheA = onHeapCache.evictBlock(cacheKey); + boolean cacheB = offHeapCache.evictBlock(cacheKey); boolean evicted = cacheA || cacheB; if (evicted) { stats.evicted(); @@ -154,9 +154,9 @@ } @Override - public int evictBlocksByPrefix(String prefix) { - onHeapCache.evictBlocksByPrefix(prefix); - offHeapCache.evictBlocksByPrefix(prefix); + public int evictBlocksByHfileName(String hfileName) { + onHeapCache.evictBlocksByHfileName(hfileName); + offHeapCache.evictBlocksByHfileName(hfileName); return 0; } Index: src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java (revision 1213570) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java (working copy) @@ -27,50 +27,44 @@ /** * Block cache interface. Anything that implements the {@link Cacheable} * interface can be put in the cache. - * - * TODO: Add filename or hash of filename to block cache key. */ public interface BlockCache { /** * Add block to cache. - * @param blockName Zero-based file block number. + * @param cacheKey The block's cache key. * @param buf The block contents wrapped in a ByteBuffer. * @param inMemory Whether block should be treated as in-memory */ - public void cacheBlock(String blockName, Cacheable buf, boolean inMemory); + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory); /** * Add block to cache (defaults to not in-memory). - * @param blockName Zero-based file block number. + * @param cacheKey The block's cache key. * @param buf The object to cache. */ - public void cacheBlock(String blockName, Cacheable buf); + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf); /** * Fetch block from cache. - * @param blockName Block number to fetch. + * @param cacheKey Block to fetch. * @param caching Whether this request has caching enabled (used for stats) * @return Block or null if block is not in 2 cache. */ - public Cacheable getBlock(String blockName, boolean caching); + public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching); /** * Evict block from cache. - * @param blockName Block name to evict + * @param cacheKey Block to evict * @return true if block existed and was evicted, false if not */ - public boolean evictBlock(String blockName); + public boolean evictBlock(BlockCacheKey cacheKey); /** - * Evicts all blocks with name starting with the given prefix. This is - * necessary in cases we need to evict all blocks that belong to a particular - * HFile. In HFile v2 all blocks consist of the storefile name (UUID), an - * underscore, and the block offset in the file. An efficient implementation - * would avoid scanning all blocks in the cache. + * Evicts all blocks for the given HFile. * * @return the number of blocks evicted */ - public int evictBlocksByPrefix(String string); + public int evictBlocksByHfileName(String hfileName); /** * Get the statistics for this block cache. Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java (revision 1213570) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java (working copy) @@ -210,7 +210,7 @@ long startTimeNs = System.nanoTime(); - String cacheKey = HFile.getBlockCacheKey(name, offset); + BlockCacheKey cacheKey = HFile.getBlockCacheKey(name, offset); // Per meta key from any given file, synchronize reads for said block synchronized (metaBlockIndexReader.getRootBlockKey(block)) { @@ -265,7 +265,7 @@ } long offset = dataBlockIndexReader.getRootBlockOffset(block); - String cacheKey = HFile.getBlockCacheKey(name, offset); + BlockCacheKey cacheKey = HFile.getBlockCacheKey(name, offset); // For any given block from any given file, synchronize reads for said // block. Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (revision 1213570) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java (working copy) @@ -19,9 +19,7 @@ */ package org.apache.hadoop.hbase.io.hfile; -import java.io.ByteArrayInputStream; import java.io.DataInput; -import java.io.DataInputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; @@ -184,7 +182,7 @@ // Check cache for block. If found return. long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block); - String cacheKey = HFile.getBlockCacheKey(name, metaBlockOffset); + BlockCacheKey cacheKey = HFile.getBlockCacheKey(name, metaBlockOffset); cacheBlock &= cacheConf.shouldCacheDataOnRead(); if (cacheConf.isBlockCacheEnabled()) { @@ -246,7 +244,7 @@ // the other choice is to duplicate work (which the cache would prevent you // from doing). - String cacheKey = HFile.getBlockCacheKey(name, dataBlockOffset); + BlockCacheKey cacheKey = HFile.getBlockCacheKey(name, dataBlockOffset); IdLock.Entry lockEntry = offsetLock.getLockEntry(dataBlockOffset); try { blockLoads.incrementAndGet(); @@ -311,8 +309,7 @@ public void close(boolean evictOnClose) throws IOException { if (evictOnClose && cacheConf.isBlockCacheEnabled()) { - int numEvicted = cacheConf.getBlockCache().evictBlocksByPrefix(name - + HFile.CACHE_KEY_SEPARATOR); + int numEvicted = cacheConf.getBlockCache().evictBlocksByHfileName(name); if (LOG.isTraceEnabled()) { LOG.trace("On close, file=" + name + " evicted=" + numEvicted + " block(s)"); Index: src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java (revision 1213570) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/CachedBlock.java (working copy) @@ -52,22 +52,22 @@ MEMORY }; - private final String blockName; + private final BlockCacheKey cacheKey; private final Cacheable buf; private volatile long accessTime; private long size; private BlockPriority priority; - public CachedBlock(String blockName, Cacheable buf, long accessTime) { - this(blockName, buf, accessTime, false); + public CachedBlock(BlockCacheKey cacheKey, Cacheable buf, long accessTime) { + this(cacheKey, buf, accessTime, false); } - public CachedBlock(String blockName, Cacheable buf, long accessTime, + public CachedBlock(BlockCacheKey cacheKey, Cacheable buf, long accessTime, boolean inMemory) { - this.blockName = blockName; + this.cacheKey = cacheKey; this.buf = buf; this.accessTime = accessTime; - this.size = ClassSize.align(blockName.length()) + this.size = ClassSize.align(cacheKey.heapSize()) + ClassSize.align(buf.heapSize()) + PER_BLOCK_OVERHEAD; if(inMemory) { this.priority = BlockPriority.MEMORY; @@ -99,11 +99,11 @@ return this.buf; } - public String getName() { - return this.blockName; + public BlockCacheKey getCacheKey() { + return this.cacheKey; } public BlockPriority getPriority() { return this.priority; } -} \ No newline at end of file +} Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (revision 1213570) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (working copy) @@ -138,9 +138,6 @@ public final static String DEFAULT_COMPRESSION = DEFAULT_COMPRESSION_ALGORITHM.getName(); - /** Separator between HFile name and offset in block cache key */ - static final char CACHE_KEY_SEPARATOR = '_'; - // For measuring latency of "typical" reads and writes static volatile AtomicLong readOps = new AtomicLong(); static volatile AtomicLong readTimeNano = new AtomicLong(); @@ -453,8 +450,8 @@ System.exit(prettyPrinter.run(args)); } - public static String getBlockCacheKey(String hfileName, long offset) { - return hfileName + CACHE_KEY_SEPARATOR + offset; + public static BlockCacheKey getBlockCacheKey(String hfileName, long offset) { + return new BlockCacheKey(hfileName, offset); } /** Index: src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java (revision 0) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java (revision 0) @@ -0,0 +1,76 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import org.apache.hadoop.hbase.io.HeapSize; +import org.apache.hadoop.hbase.util.Bytes; + +/** + * Cache Key for use with implementations of {@link BlockCache} + */ +public class BlockCacheKey implements HeapSize { + private String hfileName; + private long offset; + + /** + * Construct a new BlockCacheKey + * @param file The name of the HFile this block belongs to. + * @param offset Offset of the block into the file + */ + public BlockCacheKey(String file, long offset) { + this.hfileName = file; + this.offset = offset; + } + + @Override + public int hashCode() { + return hfileName.hashCode() * 127 + (int) (offset ^ (offset >>> 32)); + } + + @Override + public boolean equals(Object o) { + if (o instanceof BlockCacheKey) { + BlockCacheKey k = (BlockCacheKey) o; + return offset == k.offset + && (hfileName == null ? k.hfileName == null : hfileName + .equals(k.hfileName)); + } else { + return false; + } + } + + @Override + public String toString() { + return hfileName + "_" + offset; + } + + // Strings have two bytes per character due to default + // Java unicode encoding (hence the times 2). + @Override + public long heapSize() { + return 2 * hfileName.length() + Bytes.SIZEOF_LONG; + } + + // can't avoid this unfortunately + /** + * @return The hfileName portion of this cache key + */ + public String getHfileName() { + return hfileName; + } +} Index: src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java (revision 1213570) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java (working copy) @@ -33,14 +33,14 @@ */ public class SimpleBlockCache implements BlockCache { private static class Ref extends SoftReference { - public String blockId; - public Ref(String blockId, Cacheable block, ReferenceQueue q) { + public BlockCacheKey blockId; + public Ref(BlockCacheKey blockId, Cacheable block, ReferenceQueue q) { super(block, q); this.blockId = blockId; } } - private Map cache = - new HashMap(); + private Map cache = + new HashMap(); private ReferenceQueue q = new ReferenceQueue(); public int dumps = 0; @@ -68,26 +68,26 @@ return cache.size(); } - public synchronized Cacheable getBlock(String blockName, boolean caching) { + public synchronized Cacheable getBlock(BlockCacheKey cacheKey, boolean caching) { processQueue(); // clear out some crap. - Ref ref = cache.get(blockName); + Ref ref = cache.get(cacheKey); if (ref == null) return null; return ref.get(); } - public synchronized void cacheBlock(String blockName, Cacheable block) { - cache.put(blockName, new Ref(blockName, block, q)); + public synchronized void cacheBlock(BlockCacheKey cacheKey, Cacheable block) { + cache.put(cacheKey, new Ref(cacheKey, block, q)); } - public synchronized void cacheBlock(String blockName, Cacheable block, + public synchronized void cacheBlock(BlockCacheKey cacheKey, Cacheable block, boolean inMemory) { - cache.put(blockName, new Ref(blockName, block, q)); + cache.put(cacheKey, new Ref(cacheKey, block, q)); } @Override - public boolean evictBlock(String blockName) { - return cache.remove(blockName) != null; + public boolean evictBlock(BlockCacheKey cacheKey) { + return cache.remove(cacheKey) != null; } public void shutdown() { @@ -119,7 +119,7 @@ } @Override - public int evictBlocksByPrefix(String string) { + public int evictBlocksByHfileName(String string) { throw new UnsupportedOperationException(); }