diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 292afe8..59fd2c0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; +import org.apache.hadoop.hbase.io.hfile.slab.SlabCache; import org.apache.hadoop.hbase.util.DirectMemoryUtils; import org.apache.hadoop.util.StringUtils; @@ -76,14 +77,49 @@ public class CacheConfig { /** * Configuration keys for Bucket cache */ + + /** + * Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path + * to the file that will host the file-based cache. See BucketCache#getIOEngineFromName() for + * list of supported ioengine options. + * + *

Set this option and a non-zero {@link BUCKET_CACHE_SIZE_KEY} to enable bucket cache. + */ public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine"; + + /** + * When using bucket cache, this is a float that EITHER represents a percentage of total heap + * memory size to give to the cache (if < 1.0) OR, it is the capacity in megabytes of the cache. + * + *

The resultant size is further divided if {@link BUCKET_CACHE_COMBINED_KEY} is set (It is + * set by default. When false, bucket cache serves as an "L2" cache to the "L1" + * {@link LruBlockCache}). The percentage is set in + * with {@link BUCKET_CACHE_COMBINED_PERCENTAGE_KEY} float. + */ public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size"; + + /** + * If the chosen ioengine can persist its state across restarts, the path to the file to + * persist to. + */ public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY = "hbase.bucketcache.persistent.path"; + + /** + * If the bucket cache is used in league with the lru on-heap block cache (meta blocks such + * as indices and blooms are kept in the lru blockcache and the data blocks in the + * bucket cache). + */ public static final String BUCKET_CACHE_COMBINED_KEY = "hbase.bucketcache.combinedcache.enabled"; + + /** + * A float which designates how much of the overall cache to give to bucket cache + * and how much to on-heap lru cache when {@link BUCKET_CACHE_COMBINED_KEY} is set. + */ public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY = "hbase.bucketcache.percentage.in.combinedcache"; + public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads"; public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = "hbase.bucketcache.writer.queuelength"; @@ -95,6 +131,19 @@ public class CacheConfig { public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64; public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f; + /** + * Setting this float to a non-null value turns on {@link DoubleBlockCache} + * which makes use of the {@link LruBlockCache} and {@link SlabCache}. + * + * The float value of between 0 and 1 will be multiplied against the setting for + * -XX:MaxDirectMemorySize to figure what size of the offheap allocation to give + * over to slab cache. + * + * Slab cache has been little used and is likely to be deprecated in the near future. + */ + public static final String SLAB_CACHE_OFFHEAP_PERCENTAGE_KEY = + "hbase.offheapcache.percentage"; + // Defaults public static final boolean DEFAULT_CACHE_DATA_ON_READ = true; @@ -404,7 +453,7 @@ public class CacheConfig { } } LOG.info("Allocating LruBlockCache with maximum size " + - StringUtils.humanReadableInt(lruCacheSize)); + StringUtils.humanReadableInt(lruCacheSize) + ", blockSize=" + blockSize); LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize); lruCache.setVictimCache(bucketCache); if (bucketCache != null && combinedWithLru) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java index 5145199..903e6d7 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java @@ -37,15 +37,15 @@ import org.apache.hadoop.hbase.io.HeapSize; public interface Cacheable extends HeapSize { /** * Returns the length of the ByteBuffer required to serialized the object. If the - * object cannot be serialized, it should also return 0. + * object cannot be serialized, it should return 0. * - * @return int length in bytes of the serialized form. + * @return int length in bytes of the serialized form or 0 if the object cannot be cached. */ - int getSerializedLength(); /** * Serializes its data into destination. + * @param destination Where to serialize to */ void serialize(ByteBuffer destination); @@ -60,5 +60,4 @@ public interface Cacheable extends HeapSize { * @return the block type of this cached HFile block */ BlockType getBlockType(); - -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 2aae578..37221df 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -30,15 +30,14 @@ import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; /** * CombinedBlockCache is an abstraction layer that combines * {@link LruBlockCache} and {@link BucketCache}. The smaller lruCache is used - * to cache bloom blocks and index blocks , the larger bucketCache is used to - * cache data blocks. getBlock reads first from the smaller lruCache before - * looking for the block in the bucketCache. Metrics are the combined size and - * hits and misses of both caches. + * to cache bloom blocks and index blocks. The larger bucketCache is used to + * cache data blocks. {@link #getBlock(BlockCacheKey, boolean, boolean) reads + * first from the smaller lruCache before looking for the block in the bucketCache. + * Metrics are the combined size and hits and misses of both caches. * - **/ + */ @InterfaceAudience.Private public class CombinedBlockCache implements BlockCache, HeapSize { - private final LruBlockCache lruCache; private final BucketCache bucketCache; private final CombinedCacheStats combinedCacheStats; @@ -73,6 +72,8 @@ public class CombinedBlockCache implements BlockCache, HeapSize { @Override public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) { + // TODO: is there a hole here, or just awkwardness since in the lruCache getBlock + // we end up calling bucketCache.getBlock. if (lruCache.containsBlock(cacheKey)) { return lruCache.getBlock(cacheKey, caching, repeat); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 045f3bf..d5d9ad1 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -132,7 +132,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { static final int statThreadPeriod = 60 * 5; /** Concurrent map (the cache) */ - private final ConcurrentHashMap map; + private final Map map; /** Eviction lock (locked when eviction in process) */ private final ReentrantLock evictionLock = new ReentrantLock(true); @@ -190,7 +190,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { /** Whether in-memory hfile's data block has higher priority when evicting */ private boolean forceInMemory; - /** Where to send victims (blocks evicted from the cache) */ + /** Where to send victims (blocks evicted/missing from the cache) */ + // TODO: Fix it so this is not explicit refernce to a partiular BlockCache implementation. private BucketCache victimHandler = null; /** @@ -382,7 +383,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { @Override public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat) { CachedBlock cb = map.get(cacheKey); - if(cb == null) { + if (cb == null) { if (!repeat) stats.miss(caching); if (victimHandler != null) return victimHandler.getBlock(cacheKey, caching, repeat); @@ -926,5 +927,4 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { assert victimHandler == null; victimHandler = handler; } - -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java index c29edbb..090bfa3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketAllocator.java @@ -34,7 +34,7 @@ import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache.BucketEntry; /** * This class is used to allocate a block with specified size and free the block * when evicting. It manages an array of buckets, each bucket is associated with - * a size and caches elements up to this size. For completely empty bucket, this + * a size and caches elements up to this size. For a completely empty bucket, this * size could be re-specified dynamically. * * This class is not thread safe. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 33ef789..a69b7af 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -59,7 +59,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer; import org.apache.hadoop.hbase.io.hfile.CacheableDeserializerIdManager; import org.apache.hadoop.hbase.io.hfile.CombinedBlockCache; import org.apache.hadoop.hbase.io.hfile.HFileBlock; -import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.util.ConcurrentIndex; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.hbase.util.HasThread; @@ -70,21 +69,21 @@ import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ThreadFactoryBuilder; /** - * BucketCache uses {@link BucketAllocator} to allocate/free block, and use + * BucketCache uses {@link BucketAllocator} to allocate/free blocks, and uses * {@link BucketCache#ramCache} and {@link BucketCache#backingMap} in order to - * determine whether a given element hit. It could uses memory - * {@link ByteBufferIOEngine} or file {@link FileIOEngine}to store/read the - * block data. + * determine if a given element is in the cache. The bucket cache can use on-heap or + * off-heap memory {@link ByteBufferIOEngine} or in a file {@link FileIOEngine} to + * store/read the block data. * - * Eviction is using similar algorithm as + *

Eviction is via a similar algorithm as used in * {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache} * - * BucketCache could be used as mainly a block cache(see - * {@link CombinedBlockCache}), combined with LruBlockCache to decrease CMS and - * fragment by GC. + *

BucketCache can be used as mainly a block cache (see + * {@link CombinedBlockCache}), combined with LruBlockCache to decrease CMS GC and + * heap fragmentation. * - * Also could be used as a secondary cache(e.g. using Fusionio to store block) - * to enlarge cache space by + *

It also can be used as a secondary cache (e.g. using a file on ssd/fusionio to store + * blocks) to enlarge cache space via * {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache#setVictimCache} */ @InterfaceAudience.Private @@ -110,9 +109,9 @@ public class BucketCache implements BlockCache, HeapSize { IOEngine ioEngine; // Store the block in this map before writing it to cache - private ConcurrentHashMap ramCache; + private Map ramCache; // In this map, store the block's meta data like offset, length - private ConcurrentHashMap backingMap; + private Map backingMap; /** * Flag if the cache is enabled or not... We shut it off if there are IO @@ -125,8 +124,6 @@ public class BucketCache implements BlockCache, HeapSize { new ArrayList>(); WriterThread writerThreads[]; - - /** Volatile boolean to track if free space is in process or not */ private volatile boolean freeInProgress = false; private Lock freeSpaceLock = new ReentrantLock(); @@ -196,7 +193,7 @@ public class BucketCache implements BlockCache, HeapSize { // Allocate or free space for the block private BucketAllocator bucketAllocator; - + public BucketCache(String ioEngineName, long capacity, int blockSize, int writerThreadNum, int writerQLen, String persistencePath) throws FileNotFoundException, IOException { @@ -252,7 +249,10 @@ public class BucketCache implements BlockCache, HeapSize { // Run the statistics thread periodically to print the cache statistics log this.scheduleThreadPool.scheduleAtFixedRate(new StatisticsThread(this), statThreadPeriod, statThreadPeriod, TimeUnit.SECONDS); - LOG.info("Started bucket cache"); + LOG.info("Started bucket cache; capacity=" + StringUtils.byteDesc(capacity) + + ", blockSize=" + StringUtils.byteDesc(blockSize) + ", writerThreadNum=" + + writerThreadNum + ", writerQLen=" + writerQLen + ", persistencePath=" + + persistencePath); } /** @@ -359,7 +359,7 @@ public class BucketCache implements BlockCache, HeapSize { return re.getData(); } BucketEntry bucketEntry = backingMap.get(key); - if(bucketEntry!=null) { + if (bucketEntry != null) { long start = System.nanoTime(); IdLock.Entry lockEntry = null; try { @@ -391,7 +391,7 @@ public class BucketCache implements BlockCache, HeapSize { } } } - if(!repeat)cacheStats.miss(caching); + if(!repeat) cacheStats.miss(caching); return null; } @@ -430,9 +430,9 @@ public class BucketCache implements BlockCache, HeapSize { cacheStats.evicted(); return true; } - + /* - * Statistics thread. Periodically prints the cache statistics to the log. + * Statistics thread. Periodically output cache statistics to the log. */ private static class StatisticsThread extends Thread { BucketCache bucketCache; @@ -481,10 +481,6 @@ public class BucketCache implements BlockCache, HeapSize { return (long) Math.floor(bucketAllocator.getTotalSize() * DEFAULT_ACCEPT_FACTOR); } - private long minSize() { - return (long) Math.floor(bucketAllocator.getTotalSize() * DEFAULT_MIN_FACTOR); - } - private long singleSize() { return (long) Math.floor(bucketAllocator.getTotalSize() * DEFAULT_SINGLE_FACTOR * DEFAULT_MIN_FACTOR); @@ -1193,5 +1189,4 @@ public class BucketCache implements BlockCache, HeapSize { writerThread.join(); } } - -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/ByteBufferIOEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/ByteBufferIOEngine.java index 50f93be..2c3e946 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/ByteBufferIOEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/ByteBufferIOEngine.java @@ -25,12 +25,11 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.hbase.util.ByteBufferArray; /** - * IO engine that stores data on the memory using an array of ByteBuffers + * IO engine that stores data in memory using an array of ByteBuffers * {@link ByteBufferArray} */ @InterfaceAudience.Private public class ByteBufferIOEngine implements IOEngine { - private ByteBufferArray bufferArray; /** @@ -88,7 +87,7 @@ public class ByteBufferIOEngine implements IOEngine { */ @Override public void sync() { - + // Nothing to do. } /** @@ -96,6 +95,6 @@ public class ByteBufferIOEngine implements IOEngine { */ @Override public void shutdown() { - + // Nothing to do. } -} +} \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java index a1eea0b..8995a51 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/FileIOEngine.java @@ -53,6 +53,8 @@ public class FileIOEngine implements IOEngine { + StringUtils.byteDesc(fileSize), ioex); if (raf != null) raf.close(); throw ioex; + } finally { + if (raf != null) raf.close(); } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/IOEngine.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/IOEngine.java index 09313ab..ea1636a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/IOEngine.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/IOEngine.java @@ -24,12 +24,11 @@ import java.nio.ByteBuffer; import org.apache.hadoop.classification.InterfaceAudience; /** - * A class implementing IOEngine interface could support data services for + * A class implementing IOEngine interface supports data services for * {@link BucketCache}. */ @InterfaceAudience.Private public interface IOEngine { - /** * @return true if persistent storage is supported for the cache when shutdown */ @@ -63,4 +62,4 @@ public interface IOEngine { * Shutdown the IOEngine */ void shutdown(); -} +} \ No newline at end of file