diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 292afe8..59fd2c0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -29,6 +29,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; +import org.apache.hadoop.hbase.io.hfile.slab.SlabCache; import org.apache.hadoop.hbase.util.DirectMemoryUtils; import org.apache.hadoop.util.StringUtils; @@ -76,14 +77,49 @@ public class CacheConfig { /** * Configuration keys for Bucket cache */ + + /** + * Current ioengine options in include: heap, offheap and file:PATH (where PATH is the path + * to the file that will host the file-based cache. See BucketCache#getIOEngineFromName() for + * list of supported ioengine options. + * + *
Set this option and a non-zero {@link BUCKET_CACHE_SIZE_KEY} to enable bucket cache. + */ public static final String BUCKET_CACHE_IOENGINE_KEY = "hbase.bucketcache.ioengine"; + + /** + * When using bucket cache, this is a float that EITHER represents a percentage of total heap + * memory size to give to the cache (if < 1.0) OR, it is the capacity in megabytes of the cache. + * + *
The resultant size is further divided if {@link BUCKET_CACHE_COMBINED_KEY} is set (It is
+ * set by default. When false, bucket cache serves as an "L2" cache to the "L1"
+ * {@link LruBlockCache}). The percentage is set in
+ * with {@link BUCKET_CACHE_COMBINED_PERCENTAGE_KEY} float.
+ */
public static final String BUCKET_CACHE_SIZE_KEY = "hbase.bucketcache.size";
+
+ /**
+ * If the chosen ioengine can persist its state across restarts, the path to the file to
+ * persist to.
+ */
public static final String BUCKET_CACHE_PERSISTENT_PATH_KEY =
"hbase.bucketcache.persistent.path";
+
+ /**
+ * If the bucket cache is used in league with the lru on-heap block cache (meta blocks such
+ * as indices and blooms are kept in the lru blockcache and the data blocks in the
+ * bucket cache).
+ */
public static final String BUCKET_CACHE_COMBINED_KEY =
"hbase.bucketcache.combinedcache.enabled";
+
+ /**
+ * A float which designates how much of the overall cache to give to bucket cache
+ * and how much to on-heap lru cache when {@link BUCKET_CACHE_COMBINED_KEY} is set.
+ */
public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY =
"hbase.bucketcache.percentage.in.combinedcache";
+
public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads";
public static final String BUCKET_CACHE_WRITER_QUEUE_KEY =
"hbase.bucketcache.writer.queuelength";
@@ -95,6 +131,19 @@ public class CacheConfig {
public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64;
public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f;
+ /**
+ * Setting this float to a non-null value turns on {@link DoubleBlockCache}
+ * which makes use of the {@link LruBlockCache} and {@link SlabCache}.
+ *
+ * The float value of between 0 and 1 will be multiplied against the setting for
+ * Eviction is via a similar algorithm as used in
* {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache}
*
- * BucketCache could be used as mainly a block cache(see
- * {@link CombinedBlockCache}), combined with LruBlockCache to decrease CMS and
- * fragment by GC.
+ * BucketCache can be used as mainly a block cache (see
+ * {@link CombinedBlockCache}), combined with LruBlockCache to decrease CMS GC and
+ * heap fragmentation.
*
- * Also could be used as a secondary cache(e.g. using Fusionio to store block)
- * to enlarge cache space by
+ * It also can be used as a secondary cache (e.g. using a file on ssd/fusionio to store
+ * blocks) to enlarge cache space via
* {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache#setVictimCache}
*/
@InterfaceAudience.Private
@@ -110,9 +109,9 @@ public class BucketCache implements BlockCache, HeapSize {
IOEngine ioEngine;
// Store the block in this map before writing it to cache
- private ConcurrentHashMap-XX:MaxDirectMemorySize to figure what size of the offheap allocation to give
+ * over to slab cache.
+ *
+ * Slab cache has been little used and is likely to be deprecated in the near future.
+ */
+ public static final String SLAB_CACHE_OFFHEAP_PERCENTAGE_KEY =
+ "hbase.offheapcache.percentage";
+
// Defaults
public static final boolean DEFAULT_CACHE_DATA_ON_READ = true;
@@ -404,7 +453,7 @@ public class CacheConfig {
}
}
LOG.info("Allocating LruBlockCache with maximum size " +
- StringUtils.humanReadableInt(lruCacheSize));
+ StringUtils.humanReadableInt(lruCacheSize) + ", blockSize=" + blockSize);
LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize);
lruCache.setVictimCache(bucketCache);
if (bucketCache != null && combinedWithLru) {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java
index 5145199..903e6d7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/Cacheable.java
@@ -37,15 +37,15 @@ import org.apache.hadoop.hbase.io.HeapSize;
public interface Cacheable extends HeapSize {
/**
* Returns the length of the ByteBuffer required to serialized the object. If the
- * object cannot be serialized, it should also return 0.
+ * object cannot be serialized, it should return 0.
*
- * @return int length in bytes of the serialized form.
+ * @return int length in bytes of the serialized form or 0 if the object cannot be cached.
*/
-
int getSerializedLength();
/**
* Serializes its data into destination.
+ * @param destination Where to serialize to
*/
void serialize(ByteBuffer destination);
@@ -60,5 +60,4 @@ public interface Cacheable extends HeapSize {
* @return the block type of this cached HFile block
*/
BlockType getBlockType();
-
-}
+}
\ No newline at end of file
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
index 2aae578..37221df 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
@@ -30,15 +30,14 @@ import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache;
/**
* CombinedBlockCache is an abstraction layer that combines
* {@link LruBlockCache} and {@link BucketCache}. The smaller lruCache is used
- * to cache bloom blocks and index blocks , the larger bucketCache is used to
- * cache data blocks. getBlock reads first from the smaller lruCache before
- * looking for the block in the bucketCache. Metrics are the combined size and
- * hits and misses of both caches.
+ * to cache bloom blocks and index blocks. The larger bucketCache is used to
+ * cache data blocks. {@link #getBlock(BlockCacheKey, boolean, boolean) reads
+ * first from the smaller lruCache before looking for the block in the bucketCache.
+ * Metrics are the combined size and hits and misses of both caches.
*
- **/
+ */
@InterfaceAudience.Private
public class CombinedBlockCache implements BlockCache, HeapSize {
-
private final LruBlockCache lruCache;
private final BucketCache bucketCache;
private final CombinedCacheStats combinedCacheStats;
@@ -73,6 +72,8 @@ public class CombinedBlockCache implements BlockCache, HeapSize {
@Override
public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching,
boolean repeat) {
+ // TODO: is there a hole here, or just awkwardness since in the lruCache getBlock
+ // we end up calling bucketCache.getBlock.
if (lruCache.containsBlock(cacheKey)) {
return lruCache.getBlock(cacheKey, caching, repeat);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 045f3bf..d5d9ad1 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -132,7 +132,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
static final int statThreadPeriod = 60 * 5;
/** Concurrent map (the cache) */
- private final ConcurrentHashMap