diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index d119c85..f98bd15 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -113,13 +113,6 @@ public class CacheConfig { public static final String BUCKET_CACHE_COMBINED_KEY = "hbase.bucketcache.combinedcache.enabled"; - /** - * A float which designates how much of the overall cache to give to bucket cache - * and how much to on-heap lru cache when {@link #BUCKET_CACHE_COMBINED_KEY} is set. - */ - public static final String BUCKET_CACHE_COMBINED_PERCENTAGE_KEY = - "hbase.bucketcache.percentage.in.combinedcache"; - public static final String BUCKET_CACHE_WRITER_THREADS_KEY = "hbase.bucketcache.writer.threads"; public static final String BUCKET_CACHE_WRITER_QUEUE_KEY = "hbase.bucketcache.writer.queuelength"; @@ -451,7 +444,7 @@ public class CacheConfig { @VisibleForTesting static boolean blockCacheDisabled = false; - private static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) { + static long getLruCacheSize(final Configuration conf, final MemoryUsage mu) { float cachePercentage = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT); if (cachePercentage <= 0.0001f) { @@ -495,8 +488,6 @@ public class CacheConfig { int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY, DEFAULT_BUCKET_CACHE_WRITER_QUEUE); String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY); - float combinedPercentage = conf.getFloat(BUCKET_CACHE_COMBINED_PERCENTAGE_KEY, - DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE); String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY); int [] bucketSizes = null; if (configuredBucketSizes != null) { @@ -505,10 +496,6 @@ public class CacheConfig { bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i]); } } - if (combinedWithLru) { - lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize); - bucketCacheSize = (long) (combinedPercentage * bucketCacheSize); - } LOG.info("Allocating LruBlockCache size=" + StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize)); LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize, true, conf); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 5595439..ad533bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -326,7 +326,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory, final boolean cacheDataInL1) { LruCachedBlock cb = map.get(cacheKey); - if(cb != null) { + if (cb != null) { // compare the contents, if they are not equal, we are in big trouble if (compare(buf, cb.getBuffer()) != 0) { throw new RuntimeException("Cached block contents differ, which should not have happened." @@ -341,7 +341,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { long newSize = updateSizeMetrics(cb, false); map.put(cacheKey, cb); elements.incrementAndGet(); - if(newSize > acceptableSize() && !evictionInProgress) { + if (newSize > acceptableSize() && !evictionInProgress) { runEviction(); } } @@ -893,7 +893,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { // Simple calculators of sizes given factors and maxSize - private long acceptableSize() { + long acceptableSize() { return (long)Math.floor(this.maxSize * this.acceptableFactor); } private long minSize() { @@ -934,7 +934,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { /** Clears the cache. Used in tests. */ public void clearCache() { - map.clear(); + this.map.clear(); + this.elements.set(0); } /** @@ -978,6 +979,10 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { victimHandler = handler; } + BucketCache getVictimHandler() { + return this.victimHandler; + } + @Override public BlockCache[] getBlockCaches() { return null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java index 362181c..f298698 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/package-info.java @@ -24,59 +24,21 @@ * include the default, native on-heap {@link org.apache.hadoop.hbase.io.hfile.LruBlockCache} and a * {@link org.apache.hadoop.hbase.io.hfile.bucket.BucketCache} that has a bunch of deploy formats * including acting as a L2 for LruBlockCache -- when a block is evicted from LruBlockCache, it - * goes to the BucketCache and when we search a block, we look in both places -- or + * goes to the BucketCache and when we search a block, we look in both places -- or, the + * most common deploy type, * using {@link org.apache.hadoop.hbase.io.hfile.CombinedBlockCache}, BucketCache is used as * a host for data blocks with meta blocks in an instance of LruBlockCache. BucketCache * can also be onheap, offheap, and file-backed. * *

Which BlockCache should I use?

- * By default LruBlockCache is on. If you would like to cache more, and offheap, try enabling - * BucketCache. Fetching will always + * By default LruBlockCache is on. If you would like to cache more, and offheap (offheap + * usually means less GC headache), try enabling * BucketCache. Fetching will always * be slower when fetching from BucketCache but latencies tend to be less erratic over time * (roughly because GC is less). See Nick Dimiduk's * BlockCache 101 for some numbers. * *

Enabling {@link org.apache.hadoop.hbase.io.hfile.bucket.BucketCache}

- * Read the options and defaults for BucketCache in the head of the - * {@link org.apache.hadoop.hbase.io.hfile.CacheConfig}. - * - *

Here is a simple example of how to enable a 4G offheap bucket cache with 1G - * onheap cache managed by {@link org.apache.hadoop.hbase.io.hfile.CombinedBlockCache}. - * CombinedBlockCache will put DATA blocks in the BucketCache and META blocks -- INDEX and BLOOMS - * -- in an instance of the LruBlockCache. For the - * CombinedBlockCache (from the class comment), "[t]he smaller lruCache is used - * to cache bloom blocks and index blocks, the larger bucketCache is used to - * cache data blocks. getBlock reads first from the smaller lruCache before - * looking for the block in the bucketCache. Metrics are the combined size and - * hits and misses of both caches." To disable CombinedBlockCache and have the BucketCache act - * as a strict L2 cache to the L1 LruBlockCache (i.e. on eviction from L1, blocks go to L2), set - * {@link org.apache.hadoop.hbase.io.hfile.CacheConfig#BUCKET_CACHE_COMBINED_KEY} to false. By - * default, hbase.bucketcache.combinedcache.enabled (BUCKET_CACHE_COMBINED_KEY) is true. - * - *

Back to the example of setting an onheap cache of 1G and offheap of 4G with the BlockCache - * deploy managed by CombinedBlockCache. Setting hbase.bucketcache.ioengine and - * hbase.bucketcache.size > 0 enables CombinedBlockCache. - * In hbase-env.sh ensure the environment - * variable -XX:MaxDirectMemorySize is enabled and is bigger than 4G, say 5G in size: - * e.g. -XX:MaxDirectMemorySize=5G. This setting allows the JVM use offheap memory - * up to this upper limit. Allocate more than you need because there are other consumers of - * offheap memory other than BlockCache (for example DFSClient in the RegionServer uses offheap). - * In hbase-site.xml add the following configurations: -

<property>
-  <name>hbase.bucketcache.ioengine</name>
-  <value>offheap</value>
-</property>
-<property>
-  <name>hbase.bucketcache.percentage.in.combinedcache</name>
-  <value>0.8</value>
-</property>
-<property>
-  <name>hbase.bucketcache.size</name>
-  <value>5120</value>
-</property>
. Above we set a cache of 5G, 80% of which will be offheap (4G) and 1G onheap - * (with DATA blocks in BucketCache and INDEX blocks in the onheap LruBlockCache). - * Restart (or rolling restart) your cluster for the configs to take effect. Check logs to ensure - * your configurations came out as expected. + * See the HBase Reference Guide Enable BucketCache. * */ package org.apache.hadoop.hbase.io.hfile; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java index dfba277..560d7d0 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java @@ -22,6 +22,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.MemoryUsage; import java.nio.ByteBuffer; import java.util.Map; @@ -29,8 +31,10 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.SmallTests; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; +import org.apache.hadoop.hbase.util.Threads; import org.junit.After; import org.junit.Before; import org.junit.Test; @@ -198,11 +202,12 @@ public class TestCacheConfig { assertTrue(cc.getBlockCache() instanceof LruBlockCache); } + /** + * Assert that the caches are deployed with CombinedBlockCache and of the appropriate sizes. + */ @Test public void testBucketCacheConfig() { this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap"); - final float percent = 0.8f; - this.conf.setFloat(CacheConfig.BUCKET_CACHE_COMBINED_PERCENTAGE_KEY, percent); final int bcSize = 100; this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, bcSize); CacheConfig cc = new CacheConfig(this.conf); @@ -213,15 +218,66 @@ public class TestCacheConfig { BlockCache [] bcs = cbc.getBlockCaches(); assertTrue(bcs[0] instanceof LruBlockCache); LruBlockCache lbc = (LruBlockCache)bcs[0]; - long expectedBCSize = (long)(bcSize * (1.0f - percent)); - long actualBCSize = lbc.getMaxSize() / (1024 * 1024); - assertTrue(expectedBCSize == actualBCSize); + assertEquals(CacheConfig.getLruCacheSize(this.conf, + ManagementFactory.getMemoryMXBean().getHeapMemoryUsage()), lbc.getMaxSize()); assertTrue(bcs[1] instanceof BucketCache); BucketCache bc = (BucketCache)bcs[1]; // getMaxSize comes back in bytes but we specified size in MB - expectedBCSize = (long)(bcSize * percent); - actualBCSize = (long)(bc.getMaxSize() / (1024 * 1024)); - assertTrue(expectedBCSize == actualBCSize); + assertEquals(bcSize, bc.getMaxSize() / (1024 * 1024)); + } + + /** + * Assert that when BUCKET_CACHE_COMBINED_KEY is false, the non-default, that we deploy + * LruBlockCache as L1 with a BucketCache for L2. + */ + @Test (timeout=10000) + public void testBucketCacheConfigL1L2Setup() { + this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap"); + // Make lru size is smaller than bcSize for sure. Need this to be true so when eviction + // from L1 happens, it does not fail because L2 can't take the eviction because block too big. + this.conf.setFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, 0.001f); + MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); + long lruExpectedSize = CacheConfig.getLruCacheSize(this.conf, mu); + final int bcSize = 100; + long bcExpectedSize = 100 * 1024 * 1024; // MB. + assertTrue(lruExpectedSize < bcExpectedSize); + this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, bcSize); + this.conf.setBoolean(CacheConfig.BUCKET_CACHE_COMBINED_KEY, false); + CacheConfig cc = new CacheConfig(this.conf); + basicBlockCacheOps(cc, false, false); + assertTrue(cc.getBlockCache() instanceof LruBlockCache); + // TODO: Assert sizes allocated are right and proportions. + LruBlockCache lbc = (LruBlockCache)cc.getBlockCache(); + assertEquals(lruExpectedSize, lbc.getMaxSize()); + BucketCache bc = lbc.getVictimHandler(); + // getMaxSize comes back in bytes but we specified size in MB + assertEquals(bcExpectedSize, bc.getMaxSize()); + // Test the L1+L2 deploy works as we'd expect with blocks evicted from L1 going to L2. + long initialL1BlockCount = lbc.getBlockCount(); + long initialL2BlockCount = bc.getBlockCount(); + Cacheable c = new DataCacheEntry(); + BlockCacheKey bck = new BlockCacheKey("bck", 0); + lbc.cacheBlock(bck, c, false, false); + assertEquals(initialL1BlockCount + 1, lbc.getBlockCount()); + assertEquals(initialL2BlockCount, bc.getBlockCount()); + // Force evictions by putting in a block too big. + final long justTooBigSize = lbc.acceptableSize() + 1; + lbc.cacheBlock(new BlockCacheKey("bck2", 0), new DataCacheEntry() { + @Override + public long heapSize() { + return justTooBigSize; + } + + @Override + public int getSerializedLength() { + return (int)heapSize(); + } + }); + // The eviction thread in lrublockcache needs to run. + while (initialL1BlockCount != lbc.getBlockCount()) Threads.sleep(10); + assertEquals(initialL1BlockCount, lbc.getBlockCount()); + long count = bc.getBlockCount(); + assertTrue(initialL2BlockCount + 1 <= count); } /** diff --git a/src/main/docbkx/book.xml b/src/main/docbkx/book.xml index 8c92f67..123b287 100644 --- a/src/main/docbkx/book.xml +++ b/src/main/docbkx/book.xml @@ -2159,7 +2159,7 @@ rs.close(); Restart (or rolling restart) your cluster for the configurations to take effect. Check logs for errors or unexpected behavior. -
+
Enable BucketCache The usual deploy of BucketCache is via a managing class that sets up two caching tiers: an L1 onheap cache @@ -2172,8 +2172,7 @@ rs.close(); setting cacheDataInL1 via (HColumnDescriptor.setCacheDataInL1(true) or in the shell, creating or amending column families setting CACHE_DATA_IN_L1 to true: e.g. hbase(main):003:0> create 't', {NAME => 't', CONFIGURATION => {CACHE_DATA_IN_L1 => 'true'}} - The BucketCache deploy can be - onheap, offheap, or file based. You set which via the + The BucketCache deploy can be onheap, offheap, or file based. You set which via the hbase.bucketcache.ioengine setting it to heap for BucketCache running as part of the java heap, offheap for BucketCache to make allocations offheap, @@ -2191,6 +2190,8 @@ rs.close(); onheap cache. Configuration is performed on the RegionServer. Setting hbase.bucketcache.ioengine and hbase.bucketcache.size > 0 enables CombinedBlockCache. + Let us presume that the RegionServer has been set to run with a 5G heap: + i.e. HBASE_HEAPSIZE=5g. First, edit the RegionServer's hbase-env.sh and set @@ -2210,12 +2211,12 @@ rs.close(); offheap - hbase.bucketcache.percentage.in.combinedcache - 0.8 + hfile.block.cache.size + 0.2 hbase.bucketcache.size - 5120 + 4196 ]]> @@ -2224,6 +2225,11 @@ rs.close(); issues. + In the above, we set bucketcache to be 4G. The onheap lrublockcache we + configured to have 0.2 of the RegionServer's heap size (0.2 * 5G = 1G). + In other words, you configure the L1 LruBlockCache as you would normally, + as you would when there is no L2 BucketCache present. +