From 5b9cde87bc35b08db1327de7345d7d1ce08144e4 Mon Sep 17 00:00:00 2001 From: Elliott Clark Date: Tue, 10 Nov 2015 12:21:32 -0800 Subject: [PATCH] HBASE-14793 Allow limiting size of block into L1 block cache. --- .../hadoop/hbase/io/hfile/LruBlockCache.java | 17 ++++++-- .../hadoop/hbase/io/hfile/TestLruBlockCache.java | 49 ++++++++++++++++++++-- 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 04983f6..dadd79a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -140,12 +140,15 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { /** Statistics thread */ static final int statThreadPeriod = 60 * 5; + private static final String LRU_MAX_BLOCK_SIZE = "hbase.lru.max.block.size"; + private static final long DEFAULT_MAX_BLOCK_SIZE = 16L * 1024L * 1024L; /** Concurrent map (the cache) */ private final Map map; /** Eviction lock (locked when eviction in process) */ private final ReentrantLock evictionLock = new ReentrantLock(true); + private final long maxBlockSize; /** Volatile boolean to track if we are in an eviction process or not */ private volatile boolean evictionInProgress = false; @@ -223,7 +226,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { DEFAULT_SINGLE_FACTOR, DEFAULT_MULTI_FACTOR, DEFAULT_MEMORY_FACTOR, - false + false, + DEFAULT_MAX_BLOCK_SIZE ); } @@ -237,7 +241,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { conf.getFloat(LRU_SINGLE_PERCENTAGE_CONFIG_NAME, DEFAULT_SINGLE_FACTOR), conf.getFloat(LRU_MULTI_PERCENTAGE_CONFIG_NAME, DEFAULT_MULTI_FACTOR), conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, DEFAULT_MEMORY_FACTOR), - conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, DEFAULT_IN_MEMORY_FORCE_MODE) + conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, DEFAULT_IN_MEMORY_FORCE_MODE), + conf.getLong(LRU_MAX_BLOCK_SIZE, DEFAULT_MAX_BLOCK_SIZE) ); } @@ -262,7 +267,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel, float minFactor, float acceptableFactor, float singleFactor, - float multiFactor, float memoryFactor, boolean forceInMemory) { + float multiFactor, float memoryFactor, boolean forceInMemory, long maxBlockSize) { + this.maxBlockSize = maxBlockSize; if(singleFactor + multiFactor + memoryFactor != 1 || singleFactor < 0 || multiFactor < 0 || memoryFactor < 0) { throw new IllegalArgumentException("Single, multi, and memory factors " + @@ -324,6 +330,11 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { @Override public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory, final boolean cacheDataInL1) { + + if (buf.heapSize() > 12 * 1024 * 1024) { + LOG.warn("Trying to cache too large a "); + } + LruCachedBlock cb = map.get(cacheKey); if (cb != null) { // compare the contents, if they are not equal, we are in big trouble diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java index 9a548f5..0dfbe30 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java @@ -19,6 +19,8 @@ package org.apache.hadoop.hbase.io.hfile; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import java.nio.ByteBuffer; @@ -266,7 +268,8 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory - false); + false, + 16 * 1024 * 1024); CachedItem [] singleBlocks = generateFixedBlocks(5, blockSize, "single"); CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi"); @@ -386,7 +389,8 @@ public class TestLruBlockCache { 0.2f, // single 0.3f, // multi 0.5f, // memory - true); + true, + 16 * 1024 * 1024); CachedItem [] singleBlocks = generateFixedBlocks(10, blockSize, "single"); CachedItem [] multiBlocks = generateFixedBlocks(10, blockSize, "multi"); @@ -491,7 +495,8 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory - false); + false, + 16 * 1024 * 1024); CachedItem [] singleBlocks = generateFixedBlocks(20, blockSize, "single"); CachedItem [] multiBlocks = generateFixedBlocks(5, blockSize, "multi"); @@ -539,6 +544,41 @@ public class TestLruBlockCache { } + @Test + public void testMaxBlockSize() throws Exception { + long maxSize = 100000; + long blockSize = calculateBlockSize(maxSize, 10); + + LruBlockCache cache = new LruBlockCache(maxSize, blockSize, false, + (int)Math.ceil(1.2*maxSize/blockSize), + LruBlockCache.DEFAULT_LOAD_FACTOR, + LruBlockCache.DEFAULT_CONCURRENCY_LEVEL, + 0.66f, // min + 0.99f, // acceptable + 0.33f, // single + 0.33f, // multi + 0.34f, // memory + false, + 1024); + CachedItem [] tooLong = generateFixedBlocks(10, 1024+5, "long"); + CachedItem [] small = generateFixedBlocks(15, 600, "long"); + + + for (CachedItem i:tooLong) { + cache.cacheBlock(i.cacheKey, i); + } + for (CachedItem i:small) { + cache.cacheBlock(i.cacheKey, i); + } + assertEquals(15,cache.getBlockCount()); + for (CachedItem i:small) { + assertNotNull(cache.getBlock(i.cacheKey, true, false, false)); + } + for (CachedItem i:tooLong) { + assertNull(cache.getBlock(i.cacheKey, true, false, false)); + } + } + // test setMaxSize @Test public void testResizeBlockCache() throws Exception { @@ -555,7 +595,8 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory - false); + false, + 16 * 1024 * 1024); CachedItem [] singleBlocks = generateFixedBlocks(10, blockSize, "single"); CachedItem [] multiBlocks = generateFixedBlocks(10, blockSize, "multi"); -- 2.6.1