diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 4a0978a..caa26c3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -112,6 +112,10 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { */ static final String LRU_ACCEPTABLE_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.acceptable.factor"; + /** + * Hard capacity limit of cache, will reject any put if size > this * acceptable + */ + static final String LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME = "hbase.lru.blockcache.hard.capacity.limit.factor"; static final String LRU_SINGLE_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.single.percentage"; static final String LRU_MULTI_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.multi.percentage"; static final String LRU_MEMORY_PERCENTAGE_CONFIG_NAME = "hbase.lru.blockcache.memory.percentage"; @@ -140,6 +144,9 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { static final boolean DEFAULT_IN_MEMORY_FORCE_MODE = false; + /** default hard capacity limit */ + static final float DEFAULT_HARD_CAPACITY_LIMIT_FACTOR = 1.2f; + /** Statistics thread */ static final int statThreadPeriod = 60 * 5; @@ -168,6 +175,9 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { /** Cache access count (sequential ID) */ private final AtomicLong count; + /** hard capacity limit */ + private float hardCapacityLimitFactor; + /** Cache statistics */ private final CacheStats stats; @@ -225,6 +235,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { DEFAULT_SINGLE_FACTOR, DEFAULT_MULTI_FACTOR, DEFAULT_MEMORY_FACTOR, + DEFAULT_HARD_CAPACITY_LIMIT_FACTOR, false ); } @@ -239,6 +250,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { conf.getFloat(LRU_SINGLE_PERCENTAGE_CONFIG_NAME, DEFAULT_SINGLE_FACTOR), conf.getFloat(LRU_MULTI_PERCENTAGE_CONFIG_NAME, DEFAULT_MULTI_FACTOR), conf.getFloat(LRU_MEMORY_PERCENTAGE_CONFIG_NAME, DEFAULT_MEMORY_FACTOR), + conf.getFloat(LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, DEFAULT_HARD_CAPACITY_LIMIT_FACTOR), conf.getBoolean(LRU_IN_MEMORY_FORCE_MODE_CONFIG_NAME, DEFAULT_IN_MEMORY_FORCE_MODE) ); } @@ -264,7 +276,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { public LruBlockCache(long maxSize, long blockSize, boolean evictionThread, int mapInitialSize, float mapLoadFactor, int mapConcurrencyLevel, float minFactor, float acceptableFactor, float singleFactor, - float multiFactor, float memoryFactor, boolean forceInMemory) { + float multiFactor, float memoryFactor, float hardLimitFactor, boolean forceInMemory) { if(singleFactor + multiFactor + memoryFactor != 1 || singleFactor < 0 || multiFactor < 0 || memoryFactor < 0) { throw new IllegalArgumentException("Single, multi, and memory factors " + @@ -291,6 +303,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { this.elements = new AtomicLong(0); this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel); this.size = new AtomicLong(this.overhead); + this.hardCapacityLimitFactor = hardLimitFactor; if(evictionThread) { this.evictionThread = new EvictionThread(this); this.evictionThread.start(); // FindBugs SC_START_IN_CTOR @@ -338,6 +351,20 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { LOG.warn(msg); return; } + long currentSize = size.get(); + long hardLimitSize = (long) (hardCapacityLimitFactor * acceptableSize()); + if (currentSize > hardLimitSize) { + if (LOG.isTraceEnabled()) { + LOG.trace("LruBlockCache current size " + StringUtils.byteDesc(currentSize) + + " has exceeded acceptable size " + StringUtils.byteDesc(acceptableSize()) + " too many." + + " the hard limit size is " + StringUtils.byteDesc(hardLimitSize) + ", failed to put cacheKey:" + + cacheKey + " into LruBlockCache."); + } + if (!evictionInProgress) { + runEviction(); + } + return; + } cb = new LruCachedBlock(cacheKey, buf, count.incrementAndGet(), inMemory); long newSize = updateSizeMetrics(cb, false); map.put(cacheKey, cb); @@ -911,7 +938,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { public final static long CACHE_FIXED_OVERHEAD = ClassSize.align( (3 * Bytes.SIZEOF_LONG) + (9 * ClassSize.REFERENCE) + - (5 * Bytes.SIZEOF_FLOAT) + Bytes.SIZEOF_BOOLEAN + (6 * Bytes.SIZEOF_FLOAT) + Bytes.SIZEOF_BOOLEAN + ClassSize.OBJECT); @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java index d2bfa7e..6ce81dc 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheOnWrite.java @@ -180,6 +180,8 @@ public class TestCacheOnWrite { // default blockcaches.add(new CacheConfig(conf).getBlockCache()); + //set LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME to 2.0f due to HBASE-16287 + TEST_UTIL.getConfiguration().setFloat(LruBlockCache.LRU_HARD_CAPACITY_LIMIT_FACTOR_CONFIG_NAME, 2.0f); // memory BlockCache lru = new LruBlockCache(128 * 1024 * 1024, 64 * 1024, TEST_UTIL.getConfiguration()); blockcaches.add(lru); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java index ec8d31d..43479b2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java @@ -265,6 +265,7 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory + 1.2f, // limit false); CachedItem [] singleBlocks = generateFixedBlocks(5, blockSize, "single"); @@ -385,6 +386,7 @@ public class TestLruBlockCache { 0.2f, // single 0.3f, // multi 0.5f, // memory + 1.2f, // limit true); CachedItem [] singleBlocks = generateFixedBlocks(10, blockSize, "single"); @@ -490,6 +492,7 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory + 1.2f, // limit false); CachedItem [] singleBlocks = generateFixedBlocks(20, blockSize, "single"); @@ -554,6 +557,7 @@ public class TestLruBlockCache { 0.33f, // single 0.33f, // multi 0.34f, // memory + 1.2f, // limit false); CachedItem [] singleBlocks = generateFixedBlocks(10, blockSize, "single");