diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java index 5e97b80..7bafe04 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java @@ -45,6 +45,7 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner { public static final String STEP_KEY = "hbase.regionserver.heapmemory.autotuner.step"; public static final float DEFAULT_STEP_VALUE = 0.02f; // 2% + public static final float tolerance = 0.00f; private static final TunerResult TUNER_RESULT = new TunerResult(true); private static final TunerResult NO_OP_TUNER_RESULT = new TunerResult(false); @@ -57,30 +58,73 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner { private float blockCachePercentMinRange; private float blockCachePercentMaxRange; + private boolean stepDirection; // true if last time tuner increased block cache size + private boolean isFirstTuning = true; + private long prevFlushCount = 0; + private long prevEvictCount = 0; + private long prevCacheMissCount = 0; + + @Override public TunerResult tune(TunerContext context) { long blockedFlushCount = context.getBlockedFlushCount(); long unblockedFlushCount = context.getUnblockedFlushCount(); long evictCount = context.getEvictCount(); - boolean memstoreSufficient = blockedFlushCount == 0 && unblockedFlushCount == 0; - boolean blockCacheSufficient = evictCount == 0; + long cacheMissCount = context.getCacheMissCount(); + //we can consider memstore or block cache to be sufficient if + //we are using less than half of what have been already provided + boolean memstoreSufficient = (blockedFlushCount == 0 && unblockedFlushCount == 0) + || context.getCurMemStoreUsed()*2 < context.getCurMemStoreSize(); + boolean blockCacheSufficient = evictCount == 0 || + context.getCurBlockCacheUsed()*2 < context.getCurBlockCacheSize(); if (memstoreSufficient && blockCacheSufficient) { + prevFlushCount = blockedFlushCount + unblockedFlushCount; + prevEvictCount = evictCount; + prevCacheMissCount = cacheMissCount; return NO_OP_TUNER_RESULT; } float newMemstoreSize; float newBlockCacheSize; if (memstoreSufficient) { // Increase the block cache size and corresponding decrease in memstore size - newBlockCacheSize = context.getCurBlockCacheSize() + step; - newMemstoreSize = context.getCurMemStoreSize() - step; + stepDirection = true; } else if (blockCacheSufficient) { // Increase the memstore size and corresponding decrease in block cache size - newBlockCacheSize = context.getCurBlockCacheSize() - step; - newMemstoreSize = context.getCurMemStoreSize() + step; + stepDirection = false; } else { - return NO_OP_TUNER_RESULT; - // As of now not making any tuning in write/read heavy scenario. + float percentChangeInEvictCount = (float)(evictCount-prevEvictCount)/(float)(evictCount); + float percentChangeInFlushes = (float)(blockedFlushCount + unblockedFlushCount - + prevFlushCount)/(float)(blockedFlushCount + unblockedFlushCount); + float percentChangeInMisses = (float)(cacheMissCount-prevCacheMissCount) + /(float)(cacheMissCount); + if(!isFirstTuning) { + //Negative percent changes are desirable + //if it is positive , we should revert the previous step + if (percentChangeInEvictCount + percentChangeInFlushes > tolerance) { + //revert last step as it was not useful + stepDirection = !stepDirection; + } else { + //last step was useful, taking step based on current stats + if(cacheMissCount == 0) + { + stepDirection = false; + } else { + //try to balance changes in number of cache misses and flushes + stepDirection = percentChangeInMisses > percentChangeInFlushes; + } + } + } else { + stepDirection = percentChangeInMisses > percentChangeInFlushes; } + } + + if (stepDirection){ + newBlockCacheSize = context.getCurBlockCacheSize() + step; + newMemstoreSize = context.getCurMemStoreSize() - step; + } else { + newBlockCacheSize = context.getCurBlockCacheSize() - step; + newMemstoreSize = context.getCurMemStoreSize() + step; + } if (newMemstoreSize > globalMemStorePercentMaxRange) { newMemstoreSize = globalMemStorePercentMaxRange; } else if (newMemstoreSize < globalMemStorePercentMinRange) { @@ -93,6 +137,10 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner { } TUNER_RESULT.setBlockCacheSize(newBlockCacheSize); TUNER_RESULT.setMemstoreSize(newMemstoreSize); + prevFlushCount = blockedFlushCount + unblockedFlushCount; + prevEvictCount = evictCount; + prevCacheMissCount = cacheMissCount; + isFirstTuning = false; return TUNER_RESULT; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java index 5448025..0dae61c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java @@ -75,7 +75,7 @@ public class HeapMemoryManager { private final ResizableBlockCache blockCache; private final FlushRequester memStoreFlusher; - private final Server server; + private final HRegionServer server; private HeapMemoryTunerChore heapMemTunerChore = null; private final boolean tunerOn; @@ -85,7 +85,7 @@ public class HeapMemoryManager { private long maxHeapSize = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); public static HeapMemoryManager create(Configuration conf, FlushRequester memStoreFlusher, - Server server) { + HRegionServer server) { BlockCache blockCache = CacheConfig.instantiateBlockCache(conf); if (blockCache instanceof ResizableBlockCache) { return new HeapMemoryManager((ResizableBlockCache) blockCache, memStoreFlusher, server); @@ -95,7 +95,7 @@ public class HeapMemoryManager { @VisibleForTesting HeapMemoryManager(ResizableBlockCache blockCache, FlushRequester memStoreFlusher, - Server server) { + HRegionServer server) { Configuration conf = server.getConfiguration(); this.blockCache = blockCache; this.memStoreFlusher = memStoreFlusher; @@ -181,6 +181,7 @@ public class HeapMemoryManager { + blockCachePercentMaxRange); } return true; + } public void start(ChoreService service) { @@ -216,7 +217,8 @@ public class HeapMemoryManager { private HeapMemoryTuner heapMemTuner; private AtomicLong blockedFlushCount = new AtomicLong(); private AtomicLong unblockedFlushCount = new AtomicLong(); - private long evictCount = 0L; + private long evictCount = 0; + private long cacheMissCount = 0; private TunerContext tunerContext = new TunerContext(); private boolean alarming = false; @@ -264,11 +266,21 @@ public class HeapMemoryManager { } private void tune() { - long curEvictCount = blockCache.getStats().getEvictedCount(); + //TODO check if we can increase the memory boundaries + //while remaining in the limits + long curEvictCount; + long curCacheMisCount; + curEvictCount = blockCache.getStats().getEvictedCount(); tunerContext.setEvictCount(curEvictCount - evictCount); - evictCount = curEvictCount; + evictCount = curEvictCount; + curCacheMisCount = blockCache.getStats().getMissCachingCount(); + tunerContext.setCacheMissCount(curCacheMisCount-cacheMissCount); + cacheMissCount = curCacheMisCount; tunerContext.setBlockedFlushCount(blockedFlushCount.getAndSet(0)); tunerContext.setUnblockedFlushCount(unblockedFlushCount.getAndSet(0)); + tunerContext.setCurBlockCacheUsed((float)blockCache.getCurrentSize() / maxHeapSize); + tunerContext.setCurMemStoreUsed( + (float)server.getRegionServerAccounting().getGlobalMemstoreSize() / maxHeapSize); tunerContext.setCurBlockCacheSize(blockCachePercent); tunerContext.setCurMemStoreSize(globalMemStorePercent); TunerResult result = null; @@ -321,6 +333,8 @@ public class HeapMemoryManager { globalMemStorePercent = memstoreSize; memStoreFlusher.setGlobalMemstoreLimit(newMemstoreSize); } + } else { + LOG.info("No changes made by HeapMemoryTuner."); } } @@ -349,6 +363,9 @@ public class HeapMemoryManager { private long blockedFlushCount; private long unblockedFlushCount; private long evictCount; + private long cacheMissCount; + private float curBlockCacheUsed; + private float curMemStoreUsed; private float curMemStoreSize; private float curBlockCacheSize; @@ -391,6 +408,30 @@ public class HeapMemoryManager { public void setCurBlockCacheSize(float curBlockCacheSize) { this.curBlockCacheSize = curBlockCacheSize; } + + public long getCacheMissCount() { + return cacheMissCount; + } + + public void setCacheMissCount(long cacheMissCount) { + this.cacheMissCount = cacheMissCount; + } + + public float getCurBlockCacheUsed() { + return curBlockCacheUsed; + } + + public void setCurBlockCacheUsed(float curBlockCacheUsed) { + this.curBlockCacheUsed = curBlockCacheUsed; + } + + public float getCurMemStoreUsed() { + return curMemStoreUsed; + } + + public void setCurMemStoreUsed(float d) { + this.curMemStoreUsed = d; + } } /** @@ -426,4 +467,4 @@ public class HeapMemoryManager { return needsTuning; } } -} +} \ No newline at end of file diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java index 2965071..3054275 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java @@ -61,7 +61,7 @@ public class TestHeapMemoryManager { conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.75f); conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.05f); HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), - new MemstoreFlusherStub(0), new RegionServerStub(conf)); + new MemstoreFlusherStub(0), new HRegionServer(conf)); assertFalse(manager.isTunerOn()); } @@ -71,7 +71,7 @@ public class TestHeapMemoryManager { conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.05f); HeapMemoryManager manager = new HeapMemoryManager(new BlockCacheStub(0), - new MemstoreFlusherStub(0), new RegionServerStub(conf)); + new MemstoreFlusherStub(0), new HRegionServer(conf)); assertFalse(manager.isTunerOn()); } @@ -83,7 +83,7 @@ public class TestHeapMemoryManager { conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MIN_RANGE_KEY, 0.06f); try { - new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf)); + new HeapMemoryManager(blockCache, memStoreFlusher, new HRegionServer(conf)); fail(); } catch (RuntimeException e) { } @@ -91,7 +91,7 @@ public class TestHeapMemoryManager { conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.2f); conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); try { - new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub(conf)); + new HeapMemoryManager(blockCache, memStoreFlusher, new HRegionServer(conf)); fail(); } catch (RuntimeException e) { } @@ -109,7 +109,7 @@ public class TestHeapMemoryManager { conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); // Let the system start with default values for memstore heap and block cache size. HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, - new RegionServerStub(conf)); + new HRegionServer(conf)); long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; long oldBlockCacheSize = blockCache.maxSize; final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); @@ -150,7 +150,7 @@ public class TestHeapMemoryManager { conf.setLong(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_PERIOD, 1000); // Let the system start with default values for memstore heap and block cache size. HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, - new RegionServerStub(conf)); + new HRegionServer(conf)); long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; long oldBlockCacheSize = blockCache.maxSize; final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); @@ -188,7 +188,7 @@ public class TestHeapMemoryManager { HeapMemoryTuner.class); // Let the system start with default values for memstore heap and block cache size. HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, - new RegionServerStub(conf)); + new HRegionServer(conf)); final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); heapMemoryManager.start(choreService); // Now we wants to be in write mode. Set bigger memstore size from CustomHeapMemoryTuner @@ -218,7 +218,7 @@ public class TestHeapMemoryManager { conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, HeapMemoryTuner.class); HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, - new RegionServerStub(conf)); + new HRegionServer(conf)); final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); heapMemoryManager.start(choreService); CustomHeapMemoryTuner.memstoreSize = 0.78f; @@ -243,7 +243,7 @@ public class TestHeapMemoryManager { conf.setClass(HeapMemoryManager.HBASE_RS_HEAP_MEMORY_TUNER_CLASS, CustomHeapMemoryTuner.class, HeapMemoryTuner.class); HeapMemoryManager heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, - new RegionServerStub(conf)); + new HRegionServer(conf)); long oldMemstoreSize = memStoreFlusher.memstoreSize; long oldBlockCacheSize = blockCache.maxSize; final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); @@ -276,7 +276,7 @@ public class TestHeapMemoryManager { HeapMemoryTuner.class); try { - heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub( + heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new HRegionServer( conf)); fail("Should have failed as the collective heap memory need is above 80%"); } catch (Exception e) { @@ -285,7 +285,7 @@ public class TestHeapMemoryManager { // Change the max/min ranges for memstore and bock cache so as to pass the criteria check conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.6f); conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.6f); - heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new RegionServerStub( + heapMemoryManager = new HeapMemoryManager(blockCache, memStoreFlusher, new HRegionServer( conf)); long oldMemstoreSize = memStoreFlusher.memstoreSize; long oldBlockCacheSize = blockCache.maxSize; @@ -440,6 +440,9 @@ public class TestHeapMemoryManager { } } + + //Using public HRegionServer class to pass more information to HeapMemoryTuner + //so not using this class now private static class RegionServerStub implements Server { private Configuration conf; private boolean stopped = false; @@ -527,3 +530,4 @@ public class TestHeapMemoryManager { } } } +