From 61e4c1e5c04e55be30f743e20a03b81103aac25e Mon Sep 17 00:00:00 2001 From: Ashu Pachauri Date: Wed, 13 Jan 2016 13:49:43 -0800 Subject: [PATCH] Fix HeapMemoryTuner overtuning memstore HeapMemoryTuner often over tunes memstore without looking at the lower limit of the previous memstore size and causing a situation in which memstore used size suddenly exceeds the total memstore size. --- .../hbase/regionserver/DefaultHeapMemoryTuner.java | 12 ++++++++++- .../hbase/regionserver/TestHeapMemoryManager.java | 23 ++++++++++++++-------- 2 files changed, 26 insertions(+), 9 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java index b6e81dd..2b3e8ea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultHeapMemoryTuner.java @@ -269,10 +269,20 @@ class DefaultHeapMemoryTuner implements HeapMemoryTuner { newTuneDirection = StepDirection.NEUTRAL; } // Increase / decrease the memstore / block cahce sizes depending on new tuner step. + float globalMemstoreLowerMark = HeapMemorySizeUtil.getGlobalMemStoreLowerMark(conf, + context.getCurMemStoreSize()); + // We don't want to exert immediate pressure on memstore. So, we decrease its size gracefully; + // we set a minimum bar in the middle of the total memstore size and the lower limit. + float minMemstoreSize = ((globalMemstoreLowerMark + 1) * context.getCurMemStoreSize()) / 2.00f; + switch (newTuneDirection) { case INCREASE_BLOCK_CACHE_SIZE: - newBlockCacheSize = context.getCurBlockCacheSize() + step; newMemstoreSize = context.getCurMemStoreSize() - step; + if (newMemstoreSize < minMemstoreSize) { + newMemstoreSize = minMemstoreSize; + step = context.getCurMemStoreSize() - newMemstoreSize; + } + newBlockCacheSize = context.getCurBlockCacheSize() + step; rollingStatsForTunerSteps.insertDataValue(-(int)(step*100000)); decayingTunerStepSizeSum = (decayingTunerStepSizeSum - step)/2.00f; break; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java index 0e38afc..5f42a03 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java @@ -228,6 +228,7 @@ public class TestHeapMemoryManager { blockCache.setTestBlockSize((long) (maxHeapSize * 0.4 * 0.8)); regionServerAccounting.setTestMemstoreSize(0); Configuration conf = HBaseConfiguration.create(); + conf.setFloat(HeapMemorySizeUtil.MEMSTORE_SIZE_LOWER_LIMIT_KEY, 0.7f); conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MAX_RANGE_KEY, 0.75f); conf.setFloat(HeapMemoryManager.MEMSTORE_SIZE_MIN_RANGE_KEY, 0.10f); conf.setFloat(HeapMemoryManager.BLOCK_CACHE_SIZE_MAX_RANGE_KEY, 0.7f); @@ -239,6 +240,11 @@ public class TestHeapMemoryManager { new RegionServerStub(conf), new RegionServerAccountingStub()); long oldMemstoreHeapSize = memStoreFlusher.memstoreSize; long oldBlockCacheSize = blockCache.maxSize; + long oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; + long maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; + float maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; + maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ? + DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue; final ChoreService choreService = new ChoreService("TEST_SERVER_NAME"); heapMemoryManager.start(choreService); blockCache.evictBlock(null); @@ -246,20 +252,21 @@ public class TestHeapMemoryManager { blockCache.evictBlock(null); // Allow the tuner to run once and do necessary memory up waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); - assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldMemstoreHeapSize, - memStoreFlusher.memstoreSize); - assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldBlockCacheSize, - blockCache.maxSize); + assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); + assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); oldMemstoreHeapSize = memStoreFlusher.memstoreSize; oldBlockCacheSize = blockCache.maxSize; + oldMemstoreLowerMarkSize = 7 * oldMemstoreHeapSize / 10; + maxTuneSize = oldMemstoreHeapSize - (oldMemstoreLowerMarkSize + oldMemstoreHeapSize) / 2; + maxStepValue = (maxTuneSize * 1.0f) / oldMemstoreHeapSize; + maxStepValue = maxStepValue > DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE ? + DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE:maxStepValue; // Do some more evictions before the next run of HeapMemoryTuner blockCache.evictBlock(null); // Allow the tuner to run once and do necessary memory up waitForTune(memStoreFlusher, memStoreFlusher.memstoreSize); - assertHeapSpaceDelta(-(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE), oldMemstoreHeapSize, - memStoreFlusher.memstoreSize); - assertHeapSpaceDelta(DefaultHeapMemoryTuner.DEFAULT_MAX_STEP_VALUE, oldBlockCacheSize, - blockCache.maxSize); + assertHeapSpaceDelta(-maxStepValue, oldMemstoreHeapSize, memStoreFlusher.memstoreSize); + assertHeapSpaceDelta(maxStepValue, oldBlockCacheSize, blockCache.maxSize); } @Test -- 1.9.5