diff --git src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java index 590e774..2bef4b6 100644 --- src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java +++ src/main/java/org/apache/hadoop/hbase/HBaseConfiguration.java @@ -19,6 +19,7 @@ */ package org.apache.hadoop.hbase; +import java.lang.management.ManagementFactory; import java.util.Map.Entry; import org.apache.commons.logging.Log; @@ -72,8 +73,9 @@ public class HBaseConfiguration extends Configuration { } private static void checkForClusterFreeMemoryLimit(Configuration conf) { - float globalMemstoreLimit = conf.getFloat("hbase.regionserver.global.memstore.upperLimit", 0.4f); + float globalMemstoreLimit = conf.getFloat(HConstants.MEMSTORE_UPPER_LIMIT, 0.4f); int gml = (int)(globalMemstoreLimit * CONVERT_TO_PERCENTAGE); + checkMemStoreLimit(globalMemstoreLimit); float blockCacheUpperLimit = conf.getFloat(HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT); @@ -85,14 +87,28 @@ public class HBaseConfiguration extends Configuration { "Current heap configuration for MemStore and BlockCache exceeds " + "the threshold required for successful cluster operation. " + "The combined value cannot exceed 0.8. Please check " + - "the settings for hbase.regionserver.global.memstore.upperLimit and " + + "the settings for " + HConstants.MEMSTORE_UPPER_LIMIT + " and " + "hfile.block.cache.size in your configuration. " + - "hbase.regionserver.global.memstore.upperLimit is " + - globalMemstoreLimit + + HConstants.MEMSTORE_UPPER_LIMIT + " is " + globalMemstoreLimit + " hfile.block.cache.size is " + blockCacheUpperLimit); } } + private static void checkMemStoreLimit(float globalMemstoreLimit) { + long maxHeap = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); + float memstoreSize = maxHeap * globalMemstoreLimit; + int minMemstorePercentage = (int) (HConstants.MIN_MEMSTORE_PERCENTAGE * CONVERT_TO_PERCENTAGE); + if (memstoreSize < HConstants.MIN_MEMSTORE_SIZE + && globalMemstoreLimit < HConstants.MIN_MEMSTORE_PERCENTAGE) { + throw new RuntimeException( + "Current configuration for MemStore is too low. The size of the memstore has to be at least " + + memstoreSize + " bytes or " + minMemstorePercentage + " percent of the heap. " + + "Please check the settings for " + HConstants.MEMSTORE_UPPER_LIMIT + " in your " + + "configuration. " + HConstants.MEMSTORE_UPPER_LIMIT + " is " + globalMemstoreLimit + ); + } + } + public static Configuration addHbaseResources(Configuration conf) { conf.addResource("hbase-default.xml"); conf.addResource("hbase-site.xml"); diff --git src/main/java/org/apache/hadoop/hbase/HConstants.java src/main/java/org/apache/hadoop/hbase/HConstants.java index 9625299..b63fb9a 100644 --- src/main/java/org/apache/hadoop/hbase/HConstants.java +++ src/main/java/org/apache/hadoop/hbase/HConstants.java @@ -265,6 +265,17 @@ public final class HConstants { public static final String HREGION_MEMSTORE_FLUSH_SIZE = "hbase.hregion.memstore.flush.size"; + public static final String MEMSTORE_LOWER_LIMIT = + "hbase.regionserver.global.memstore.lowerLimit"; + + /** Conf key for the memstore upper limit */ + public static final String MEMSTORE_UPPER_LIMIT = + "hbase.regionserver.global.memstore.upperLimit"; + + /** Minimum size of the memstore must either be 100MB or 10 percent of the heap */ + public static final int MIN_MEMSTORE_SIZE = 104857600; + public static final float MIN_MEMSTORE_PERCENTAGE = .1f; + /** Default size of a reservation block */ public static final int DEFAULT_SIZE_RESERVATION_BLOCK = 1024 * 1024 * 5; diff --git src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java index 7f704d8..07e63df 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java @@ -78,10 +78,6 @@ class MemStoreFlusher extends HasThread implements FlushRequester { private static final float DEFAULT_UPPER = 0.4f; private static final float DEFAULT_LOWER = 0.35f; - private static final String UPPER_KEY = - "hbase.regionserver.global.memstore.upperLimit"; - private static final String LOWER_KEY = - "hbase.regionserver.global.memstore.lowerLimit"; private long blockingWaitTime; private final Counter updatesBlockedMsHighWater = new Counter(); @@ -97,13 +93,12 @@ class MemStoreFlusher extends HasThread implements FlushRequester { this.threadWakeFrequency = conf.getLong(HConstants.THREAD_WAKE_FREQUENCY, 10 * 1000); long max = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax(); - this.globalMemStoreLimit = globalMemStoreLimit(max, DEFAULT_UPPER, - UPPER_KEY, conf); - long lower = globalMemStoreLimit(max, DEFAULT_LOWER, LOWER_KEY, conf); + this.globalMemStoreLimit = (long) (max * conf.getFloat(HConstants.MEMSTORE_LOWER_LIMIT, DEFAULT_UPPER)); + long lower = (long) (max * conf.getFloat(HConstants.MEMSTORE_LOWER_LIMIT, DEFAULT_LOWER)); if (lower > this.globalMemStoreLimit) { lower = this.globalMemStoreLimit; LOG.info("Setting globalMemStoreLimitLowMark == globalMemStoreLimit " + - "because supplied " + LOWER_KEY + " was > " + UPPER_KEY); + "because supplied " + HConstants.MEMSTORE_LOWER_LIMIT + " was > " + HConstants.MEMSTORE_UPPER_LIMIT); } this.globalMemStoreLimitLowMark = lower; @@ -116,32 +111,6 @@ class MemStoreFlusher extends HasThread implements FlushRequester { ", maxHeap=" + StringUtils.humanReadableInt(max)); } - /** - * Calculate size using passed key for configured - * percentage of max. - * @param max - * @param defaultLimit - * @param key - * @param c - * @return Limit. - */ - static long globalMemStoreLimit(final long max, - final float defaultLimit, final String key, final Configuration c) { - float limit = c.getFloat(key, defaultLimit); - return getMemStoreLimit(max, limit, defaultLimit); - } - - static long getMemStoreLimit(final long max, final float limit, - final float defaultLimit) { - float effectiveLimit = limit; - if (limit >= 0.9f || limit < 0.1f) { - LOG.warn("Setting global memstore limit to default of " + defaultLimit + - " because supplied value outside allowed range of 0.1 -> 0.9"); - effectiveLimit = defaultLimit; - } - return (long)(max * effectiveLimit); - } - public Counter getUpdatesBlockedMsHighWater() { return this.updatesBlockedMsHighWater; }