From 912db0f2f1470cbc2c9870b87742fa1568aa0afb Mon Sep 17 00:00:00 2001 From: Biju Nair Date: Tue, 29 Aug 2017 16:28:11 -0400 Subject: [PATCH] HBASE-18708 Configure on-heap bucketCache size using percentage of heap size --- .../hadoop/hbase/io/util/HeapMemorySizeUtil.java | 36 +++++++++++++--------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java index 4369305ee5..32e4aa9305 100644 --- a/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java +++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/io/util/HeapMemorySizeUtil.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.HConstants; +import static org.apache.hadoop.hbase.HConstants.BUCKET_CACHE_IOENGINE_KEY; @InterfaceAudience.Private public class HeapMemorySizeUtil { @@ -164,13 +165,12 @@ public class HeapMemorySizeUtil { // L2 block cache can be on heap when IOEngine is "heap" if (bucketCacheIOEngineName != null && bucketCacheIOEngineName.startsWith("heap")) { float bucketCachePercentage = conf.getFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0F); - long max = -1L; - final MemoryUsage usage = safeGetHeapMemoryUsage(); - if (usage != null) { - max = usage.getMax(); + if (bucketCachePercentage < 1) { + l2CachePercent = bucketCachePercentage; + } else { + throw new IllegalArgumentException(HConstants.BUCKET_CACHE_SIZE_KEY + + " must be between 0.0 and 1.0, and not > 1.0 for offheap buckeCache"); } - l2CachePercent = bucketCachePercentage < 1 ? bucketCachePercentage - : (bucketCachePercentage * 1024 * 1024) / max; } return l2CachePercent; } @@ -207,19 +207,25 @@ public class HeapMemorySizeUtil { public static long getBucketCacheSize(final Configuration conf) { final float bucketCachePercentage = conf.getFloat(HConstants.BUCKET_CACHE_SIZE_KEY, 0F); long bucketCacheSize; - // Values < 1 are treated as % of heap - if (bucketCachePercentage < 1) { - long max = -1L; - final MemoryUsage usage = safeGetHeapMemoryUsage(); - if (usage != null) { - max = usage.getMax(); + String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null); + if (bucketCacheIOEngineName.startsWith("heap")) { + // Values < 1 are treated as % of heap + if (bucketCachePercentage < 1) { + long max = -1L; + final MemoryUsage usage = safeGetHeapMemoryUsage(); + if (usage != null) { + max = usage.getMax(); + } + bucketCacheSize = (long)(max * bucketCachePercentage); + } else { + bucketCacheSize = -1; + throw new IllegalArgumentException(HConstants.BUCKET_CACHE_SIZE_KEY + + " must be between 0.0 and 1.0, and not > 1.0 for offheap buckeCache"); } - bucketCacheSize = (long)(max * bucketCachePercentage); - // values >= 1 are treated as # of MiB } else { + // values >= 1 are treated as # of MiB bucketCacheSize = (long)(bucketCachePercentage * 1024 * 1024); } return bucketCacheSize; } - } -- 2.11.0 (Apple Git-81)