From 470a0c4411d0b729de09291ca464ec68dcb2e44e Mon Sep 17 00:00:00 2001 From: Biju Nair Date: Thu, 31 Aug 2017 17:47:12 -0400 Subject: [PATCH 1/2] HBASE-18737 Display configured max size of memstore and cache on RS UI --- .../apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java | 5 +++++ .../hbase/regionserver/MetricsRegionServerWrapper.java | 1 + .../hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon | 13 +++++++++---- .../hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon | 6 +++--- .../hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon | 8 +++++++- .../java/org/apache/hadoop/hbase/io/hfile/BlockCache.java | 6 ++++++ .../apache/hadoop/hbase/io/hfile/CombinedBlockCache.java | 5 +++++ .../org/apache/hadoop/hbase/io/hfile/LruBlockCache.java | 2 ++ .../apache/hadoop/hbase/io/hfile/bucket/BucketCache.java | 3 ++- .../hbase/regionserver/MetricsRegionServerWrapperImpl.java | 5 +++++ .../hbase/regionserver/MetricsRegionServerWrapperStub.java | 5 +++++ .../hadoop/hbase/regionserver/TestHeapMemoryManager.java | 5 +++++ 12 files changed, 55 insertions(+), 9 deletions(-) diff --git a/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java b/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java index 965bcbe927..3df9d08db6 100644 --- a/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java +++ b/hbase-external-blockcache/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java @@ -199,6 +199,11 @@ public class MemcachedBlockCache implements BlockCache { } @Override + public long getMaxSize() { + return 0; + } + + @Override public long getFreeSize() { return 0; } diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index ccb9de2afe..e41238daeb 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -207,6 +207,7 @@ public interface MetricsRegionServerWrapper { */ int getFlushQueueSize(); + public long getMemstoreLimit(); /** * Get the size (in bytes) of the block cache that is free. */ diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon index 3afd4f9b86..d31f5f5737 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon @@ -326,25 +326,30 @@ are combined counts. Request count is sum of hits and misses.

- Count + Cache Size + <% String.format("%,d", bc.getMaxSize()) %> + Size of cache + + + Block Count <% String.format("%,d", bc.getBlockCount()) %> Count of Blocks <%if !bucketCache %> - Count + Data Block Count <% String.format("%,d", bc.getDataBlockCount()) %> Count of DATA Blocks - Size + Size of Blocks <% TraditionalBinaryPrefix.long2String(bc.getCurrentSize(), "B", 1) %> Size of Blocks <%if !bucketCache %> - Size + Size of Data Blocks <% TraditionalBinaryPrefix.long2String(bc.getCurrentDataSize(), "B", 1) %> Size of DATA Blocks diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon index 61795e0416..6b403b2d17 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/RSStatusTmpl.jamon @@ -108,12 +108,12 @@ org.apache.hadoop.hbase.zookeeper.MasterAddressTracker;
- <& ../common/TaskMonitorTmpl; filter = filter &> +

Block Cache

+ <& BlockCacheTmpl; cacheConfig = regionServer.getCacheConfig(); config = regionServer.getConfiguration() &>
-

Block Cache

- <& BlockCacheTmpl; cacheConfig = regionServer.getCacheConfig(); config = regionServer.getConfiguration() &> + <& ../common/TaskMonitorTmpl; filter = filter &>
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon index f901030032..1a8b274feb 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/ServerMetricsTmpl.jamon @@ -112,6 +112,7 @@ MetricsRegionServerWrapper mWrap; Direct Memory Used Direct Memory Configured Memstore Size + Memstore Limit @@ -127,7 +128,12 @@ MetricsRegionServerWrapper mWrap; <% TraditionalBinaryPrefix.long2String(DirectMemoryUtils.getDirectMemorySize(), "B", 1) %> - <% TraditionalBinaryPrefix.long2String(mWrap.getMemstoreSize(), "B", 1) %> + + <% TraditionalBinaryPrefix.long2String(mWrap.getMemstoreSize(), "B", 1) %> + + + <% TraditionalBinaryPrefix.long2String(mWrap.getMemstoreLimit(), "B", 1) %> + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index 3674033eb6..a8cefacbac 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -89,6 +89,12 @@ public interface BlockCache extends Iterable { * @return size of cache, in bytes */ long size(); + + /** + * Returns the Max size of the block cache, in bytes. + * @return size of cache, in bytes + */ + long getMaxSize(); /** * Returns the free size of the block cache, in bytes. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index abccdfc24d..7cba9e7864 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -112,6 +112,11 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize { public long size() { return lruCache.size() + l2Cache.size(); } + + @Override + public long getMaxSize() { + return lruCache.getMaxSize() + l2Cache.getMaxSize(); + } @Override public long getCurrentDataSize() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index ad81c50c79..1703eca54d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -840,6 +840,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { * * @return max size in bytes */ + + @Override public long getMaxSize() { return this.maxSize; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 939d53a903..0ad2ef81ba 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -352,7 +352,8 @@ public class BucketCache implements BlockCache, HeapSize { boolean isCacheEnabled() { return this.cacheEnabled; } - + + @Override public long getMaxSize() { return this.cacheCapacity; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index f43db93243..f84aa3046c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -274,6 +274,11 @@ class MetricsRegionServerWrapperImpl } return this.blockCache.getBlockCount(); } + + @Override + public long getMemstoreLimit() { + return this.regionServer.getRegionServerAccounting().getGlobalMemstoreLimit(); + } @Override public long getBlockCacheSize() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 1c3f6f32a4..65f318fc71 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -204,6 +204,11 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe public int getFlushQueueSize() { return 412; } + + @Override + public long getMemstoreLimit() { + return 419; + } @Override public long getBlockCacheFreeSize() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java index 0840ac5027..524237e9e1 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java @@ -738,6 +738,11 @@ public class TestHeapMemoryManager { } @Override + public long getMaxSize() { + return 0; + } + + @Override public long getFreeSize() { return 0; } -- 2.11.0 (Apple Git-81) From 4dcc91ddcbf0f37f18fbe6d75633af5a3211dfaa Mon Sep 17 00:00:00 2001 From: Biju Nair Date: Fri, 1 Sep 2017 13:44:15 -0400 Subject: [PATCH 2/2] Change for review comment --- .../org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon index d31f5f5737..852d84d7af 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon @@ -327,7 +327,7 @@ are combined counts. Request count is sum of hits and misses.

Cache Size - <% String.format("%,d", bc.getMaxSize()) %> + <% TraditionalBinaryPrefix.long2String(bc.getMaxSize(), "B", 1) %> Size of cache -- 2.11.0 (Apple Git-81)