From f7bf6a6ed4bbab5f67d2e2a85d55ef0f801442a3 Mon Sep 17 00:00:00 2001 From: Biju Nair Date: Tue, 15 Aug 2017 12:28:48 -0400 Subject: [PATCH] HBASE-18532 Improve cache related stats rendered on RS UI --- .../hbase/tmpl/regionserver/BlockCacheTmpl.jamon | 25 ++++----------------- .../apache/hadoop/hbase/io/hfile/BlockCache.java | 12 ++++++++++ .../hadoop/hbase/io/hfile/CombinedBlockCache.java | 10 +++++++++ .../hadoop/hbase/io/hfile/LruBlockCache.java | 26 ++++++++++++++++++++++ .../hadoop/hbase/io/hfile/MemcachedBlockCache.java | 10 +++++++++ .../hadoop/hbase/io/hfile/bucket/BucketCache.java | 10 +++++++++ .../hbase/regionserver/TestHeapMemoryManager.java | 10 +++++++++ 7 files changed, 82 insertions(+), 21 deletions(-) diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon index b13a1d9d60..298e2bf949 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon @@ -335,25 +335,25 @@ are combined counts. Request count is sum of hits and misses.

Count - <% String.format("%,d", cbsbf.getCount()) %> + <% String.format("%,d", bc.getBlockCount()) %> Count of Blocks <%if !bucketCache %> Count - <% String.format("%,d", cbsbf.getDataCount()) %> + <% String.format("%,d", bc.getDataBlockCount()) %> Count of DATA Blocks Size - <% StringUtils.humanReadableInt(cbsbf.getSize()) %> + <% StringUtils.humanReadableInt(bc.getCurrentSize()) %> Size of Blocks <%if !bucketCache %> Size - <% StringUtils.humanReadableInt(cbsbf.getDataSize()) %> + <% StringUtils.humanReadableInt(bc.getCurrentDataSize()) %> Size of DATA Blocks @@ -377,23 +377,6 @@ are combined counts. Request count is sum of hits and misses.

View block cache as JSON | Block cache as JSON by file

<%if bucketCache %>

BucketCache does not discern between DATA and META blocks so we do not show DATA counts (If deploy is using CombinedBlockCache, BucketCache is only DATA blocks

-

BucketCache Buckets

- - - - - - - -<%for Bucket bucket: buckets %> - - - - - - - -
Bucket OffsetAllocation SizeFree BytesUsed Bytes
<% bucket.getBaseOffset() %><% bucket.getItemAllocationSize() %><% bucket.getFreeBytes() %><% bucket.getUsedBytes() %>
<%java> cbsbf = null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java index 57c4be92c0..faffb78295 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java @@ -100,12 +100,24 @@ public interface BlockCache extends Iterable { * @return occupied space in cache, in bytes */ long getCurrentSize(); + + /** + * Returns the occupied size of data blocks, in bytes. + * @return occupied space in cache, in bytes + */ + long getCurrentDataSize(); /** * Returns the number of blocks currently cached in the block cache. * @return number of blocks in the cache */ long getBlockCount(); + + /** + * Returns the number of data blocks currently cached in the block cache. + * @return number of blocks in the cache + */ + long getDataBlockCount(); /** * @return Iterator over the blocks in the cache. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 7725cf9290..6d497609f8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -113,6 +113,11 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize { public long size() { return lruCache.size() + l2Cache.size(); } + + @Override + public long getCurrentDataSize() { + return lruCache.getCurrentDataSize(); + } @Override public long getFreeSize() { @@ -128,6 +133,11 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize { public long getBlockCount() { return lruCache.getBlockCount() + l2Cache.getBlockCount(); } + + @Override + public long getDataBlockCount() { + return lruCache.getDataBlockCount(); + } private static class CombinedCacheStats extends CacheStats { private final CacheStats lruCacheStats; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 48e4cadd1c..0b7f92d801 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -161,9 +161,15 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { /** Current size of cache */ private final AtomicLong size; + + /** Current size of data blocks */ + private final AtomicLong dataBlockSize; /** Current number of cached elements */ private final AtomicLong elements; + + /** Current number of cached data block elements */ + private final AtomicLong dataBlockElements; /** Cache access count (sequential ID) */ private final AtomicLong count; @@ -289,6 +295,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { this.stats = new CacheStats(this.getClass().getSimpleName()); this.count = new AtomicLong(0); this.elements = new AtomicLong(0); + this.dataBlockElements = new AtomicLong(0); + this.dataBlockSize = new AtomicLong(0); this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel); this.size = new AtomicLong(this.overhead); if(evictionThread) { @@ -342,6 +350,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { long newSize = updateSizeMetrics(cb, false); map.put(cacheKey, cb); long val = elements.incrementAndGet(); + if (buf.getBlockType().isData()) + dataBlockElements.incrementAndGet(); if (LOG.isTraceEnabled()) { long size = map.size(); assertCounterSanity(size, val); @@ -399,9 +409,13 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { */ protected long updateSizeMetrics(LruCachedBlock cb, boolean evict) { long heapsize = cb.heapSize(); + BlockType bt = cb.getBuffer().getBlockType(); if (evict) { heapsize *= -1; } + if (bt != null && bt.isData()) { + dataBlockSize.addAndGet(heapsize); + } return size.addAndGet(heapsize); } @@ -497,6 +511,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { long size = map.size(); assertCounterSanity(size, val); } + if (block.getBuffer().getBlockType().isData()) + dataBlockElements.decrementAndGet(); stats.evicted(block.getCachedTime()); if (evictedByEvictionProcess && victimHandler != null) { if (victimHandler instanceof BucketCache) { @@ -756,6 +772,11 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { public long getCurrentSize() { return this.size.get(); } + + @Override + public long getCurrentDataSize() { + return this.dataBlockSize.get(); + } @Override public long getFreeSize() { @@ -771,6 +792,11 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { public long getBlockCount() { return this.elements.get(); } + + @Override + public long getDataBlockCount() { + return this.dataBlockElements.get(); + } EvictionThread getEvictionThread() { return this.evictionThread; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java index 57e7f2827a..20c2e92186 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java @@ -204,11 +204,21 @@ public class MemcachedBlockCache implements BlockCache { public long getCurrentSize() { return 0; } + + @Override + public long getCurrentDataSize() { + return 0; + } @Override public long getBlockCount() { return 0; } + + @Override + public long getDataBlockCount() { + return 0; + } @Override public Iterator iterator() { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 45c75e36f3..8f848d5d0e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -1025,6 +1025,11 @@ public class BucketCache implements BlockCache, HeapSize { public long heapSize() { return this.heapSize.get(); } + + @Override + public long getCurrentDataSize() { + return 0; + } @Override public long size() { @@ -1040,6 +1045,11 @@ public class BucketCache implements BlockCache, HeapSize { public long getBlockCount() { return this.blockNumber.get(); } + + @Override + public long getDataBlockCount() { + return 0; + } @Override public long getCurrentSize() { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java index e8c1ee8dc8..20b879a9b8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java @@ -381,11 +381,21 @@ public class TestHeapMemoryManager { public long getCurrentSize() { return 0; } + + @Override + public long getCurrentDataSize() { + return 0; + } @Override public long getBlockCount() { return 0; } + + @Override + public long getDataBlockCount() { + return 0; + } @Override public void setMaxSize(long size) { -- 2.11.0 (Apple Git-81)