From f7bf6a6ed4bbab5f67d2e2a85d55ef0f801442a3 Mon Sep 17 00:00:00 2001
From: Biju Nair
Date: Tue, 15 Aug 2017 12:28:48 -0400
Subject: [PATCH] HBASE-18532 Improve cache related stats rendered on RS UI
---
.../hbase/tmpl/regionserver/BlockCacheTmpl.jamon | 25 ++++-----------------
.../apache/hadoop/hbase/io/hfile/BlockCache.java | 12 ++++++++++
.../hadoop/hbase/io/hfile/CombinedBlockCache.java | 10 +++++++++
.../hadoop/hbase/io/hfile/LruBlockCache.java | 26 ++++++++++++++++++++++
.../hadoop/hbase/io/hfile/MemcachedBlockCache.java | 10 +++++++++
.../hadoop/hbase/io/hfile/bucket/BucketCache.java | 10 +++++++++
.../hbase/regionserver/TestHeapMemoryManager.java | 10 +++++++++
7 files changed, 82 insertions(+), 21 deletions(-)
diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
index b13a1d9d60..298e2bf949 100644
--- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
+++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon
@@ -335,25 +335,25 @@ are combined counts. Request count is sum of hits and misses.
%if>
| Count |
- <% String.format("%,d", cbsbf.getCount()) %> |
+ <% String.format("%,d", bc.getBlockCount()) %> |
Count of Blocks |
<%if !bucketCache %>
| Count |
- <% String.format("%,d", cbsbf.getDataCount()) %> |
+ <% String.format("%,d", bc.getDataBlockCount()) %> |
Count of DATA Blocks |
%if>
| Size |
- <% StringUtils.humanReadableInt(cbsbf.getSize()) %> |
+ <% StringUtils.humanReadableInt(bc.getCurrentSize()) %> |
Size of Blocks |
<%if !bucketCache %>
| Size |
- <% StringUtils.humanReadableInt(cbsbf.getDataSize()) %> |
+ <% StringUtils.humanReadableInt(bc.getCurrentDataSize()) %> |
Size of DATA Blocks |
%if>
@@ -377,23 +377,6 @@ are combined counts. Request count is sum of hits and misses.
View block cache as JSON | Block cache as JSON by file
<%if bucketCache %>
BucketCache does not discern between DATA and META blocks so we do not show DATA counts (If deploy is using CombinedBlockCache, BucketCache is only DATA blocks
-BucketCache Buckets
-
-
- | Bucket Offset |
- Allocation Size |
- Free Bytes |
- Used Bytes |
-
-<%for Bucket bucket: buckets %>
-
- | <% bucket.getBaseOffset() %> |
- <% bucket.getItemAllocationSize() %> |
- <% bucket.getFreeBytes() %> |
- <% bucket.getUsedBytes() %> |
-
-%for>
-
%if>
<%java>
cbsbf = null;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
index 57c4be92c0..faffb78295 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java
@@ -100,12 +100,24 @@ public interface BlockCache extends Iterable {
* @return occupied space in cache, in bytes
*/
long getCurrentSize();
+
+ /**
+ * Returns the occupied size of data blocks, in bytes.
+ * @return occupied space in cache, in bytes
+ */
+ long getCurrentDataSize();
/**
* Returns the number of blocks currently cached in the block cache.
* @return number of blocks in the cache
*/
long getBlockCount();
+
+ /**
+ * Returns the number of data blocks currently cached in the block cache.
+ * @return number of blocks in the cache
+ */
+ long getDataBlockCount();
/**
* @return Iterator over the blocks in the cache.
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
index 7725cf9290..6d497609f8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java
@@ -113,6 +113,11 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
public long size() {
return lruCache.size() + l2Cache.size();
}
+
+ @Override
+ public long getCurrentDataSize() {
+ return lruCache.getCurrentDataSize();
+ }
@Override
public long getFreeSize() {
@@ -128,6 +133,11 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize {
public long getBlockCount() {
return lruCache.getBlockCount() + l2Cache.getBlockCount();
}
+
+ @Override
+ public long getDataBlockCount() {
+ return lruCache.getDataBlockCount();
+ }
private static class CombinedCacheStats extends CacheStats {
private final CacheStats lruCacheStats;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
index 48e4cadd1c..0b7f92d801 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java
@@ -161,9 +161,15 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
/** Current size of cache */
private final AtomicLong size;
+
+ /** Current size of data blocks */
+ private final AtomicLong dataBlockSize;
/** Current number of cached elements */
private final AtomicLong elements;
+
+ /** Current number of cached data block elements */
+ private final AtomicLong dataBlockElements;
/** Cache access count (sequential ID) */
private final AtomicLong count;
@@ -289,6 +295,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
this.stats = new CacheStats(this.getClass().getSimpleName());
this.count = new AtomicLong(0);
this.elements = new AtomicLong(0);
+ this.dataBlockElements = new AtomicLong(0);
+ this.dataBlockSize = new AtomicLong(0);
this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel);
this.size = new AtomicLong(this.overhead);
if(evictionThread) {
@@ -342,6 +350,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
long newSize = updateSizeMetrics(cb, false);
map.put(cacheKey, cb);
long val = elements.incrementAndGet();
+ if (buf.getBlockType().isData())
+ dataBlockElements.incrementAndGet();
if (LOG.isTraceEnabled()) {
long size = map.size();
assertCounterSanity(size, val);
@@ -399,9 +409,13 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
*/
protected long updateSizeMetrics(LruCachedBlock cb, boolean evict) {
long heapsize = cb.heapSize();
+ BlockType bt = cb.getBuffer().getBlockType();
if (evict) {
heapsize *= -1;
}
+ if (bt != null && bt.isData()) {
+ dataBlockSize.addAndGet(heapsize);
+ }
return size.addAndGet(heapsize);
}
@@ -497,6 +511,8 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
long size = map.size();
assertCounterSanity(size, val);
}
+ if (block.getBuffer().getBlockType().isData())
+ dataBlockElements.decrementAndGet();
stats.evicted(block.getCachedTime());
if (evictedByEvictionProcess && victimHandler != null) {
if (victimHandler instanceof BucketCache) {
@@ -756,6 +772,11 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
public long getCurrentSize() {
return this.size.get();
}
+
+ @Override
+ public long getCurrentDataSize() {
+ return this.dataBlockSize.get();
+ }
@Override
public long getFreeSize() {
@@ -771,6 +792,11 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize {
public long getBlockCount() {
return this.elements.get();
}
+
+ @Override
+ public long getDataBlockCount() {
+ return this.dataBlockElements.get();
+ }
EvictionThread getEvictionThread() {
return this.evictionThread;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
index 57e7f2827a..20c2e92186 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java
@@ -204,11 +204,21 @@ public class MemcachedBlockCache implements BlockCache {
public long getCurrentSize() {
return 0;
}
+
+ @Override
+ public long getCurrentDataSize() {
+ return 0;
+ }
@Override
public long getBlockCount() {
return 0;
}
+
+ @Override
+ public long getDataBlockCount() {
+ return 0;
+ }
@Override
public Iterator iterator() {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
index 45c75e36f3..8f848d5d0e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java
@@ -1025,6 +1025,11 @@ public class BucketCache implements BlockCache, HeapSize {
public long heapSize() {
return this.heapSize.get();
}
+
+ @Override
+ public long getCurrentDataSize() {
+ return 0;
+ }
@Override
public long size() {
@@ -1040,6 +1045,11 @@ public class BucketCache implements BlockCache, HeapSize {
public long getBlockCount() {
return this.blockNumber.get();
}
+
+ @Override
+ public long getDataBlockCount() {
+ return 0;
+ }
@Override
public long getCurrentSize() {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
index e8c1ee8dc8..20b879a9b8 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java
@@ -381,11 +381,21 @@ public class TestHeapMemoryManager {
public long getCurrentSize() {
return 0;
}
+
+ @Override
+ public long getCurrentDataSize() {
+ return 0;
+ }
@Override
public long getBlockCount() {
return 0;
}
+
+ @Override
+ public long getDataBlockCount() {
+ return 0;
+ }
@Override
public void setMaxSize(long size) {
--
2.11.0 (Apple Git-81)