diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon index fa83fc3..f7c23da 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon @@ -30,11 +30,13 @@ Configuration config; bcName = bc.getClass().getSimpleName(); } BlockCache [] bcs = cacheConfig == null? null: cacheConfig.getBlockCache() == null? null: cacheConfig.getBlockCache().getBlockCaches(); + // If more than one bc, show evictions in each bc listing so can compare + boolean evictions = bcs != null && bcs.length > 1; <%import> java.util.Map; org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.CachedBlocksByFile; -org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.AgeSnapshot; +org.apache.hadoop.hbase.io.hfile.AgeSnapshot; org.apache.hadoop.hbase.io.hfile.CachedBlock; org.apache.hadoop.conf.Configuration; org.apache.hadoop.hbase.io.hfile.CacheConfig; @@ -64,10 +66,10 @@ org.apache.hadoop.util.StringUtils; <& bc_stats; cacheConfig = cacheConfig &>
- <& bc_l; bc = bcs == null? bc: bcs[0]; name = "L1" &> + <& bc_l; bc = bcs == null? bc: bcs[0]; name = "L1"; evictions = evictions; &>
- <& bc_l; bc = bcs == null? null: bcs.length <= 1? null: bcs[1]; name = "L2" &> + <& bc_l; bc = bcs == null? null: bcs.length <= 1? null: bcs[1]; name = "L2"; evictions = evictions; &>
@@ -168,6 +170,35 @@ org.apache.hadoop.util.StringUtils; +<%def evictions_tmpl> +<%args> + BlockCache bc; + +<%java> + AgeSnapshot ageAtEvictionSnapshot = bc.getStats().getAgeAtEvictionSnapshot(); + + + Evicted + <% String.format("%,d", bc.getStats().getEvictedCount()) %> + The total number of blocks evicted + + + Evictions + <% String.format("%,d", bc.getStats().getEvictionCount()) %> + The total number of times an eviction has occurred + + + Mean + <% String.format("%,d", (long)(ageAtEvictionSnapshot.getMean()/1000000)) %> + Mean age of Blocks at eviction time (milliseconds) + + + StdDev + <% String.format("%,d", (long)(ageAtEvictionSnapshot.getStdDev()/1000000)) %> + Standard Deviation for age of Blocks at eviction time + + + <%def bc_stats> <%args> CacheConfig cacheConfig; @@ -196,16 +227,7 @@ org.apache.hadoop.util.StringUtils; <% String.format("%,d", cacheConfig.getBlockCache().getBlockCount()) %> Number of blocks in block cache - - Evicted - <% String.format("%,d", cacheConfig.getBlockCache().getStats().getEvictedCount()) %> - The total number of blocks evicted - - - Evictions - <% String.format("%,d", cacheConfig.getBlockCache().getStats().getEvictionCount()) %> - The total number of times an eviction has occurred - + <& evictions_tmpl; bc = cacheConfig.getBlockCache(); &> Hits <% String.format("%,d", cacheConfig.getBlockCache().getStats().getHitCount()) %> @@ -241,11 +263,12 @@ are combined counts. Request count is sum of hits and misses.

<%args> BlockCache bc; String name; + boolean evictions; <%if bc == null %>

No <% name %> deployed

<%else> -<& block_cache; bc = bc; name = name; &> +<& block_cache; bc = bc; name = name; evictions = evictions; &> @@ -253,6 +276,7 @@ are combined counts. Request count is sum of hits and misses.

<%args> BlockCache bc; String name; + boolean evictions; <%java> final long nanosPerSecond = 1000000000; @@ -260,7 +284,7 @@ are combined counts. Request count is sum of hits and misses.

String bcName = bc.getClass().getSimpleName(); org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.CachedBlocksByFile cbsbf = org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.getLoadedCachedBlocksByFile(config, bc); - AgeSnapshot snapshot = cbsbf.getAgeSnapshot(); + AgeSnapshot cbsbfSnapshot = cbsbf.getAgeInCacheSnapshot(); boolean bucketCache = bc.getClass().getSimpleName().equals("BucketCache"); BucketCacheStats bucketCacheStats = null; @@ -274,7 +298,7 @@ are combined counts. Request count is sum of hits and misses.

} <%if cbsbf.isFull() %> -

Too many blocks! Listing out the first <% snapshot.getMax() %> only (hbase.ui.blockcache.by.file.max)

+

Too many blocks! Listing out the first <% cbsbfSnapshot.getMax() %> only (hbase.ui.blockcache.by.file.max)

@@ -317,49 +341,8 @@ are combined counts. Request count is sum of hits and misses.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +<%if evictions %><& evictions_tmpl; bc = bc; &> <%if bucketCache %> diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon index 184076e..c5002b5 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon @@ -53,7 +53,7 @@ com.yammer.metrics.stats.Snapshot; } CachedBlocksByFile cbsbf = BlockCacheUtil.getLoadedCachedBlocksByFile(conf, bc); -<%if bcv.equals("file") %><& bc_by_file; cbsbf = cbsbf; &><%else>{<% BlockCacheUtil.toJSON(bc) %>, <% cbsbf %> } +<%if bcv.equals("file") %><& bc_by_file; cbsbf = cbsbf; &><%else>[ <% BlockCacheUtil.toJSON(bc) %>, <% BlockCacheUtil.toJSON(cbsbf) %> ] <%java> cbsbf = null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index d146084..f793d4b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -160,57 +160,6 @@ public class BlockCacheUtil { } /** - * Snapshot of block cache age in cache. - * This object is preferred because we can control how it is serialized out when JSON'ing. - */ - @JsonIgnoreProperties({"ageHistogram", "snapshot"}) - public static class AgeSnapshot { - private final Histogram ageHistogram; - private final Snapshot snapshot; - - AgeSnapshot(final Histogram ageHistogram) { - this.ageHistogram = ageHistogram; - this.snapshot = ageHistogram.getSnapshot(); - } - - public double get75thPercentile() { - return snapshot.get75thPercentile(); - } - - public double get95thPercentile() { - return snapshot.get95thPercentile(); - } - - public double get98thPercentile() { - return snapshot.get98thPercentile(); - } - - public double get999thPercentile() { - return snapshot.get999thPercentile(); - } - - public double get99thPercentile() { - return snapshot.get99thPercentile(); - } - - public double getMean() { - return this.ageHistogram.mean(); - } - - public double getMax() { - return ageHistogram.max(); - } - - public double getMin() { - return ageHistogram.min(); - } - - public double getStdDev() { - return ageHistogram.stdDev(); - } - } - - /** * Get a {@link CachedBlocksByFile} instance and load it up by iterating content in * {@link BlockCache}. * @param conf Used to read configurations @@ -319,7 +268,7 @@ public class BlockCacheUtil { return dataSize; } - public AgeSnapshot getAgeSnapshot() { + public AgeSnapshot getAgeInCacheSnapshot() { return new AgeSnapshot(this.age); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java index 4674eff..cad7781 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java @@ -22,11 +22,19 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.classification.InterfaceAudience; +import com.yammer.metrics.core.Histogram; +import com.yammer.metrics.core.MetricsRegistry; + /** * Class that implements cache metrics. */ @InterfaceAudience.Private public class CacheStats { + /** + * Needed making histograms. + */ + private static final MetricsRegistry METRICS = new MetricsRegistry(); + /** Sliding window statistics. The number of metric periods to include in * sliding window hit ratio calculations. */ @@ -78,6 +86,11 @@ public class CacheStats { private long lastRequestCachingCount = 0; /** Current window index (next to be updated) */ private int windowIndex = 0; + /** + * Keep running age at eviction time + */ + private Histogram ageAtEviction = METRICS.newHistogram(CacheStats.class, "ageAtEviction"); + private long startTime = System.nanoTime(); public CacheStats() { this(DEFAULT_WINDOW_PERIODS); @@ -93,10 +106,13 @@ public class CacheStats { @Override public String toString() { + AgeSnapshot snapshot = getAgeAtEvictionSnapshot(); return "hitCount=" + getHitCount() + ", hitCachingCount=" + getHitCachingCount() + ", missCount=" + getMissCount() + ", missCachingCount=" + getMissCachingCount() + ", evictionCount=" + getEvictionCount() + - ", evictedBlockCount=" + getEvictedCount(); + ", evictedBlockCount=" + getEvictedCount() + + ", evictedAgeMean=" + snapshot.getMean() + + ", evictedAgeStdDev=" + snapshot.getStdDev(); } public void miss(boolean caching) { @@ -113,8 +129,9 @@ public class CacheStats { evictionCount.incrementAndGet(); } - public void evicted() { - evictedBlockCount.incrementAndGet(); + public void evicted(final long t) { + this.ageAtEviction.update(t - this.startTime); + this.evictedBlockCount.incrementAndGet(); } public long getRequestCount() { @@ -146,7 +163,7 @@ public class CacheStats { } public long getEvictedCount() { - return evictedBlockCount.get(); + return this.evictedBlockCount.get(); } public double getHitRatio() { @@ -210,6 +227,10 @@ public class CacheStats { return Double.isNaN(ratio) ? 0 : ratio; } + public AgeSnapshot getAgeAtEvictionSnapshot() { + return new AgeSnapshot(this.ageAtEviction); + } + private static long sum(long [] counts) { long sum = 0; for (long count : counts) sum += count; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index ad533bf..3124b4e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -460,7 +460,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { map.remove(block.getCacheKey()); updateSizeMetrics(block, true); elements.decrementAndGet(); - stats.evicted(); + stats.evicted(block.getCachedTime()); if (evictedByEvictionProcess && victimHandler != null) { boolean wait = getCurrentSize() < acceptableSize(); boolean inMemory = block.getPriority() == BlockPriority.MEMORY; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index e3a8118..11979be 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -447,7 +447,7 @@ public class BucketCache implements BlockCache, HeapSize { } } } - cacheStats.evicted(); + cacheStats.evicted(bucketEntry == null? 0: bucketEntry.getCachedTime()); return true; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java index cfe9078..bbf385b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java @@ -67,6 +67,11 @@ public class TestBlockCacheReporting { bc.getBlock(bcki, true, false, true); } assertEquals(2 * count /*Data and Index blocks*/, bc.getStats().getHitCount()); + BlockCacheKey bckd = new BlockCacheKey("f", 0); + BlockCacheKey bcki = new BlockCacheKey("f", 0 + count); + bc.evictBlock(bckd); + bc.evictBlock(bcki); + bc.getStats().getEvictedCount(); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java index 1d06a0e..895de0a 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java @@ -288,13 +288,13 @@ public class TestHeapMemoryManager { @Override public boolean evictBlock(BlockCacheKey cacheKey) { - stats.evicted(); + stats.evicted(0); return false; } @Override public int evictBlocksByHfileName(String hfileName) { - stats.evicted(); // Just assuming only one block for file here. + stats.evicted(0); // Just assuming only one block for file here. return 0; }
<% StringUtils.humanReadableInt(cbsbf.getDataSize()) %> Size of DATA Blocks
Evicted<% String.format("%,d", bc.getStats().getEvictedCount()) %>The total number of blocks evicted
Evictions<% String.format("%,d", bc.getStats().getEvictionCount()) %>The total number of times an eviction has occurred
Mean<% String.format("%,d", (long)(snapshot.getMean()/nanosPerSecond)) %>Mean age of Blocks in cache (seconds)
StdDev<% String.format("%,d", (long)(snapshot.getStdDev()/nanosPerSecond)) %>Age standard deviation of Blocks in cache
Min<% String.format("%,d", (long)(snapshot.getMin()/nanosPerSecond)) %>Min age of Blocks in cache (seconds)
Max<% String.format("%,d", (long)(snapshot.getMax()/nanosPerSecond)) %>Max age of Blocks in cache (seconds)
95th Percentile<% String.format("%,d", (long)(snapshot.get95thPercentile()/nanosPerSecond)) %>95th percentile of age of Blocks in cache (seconds)
99th Percentile<% String.format("%,d", (long)(snapshot.get99thPercentile()/nanosPerSecond)) %>99th percentile of age of Blocks in cache (seconds)
Hits per Second