diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh
index 9059d12..8ce93ed 100644
--- a/conf/hbase-env.sh
+++ b/conf/hbase-env.sh
@@ -70,6 +70,7 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC"
# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=SET_THIS_TO_HOW_MANY_GIGS_OF_OFFHEAP"
# For example, to allocate 8G of offheap, set SET_THIS_TO_HOW_MANY_GIGS_OF_OFFHEAP to 8G as in:
# export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=8G"
+export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=1G"
# See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations
# needed setting up off-heap block caching.
diff --git a/conf/hbase-site.xml b/conf/hbase-site.xml
index c516ac7..6d4b9b7 100644
--- a/conf/hbase-site.xml
+++ b/conf/hbase-site.xml
@@ -21,4 +21,17 @@
*/
-->
No <% name %> deployed
<%else> -<& block_cache; bc = bc; name = name; &> +<& block_cache; bc = bc; name = name; evictions = evictions; &> %if> %def> @@ -253,6 +283,7 @@ are combined counts. Request count is sum of hits and misses. <%args> BlockCache bc; String name; + boolean evictions; %args> <%java> final long nanosPerSecond = 1000000000; @@ -260,7 +291,7 @@ are combined counts. Request count is sum of hits and misses. String bcName = bc.getClass().getSimpleName(); org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.CachedBlocksByFile cbsbf = org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.getLoadedCachedBlocksByFile(config, bc); - AgeSnapshot snapshot = cbsbf.getAgeSnapshot(); + AgeSnapshot cbsbfSnapshot = cbsbf.getAgeInCacheSnapshot(); boolean bucketCache = bc.getClass().getSimpleName().equals("BucketCache"); BucketCacheStats bucketCacheStats = null; @@ -274,7 +305,7 @@ are combined counts. Request count is sum of hits and misses. } %java> <%if cbsbf.isFull() %> -Too many blocks! Listing out the first <% snapshot.getMax() %> only (hbase.ui.blockcache.by.file.max)
+Too many blocks! Listing out the first <% cbsbfSnapshot.getMax() %> only (hbase.ui.blockcache.by.file.max)
%if>| <% StringUtils.humanReadableInt(cbsbf.getDataSize()) %> | Size of DATA Blocks | |
| Evicted | -<% String.format("%,d", bc.getStats().getEvictedCount()) %> | -The total number of blocks evicted | -
| Evictions | -<% String.format("%,d", bc.getStats().getEvictionCount()) %> | -The total number of times an eviction has occurred | -
| Mean | -<% String.format("%,d", (long)(snapshot.getMean()/nanosPerSecond)) %> | -Mean age of Blocks in cache (seconds) | -
| StdDev | -<% String.format("%,d", (long)(snapshot.getStdDev()/nanosPerSecond)) %> | -Age standard deviation of Blocks in cache | -
| Min | -<% String.format("%,d", (long)(snapshot.getMin()/nanosPerSecond)) %> | -Min age of Blocks in cache (seconds) | -
| Max | -<% String.format("%,d", (long)(snapshot.getMax()/nanosPerSecond)) %> | -Max age of Blocks in cache (seconds) | -
| 95th Percentile | -<% String.format("%,d", (long)(snapshot.get95thPercentile()/nanosPerSecond)) %> | -95th percentile of age of Blocks in cache (seconds) | -
| 99th Percentile | -<% String.format("%,d", (long)(snapshot.get99thPercentile()/nanosPerSecond)) %> | -99th percentile of age of Blocks in cache (seconds) | -
| Hits per Second | diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon index 184076e..c5002b5 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon @@ -53,7 +53,7 @@ com.yammer.metrics.stats.Snapshot; } CachedBlocksByFile cbsbf = BlockCacheUtil.getLoadedCachedBlocksByFile(conf, bc); %java> -<%if bcv.equals("file") %><& bc_by_file; cbsbf = cbsbf; &><%else>{<% BlockCacheUtil.toJSON(bc) %>, <% cbsbf %> }%if> +<%if bcv.equals("file") %><& bc_by_file; cbsbf = cbsbf; &><%else>[ <% BlockCacheUtil.toJSON(bc) %>, <% BlockCacheUtil.toJSON(cbsbf) %> ]%if> <%java> cbsbf = null; %java> diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java new file mode 100644 index 0000000..24a4e32 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; + +import com.yammer.metrics.core.Histogram; +import com.yammer.metrics.stats.Snapshot; + +/** + * Snapshot of block cache age in cache. + * This object is preferred because we can control how it is serialized out when JSON'ing. + */ +@JsonIgnoreProperties({"ageHistogram", "snapshot"}) +public class AgeSnapshot { + private final Histogram ageHistogram; + private final Snapshot snapshot; + + AgeSnapshot(final Histogram ageHistogram) { + this.ageHistogram = ageHistogram; + this.snapshot = ageHistogram.getSnapshot(); + } + + public double get75thPercentile() { + return snapshot.get75thPercentile(); + } + + public double get95thPercentile() { + return snapshot.get95thPercentile(); + } + + public double get98thPercentile() { + return snapshot.get98thPercentile(); + } + + public double get999thPercentile() { + return snapshot.get999thPercentile(); + } + + public double get99thPercentile() { + return snapshot.get99thPercentile(); + } + + public double getMean() { + return this.ageHistogram.mean(); + } + + public double getMax() { + return ageHistogram.max(); + } + + public double getMin() { + return ageHistogram.min(); + } + + public double getStdDev() { + return ageHistogram.stdDev(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index d146084..f793d4b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -160,57 +160,6 @@ public class BlockCacheUtil { } /** - * Snapshot of block cache age in cache. - * This object is preferred because we can control how it is serialized out when JSON'ing. - */ - @JsonIgnoreProperties({"ageHistogram", "snapshot"}) - public static class AgeSnapshot { - private final Histogram ageHistogram; - private final Snapshot snapshot; - - AgeSnapshot(final Histogram ageHistogram) { - this.ageHistogram = ageHistogram; - this.snapshot = ageHistogram.getSnapshot(); - } - - public double get75thPercentile() { - return snapshot.get75thPercentile(); - } - - public double get95thPercentile() { - return snapshot.get95thPercentile(); - } - - public double get98thPercentile() { - return snapshot.get98thPercentile(); - } - - public double get999thPercentile() { - return snapshot.get999thPercentile(); - } - - public double get99thPercentile() { - return snapshot.get99thPercentile(); - } - - public double getMean() { - return this.ageHistogram.mean(); - } - - public double getMax() { - return ageHistogram.max(); - } - - public double getMin() { - return ageHistogram.min(); - } - - public double getStdDev() { - return ageHistogram.stdDev(); - } - } - - /** * Get a {@link CachedBlocksByFile} instance and load it up by iterating content in * {@link BlockCache}. * @param conf Used to read configurations @@ -319,7 +268,7 @@ public class BlockCacheUtil { return dataSize; } - public AgeSnapshot getAgeSnapshot() { + public AgeSnapshot getAgeInCacheSnapshot() { return new AgeSnapshot(this.age); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java index 4674eff..b881c2e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java @@ -22,11 +22,19 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.classification.InterfaceAudience; +import com.yammer.metrics.core.Histogram; +import com.yammer.metrics.core.MetricsRegistry; + /** * Class that implements cache metrics. */ @InterfaceAudience.Private public class CacheStats { + /** + * Needed making histograms. + */ + private static final MetricsRegistry METRICS = new MetricsRegistry(); + /** Sliding window statistics. The number of metric periods to include in * sliding window hit ratio calculations. */ @@ -78,25 +86,34 @@ public class CacheStats { private long lastRequestCachingCount = 0; /** Current window index (next to be updated) */ private int windowIndex = 0; + /** + * Keep running age at eviction time + */ + private Histogram ageAtEviction; + private long startTime = System.nanoTime(); - public CacheStats() { - this(DEFAULT_WINDOW_PERIODS); + public CacheStats(final String name) { + this(name, DEFAULT_WINDOW_PERIODS); } - public CacheStats(int numPeriodsInWindow) { + public CacheStats(final String name, int numPeriodsInWindow) { this.numPeriodsInWindow = numPeriodsInWindow; this.hitCounts = initializeZeros(numPeriodsInWindow); this.hitCachingCounts = initializeZeros(numPeriodsInWindow); this.requestCounts = initializeZeros(numPeriodsInWindow); this.requestCachingCounts = initializeZeros(numPeriodsInWindow); + this.ageAtEviction = METRICS.newHistogram(CacheStats.class, name + ".ageAtEviction"); } @Override public String toString() { + AgeSnapshot snapshot = getAgeAtEvictionSnapshot(); return "hitCount=" + getHitCount() + ", hitCachingCount=" + getHitCachingCount() + ", missCount=" + getMissCount() + ", missCachingCount=" + getMissCachingCount() + ", evictionCount=" + getEvictionCount() + - ", evictedBlockCount=" + getEvictedCount(); + ", evictedBlockCount=" + getEvictedCount() + + ", evictedAgeMean=" + snapshot.getMean() + + ", evictedAgeStdDev=" + snapshot.getStdDev(); } public void miss(boolean caching) { @@ -113,8 +130,11 @@ public class CacheStats { evictionCount.incrementAndGet(); } - public void evicted() { - evictedBlockCount.incrementAndGet(); + public void evicted(final long t) { + long duration = t - this.startTime; + if (duration < 0) throw new IllegalStateException("t=" + t + ", startTime=" + this.startTime); + this.ageAtEviction.update(duration); + this.evictedBlockCount.incrementAndGet(); } public long getRequestCount() { @@ -146,7 +166,7 @@ public class CacheStats { } public long getEvictedCount() { - return evictedBlockCount.get(); + return this.evictedBlockCount.get(); } public double getHitRatio() { @@ -210,6 +230,10 @@ public class CacheStats { return Double.isNaN(ratio) ? 0 : ratio; } + public AgeSnapshot getAgeAtEvictionSnapshot() { + return new AgeSnapshot(this.ageAtEviction); + } + private static long sum(long [] counts) { long sum = 0; for (long count : counts) sum += count; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 84f8ea3..c89073d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -127,6 +127,7 @@ public class CombinedBlockCache implements BlockCache, HeapSize { private final CacheStats bucketCacheStats; CombinedCacheStats(CacheStats lbcStats, CacheStats fcStats) { + super("CombinedBlockCache"); this.lruCacheStats = lbcStats; this.bucketCacheStats = fcStats; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index ad533bf..a5ffaa2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -285,7 +285,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { this.singleFactor = singleFactor; this.multiFactor = multiFactor; this.memoryFactor = memoryFactor; - this.stats = new CacheStats(); + this.stats = new CacheStats(this.getClass().getSimpleName()); this.count = new AtomicLong(0); this.elements = new AtomicLong(0); this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel); @@ -460,7 +460,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { map.remove(block.getCacheKey()); updateSizeMetrics(block, true); elements.decrementAndGet(); - stats.evicted(); + stats.evicted(block.getCachedTime()); if (evictedByEvictionProcess && victimHandler != null) { boolean wait = getCurrentSize() < acceptableSize(); boolean inMemory = block.getPriority() == BlockPriority.MEMORY; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index e3a8118..11979be 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -447,7 +447,7 @@ public class BucketCache implements BlockCache, HeapSize { } } } - cacheStats.evicted(); + cacheStats.evicted(bucketEntry == null? 0: bucketEntry.getCachedTime()); return true; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java index cffe905..5a290bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java @@ -34,6 +34,10 @@ public class BucketCacheStats extends CacheStats { private final static int nanoTime = 1000000; private long lastLogTime = EnvironmentEdgeManager.currentTimeMillis(); + BucketCacheStats() { + super("BucketCache"); + } + @Override public String toString() { return super.toString() + ", ioHitsPerSecond=" + getIOHitsPerSecond() + diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java index cfe9078..bbf385b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java @@ -67,6 +67,11 @@ public class TestBlockCacheReporting { bc.getBlock(bcki, true, false, true); } assertEquals(2 * count /*Data and Index blocks*/, bc.getStats().getHitCount()); + BlockCacheKey bckd = new BlockCacheKey("f", 0); + BlockCacheKey bcki = new BlockCacheKey("f", 0 + count); + bc.evictBlock(bckd); + bc.evictBlock(bcki); + bc.getStats().getEvictedCount(); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java index fa48676..d3854b2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java @@ -607,7 +607,7 @@ public class TestLruBlockCache { double delta = 0.01; // 3 total periods - CacheStats stats = new CacheStats(3); + CacheStats stats = new CacheStats("test"); // No accesses, should be 0 stats.rollMetricsPeriod(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java index 1d06a0e..9c0cbf8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java @@ -261,8 +261,7 @@ public class TestHeapMemoryManager { } private static class BlockCacheStub implements ResizableBlockCache { - - CacheStats stats = new CacheStats(); + CacheStats stats = new CacheStats("test"); long maxSize = 0; public BlockCacheStub(long size){ @@ -288,13 +287,13 @@ public class TestHeapMemoryManager { @Override public boolean evictBlock(BlockCacheKey cacheKey) { - stats.evicted(); + stats.evicted(0); return false; } @Override public int evictBlocksByHfileName(String hfileName) { - stats.evicted(); // Just assuming only one block for file here. + stats.evicted(0); // Just assuming only one block for file here. return 0; }