diff --git a/conf/hbase-env.sh b/conf/hbase-env.sh index 9059d12..8ce93ed 100644 --- a/conf/hbase-env.sh +++ b/conf/hbase-env.sh @@ -70,6 +70,7 @@ export HBASE_OPTS="-XX:+UseConcMarkSweepGC" # export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=SET_THIS_TO_HOW_MANY_GIGS_OF_OFFHEAP" # For example, to allocate 8G of offheap, set SET_THIS_TO_HOW_MANY_GIGS_OF_OFFHEAP to 8G as in: # export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=8G" +export HBASE_OPTS="$HBASE_OPTS -XX:MaxDirectMemorySize=1G" # See the package documentation for org.apache.hadoop.hbase.io.hfile for other configurations # needed setting up off-heap block caching. diff --git a/conf/hbase-site.xml b/conf/hbase-site.xml index c516ac7..6d4b9b7 100644 --- a/conf/hbase-site.xml +++ b/conf/hbase-site.xml @@ -21,4 +21,17 @@ */ --> + + hfile.block.cache.size + 0.05 + + + hbase.bucketcache.ioengine + offheap + + + hbase.bucketcache.size + 32 + + diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon index fa83fc3..702122a 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon @@ -30,11 +30,13 @@ Configuration config; bcName = bc.getClass().getSimpleName(); } BlockCache [] bcs = cacheConfig == null? null: cacheConfig.getBlockCache() == null? null: cacheConfig.getBlockCache().getBlockCaches(); + // If more than one bc, show evictions in each bc listing so can compare + boolean evictions = bcs != null && bcs.length > 1; <%import> java.util.Map; org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.CachedBlocksByFile; -org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.AgeSnapshot; +org.apache.hadoop.hbase.io.hfile.AgeSnapshot; org.apache.hadoop.hbase.io.hfile.CachedBlock; org.apache.hadoop.conf.Configuration; org.apache.hadoop.hbase.io.hfile.CacheConfig; @@ -64,10 +66,10 @@ org.apache.hadoop.util.StringUtils; <& bc_stats; cacheConfig = cacheConfig &>
- <& bc_l; bc = bcs == null? bc: bcs[0]; name = "L1" &> + <& bc_l; bc = bcs == null? bc: bcs[0]; name = "L1"; evictions = evictions; &>
- <& bc_l; bc = bcs == null? null: bcs.length <= 1? null: bcs[1]; name = "L2" &> + <& bc_l; bc = bcs == null? null: bcs.length <= 1? null: bcs[1]; name = "L2"; evictions = evictions; &>
@@ -168,6 +170,42 @@ org.apache.hadoop.util.StringUtils; +<%def evictions_tmpl> +<%args> + BlockCache bc; + +<%java> + AgeSnapshot ageAtEvictionSnapshot = bc.getStats().getAgeAtEvictionSnapshot(); + // Only show if non-zero mean and stddev as is the case in combinedblockcache + double mean = ageAtEvictionSnapshot.getMean(); + double stddev = ageAtEvictionSnapshot.getStdDev(); + + + Evicted + <% String.format("%,d", bc.getStats().getEvictedCount()) %> + The total number of blocks evicted + + + Evictions + <% String.format("%,d", bc.getStats().getEvictionCount()) %> + The total number of times an eviction has occurred + +<%if mean > 0 %> + + Mean + <% String.format("%,d", (long)(ageAtEvictionSnapshot.getMean()/(1000000 * 1000))) %> + Mean age of Blocks at eviction time (seconds) + + +<%if stddev > 0 %> + + StdDev + <% String.format("%,d", (long)(ageAtEvictionSnapshot.getStdDev()/1000000)) %> + Standard Deviation for age of Blocks at eviction time + + + + <%def bc_stats> <%args> CacheConfig cacheConfig; @@ -196,16 +234,7 @@ org.apache.hadoop.util.StringUtils; <% String.format("%,d", cacheConfig.getBlockCache().getBlockCount()) %> Number of blocks in block cache - - Evicted - <% String.format("%,d", cacheConfig.getBlockCache().getStats().getEvictedCount()) %> - The total number of blocks evicted - - - Evictions - <% String.format("%,d", cacheConfig.getBlockCache().getStats().getEvictionCount()) %> - The total number of times an eviction has occurred - + <& evictions_tmpl; bc = cacheConfig.getBlockCache(); &> Hits <% String.format("%,d", cacheConfig.getBlockCache().getStats().getHitCount()) %> @@ -241,11 +270,12 @@ are combined counts. Request count is sum of hits and misses.

<%args> BlockCache bc; String name; + boolean evictions; <%if bc == null %>

No <% name %> deployed

<%else> -<& block_cache; bc = bc; name = name; &> +<& block_cache; bc = bc; name = name; evictions = evictions; &> @@ -253,6 +283,7 @@ are combined counts. Request count is sum of hits and misses.

<%args> BlockCache bc; String name; + boolean evictions; <%java> final long nanosPerSecond = 1000000000; @@ -260,7 +291,7 @@ are combined counts. Request count is sum of hits and misses.

String bcName = bc.getClass().getSimpleName(); org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.CachedBlocksByFile cbsbf = org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.getLoadedCachedBlocksByFile(config, bc); - AgeSnapshot snapshot = cbsbf.getAgeSnapshot(); + AgeSnapshot cbsbfSnapshot = cbsbf.getAgeInCacheSnapshot(); boolean bucketCache = bc.getClass().getSimpleName().equals("BucketCache"); BucketCacheStats bucketCacheStats = null; @@ -274,7 +305,7 @@ are combined counts. Request count is sum of hits and misses.

} <%if cbsbf.isFull() %> -

Too many blocks! Listing out the first <% snapshot.getMax() %> only (hbase.ui.blockcache.by.file.max)

+

Too many blocks! Listing out the first <% cbsbfSnapshot.getMax() %> only (hbase.ui.blockcache.by.file.max)

@@ -317,49 +348,8 @@ are combined counts. Request count is sum of hits and misses.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +<%if evictions %><& evictions_tmpl; bc = bc; &> <%if bucketCache %> diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon index 184076e..c5002b5 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon @@ -53,7 +53,7 @@ com.yammer.metrics.stats.Snapshot; } CachedBlocksByFile cbsbf = BlockCacheUtil.getLoadedCachedBlocksByFile(conf, bc); -<%if bcv.equals("file") %><& bc_by_file; cbsbf = cbsbf; &><%else>{<% BlockCacheUtil.toJSON(bc) %>, <% cbsbf %> } +<%if bcv.equals("file") %><& bc_by_file; cbsbf = cbsbf; &><%else>[ <% BlockCacheUtil.toJSON(bc) %>, <% BlockCacheUtil.toJSON(cbsbf) %> ] <%java> cbsbf = null; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java new file mode 100644 index 0000000..24a4e32 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.io.hfile; + +import org.codehaus.jackson.annotate.JsonIgnoreProperties; + +import com.yammer.metrics.core.Histogram; +import com.yammer.metrics.stats.Snapshot; + +/** + * Snapshot of block cache age in cache. + * This object is preferred because we can control how it is serialized out when JSON'ing. + */ +@JsonIgnoreProperties({"ageHistogram", "snapshot"}) +public class AgeSnapshot { + private final Histogram ageHistogram; + private final Snapshot snapshot; + + AgeSnapshot(final Histogram ageHistogram) { + this.ageHistogram = ageHistogram; + this.snapshot = ageHistogram.getSnapshot(); + } + + public double get75thPercentile() { + return snapshot.get75thPercentile(); + } + + public double get95thPercentile() { + return snapshot.get95thPercentile(); + } + + public double get98thPercentile() { + return snapshot.get98thPercentile(); + } + + public double get999thPercentile() { + return snapshot.get999thPercentile(); + } + + public double get99thPercentile() { + return snapshot.get99thPercentile(); + } + + public double getMean() { + return this.ageHistogram.mean(); + } + + public double getMax() { + return ageHistogram.max(); + } + + public double getMin() { + return ageHistogram.min(); + } + + public double getStdDev() { + return ageHistogram.stdDev(); + } +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index d146084..f793d4b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -160,57 +160,6 @@ public class BlockCacheUtil { } /** - * Snapshot of block cache age in cache. - * This object is preferred because we can control how it is serialized out when JSON'ing. - */ - @JsonIgnoreProperties({"ageHistogram", "snapshot"}) - public static class AgeSnapshot { - private final Histogram ageHistogram; - private final Snapshot snapshot; - - AgeSnapshot(final Histogram ageHistogram) { - this.ageHistogram = ageHistogram; - this.snapshot = ageHistogram.getSnapshot(); - } - - public double get75thPercentile() { - return snapshot.get75thPercentile(); - } - - public double get95thPercentile() { - return snapshot.get95thPercentile(); - } - - public double get98thPercentile() { - return snapshot.get98thPercentile(); - } - - public double get999thPercentile() { - return snapshot.get999thPercentile(); - } - - public double get99thPercentile() { - return snapshot.get99thPercentile(); - } - - public double getMean() { - return this.ageHistogram.mean(); - } - - public double getMax() { - return ageHistogram.max(); - } - - public double getMin() { - return ageHistogram.min(); - } - - public double getStdDev() { - return ageHistogram.stdDev(); - } - } - - /** * Get a {@link CachedBlocksByFile} instance and load it up by iterating content in * {@link BlockCache}. * @param conf Used to read configurations @@ -319,7 +268,7 @@ public class BlockCacheUtil { return dataSize; } - public AgeSnapshot getAgeSnapshot() { + public AgeSnapshot getAgeInCacheSnapshot() { return new AgeSnapshot(this.age); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java index 4674eff..b881c2e 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java @@ -22,11 +22,19 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.classification.InterfaceAudience; +import com.yammer.metrics.core.Histogram; +import com.yammer.metrics.core.MetricsRegistry; + /** * Class that implements cache metrics. */ @InterfaceAudience.Private public class CacheStats { + /** + * Needed making histograms. + */ + private static final MetricsRegistry METRICS = new MetricsRegistry(); + /** Sliding window statistics. The number of metric periods to include in * sliding window hit ratio calculations. */ @@ -78,25 +86,34 @@ public class CacheStats { private long lastRequestCachingCount = 0; /** Current window index (next to be updated) */ private int windowIndex = 0; + /** + * Keep running age at eviction time + */ + private Histogram ageAtEviction; + private long startTime = System.nanoTime(); - public CacheStats() { - this(DEFAULT_WINDOW_PERIODS); + public CacheStats(final String name) { + this(name, DEFAULT_WINDOW_PERIODS); } - public CacheStats(int numPeriodsInWindow) { + public CacheStats(final String name, int numPeriodsInWindow) { this.numPeriodsInWindow = numPeriodsInWindow; this.hitCounts = initializeZeros(numPeriodsInWindow); this.hitCachingCounts = initializeZeros(numPeriodsInWindow); this.requestCounts = initializeZeros(numPeriodsInWindow); this.requestCachingCounts = initializeZeros(numPeriodsInWindow); + this.ageAtEviction = METRICS.newHistogram(CacheStats.class, name + ".ageAtEviction"); } @Override public String toString() { + AgeSnapshot snapshot = getAgeAtEvictionSnapshot(); return "hitCount=" + getHitCount() + ", hitCachingCount=" + getHitCachingCount() + ", missCount=" + getMissCount() + ", missCachingCount=" + getMissCachingCount() + ", evictionCount=" + getEvictionCount() + - ", evictedBlockCount=" + getEvictedCount(); + ", evictedBlockCount=" + getEvictedCount() + + ", evictedAgeMean=" + snapshot.getMean() + + ", evictedAgeStdDev=" + snapshot.getStdDev(); } public void miss(boolean caching) { @@ -113,8 +130,11 @@ public class CacheStats { evictionCount.incrementAndGet(); } - public void evicted() { - evictedBlockCount.incrementAndGet(); + public void evicted(final long t) { + long duration = t - this.startTime; + if (duration < 0) throw new IllegalStateException("t=" + t + ", startTime=" + this.startTime); + this.ageAtEviction.update(duration); + this.evictedBlockCount.incrementAndGet(); } public long getRequestCount() { @@ -146,7 +166,7 @@ public class CacheStats { } public long getEvictedCount() { - return evictedBlockCount.get(); + return this.evictedBlockCount.get(); } public double getHitRatio() { @@ -210,6 +230,10 @@ public class CacheStats { return Double.isNaN(ratio) ? 0 : ratio; } + public AgeSnapshot getAgeAtEvictionSnapshot() { + return new AgeSnapshot(this.ageAtEviction); + } + private static long sum(long [] counts) { long sum = 0; for (long count : counts) sum += count; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 84f8ea3..c89073d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -127,6 +127,7 @@ public class CombinedBlockCache implements BlockCache, HeapSize { private final CacheStats bucketCacheStats; CombinedCacheStats(CacheStats lbcStats, CacheStats fcStats) { + super("CombinedBlockCache"); this.lruCacheStats = lbcStats; this.bucketCacheStats = fcStats; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index ad533bf..a5ffaa2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -285,7 +285,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { this.singleFactor = singleFactor; this.multiFactor = multiFactor; this.memoryFactor = memoryFactor; - this.stats = new CacheStats(); + this.stats = new CacheStats(this.getClass().getSimpleName()); this.count = new AtomicLong(0); this.elements = new AtomicLong(0); this.overhead = calculateOverhead(maxSize, blockSize, mapConcurrencyLevel); @@ -460,7 +460,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { map.remove(block.getCacheKey()); updateSizeMetrics(block, true); elements.decrementAndGet(); - stats.evicted(); + stats.evicted(block.getCachedTime()); if (evictedByEvictionProcess && victimHandler != null) { boolean wait = getCurrentSize() < acceptableSize(); boolean inMemory = block.getPriority() == BlockPriority.MEMORY; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index e3a8118..11979be 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -447,7 +447,7 @@ public class BucketCache implements BlockCache, HeapSize { } } } - cacheStats.evicted(); + cacheStats.evicted(bucketEntry == null? 0: bucketEntry.getCachedTime()); return true; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java index cffe905..5a290bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCacheStats.java @@ -34,6 +34,10 @@ public class BucketCacheStats extends CacheStats { private final static int nanoTime = 1000000; private long lastLogTime = EnvironmentEdgeManager.currentTimeMillis(); + BucketCacheStats() { + super("BucketCache"); + } + @Override public String toString() { return super.toString() + ", ioHitsPerSecond=" + getIOHitsPerSecond() + diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java index cfe9078..bbf385b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java @@ -67,6 +67,11 @@ public class TestBlockCacheReporting { bc.getBlock(bcki, true, false, true); } assertEquals(2 * count /*Data and Index blocks*/, bc.getStats().getHitCount()); + BlockCacheKey bckd = new BlockCacheKey("f", 0); + BlockCacheKey bcki = new BlockCacheKey("f", 0 + count); + bc.evictBlock(bckd); + bc.evictBlock(bcki); + bc.getStats().getEvictedCount(); } @Test diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java index fa48676..d3854b2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java @@ -607,7 +607,7 @@ public class TestLruBlockCache { double delta = 0.01; // 3 total periods - CacheStats stats = new CacheStats(3); + CacheStats stats = new CacheStats("test"); // No accesses, should be 0 stats.rollMetricsPeriod(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java index 1d06a0e..9c0cbf8 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java @@ -261,8 +261,7 @@ public class TestHeapMemoryManager { } private static class BlockCacheStub implements ResizableBlockCache { - - CacheStats stats = new CacheStats(); + CacheStats stats = new CacheStats("test"); long maxSize = 0; public BlockCacheStub(long size){ @@ -288,13 +287,13 @@ public class TestHeapMemoryManager { @Override public boolean evictBlock(BlockCacheKey cacheKey) { - stats.evicted(); + stats.evicted(0); return false; } @Override public int evictBlocksByHfileName(String hfileName) { - stats.evicted(); // Just assuming only one block for file here. + stats.evicted(0); // Just assuming only one block for file here. return 0; }
<% StringUtils.humanReadableInt(cbsbf.getDataSize()) %> Size of DATA Blocks
Evicted<% String.format("%,d", bc.getStats().getEvictedCount()) %>The total number of blocks evicted
Evictions<% String.format("%,d", bc.getStats().getEvictionCount()) %>The total number of times an eviction has occurred
Mean<% String.format("%,d", (long)(snapshot.getMean()/nanosPerSecond)) %>Mean age of Blocks in cache (seconds)
StdDev<% String.format("%,d", (long)(snapshot.getStdDev()/nanosPerSecond)) %>Age standard deviation of Blocks in cache
Min<% String.format("%,d", (long)(snapshot.getMin()/nanosPerSecond)) %>Min age of Blocks in cache (seconds)
Max<% String.format("%,d", (long)(snapshot.getMax()/nanosPerSecond)) %>Max age of Blocks in cache (seconds)
95th Percentile<% String.format("%,d", (long)(snapshot.get95thPercentile()/nanosPerSecond)) %>95th percentile of age of Blocks in cache (seconds)
99th Percentile<% String.format("%,d", (long)(snapshot.get99thPercentile()/nanosPerSecond)) %>99th percentile of age of Blocks in cache (seconds)
Hits per Second