Index: src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java =================================================================== --- src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (revision 1029121) +++ src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java (working copy) @@ -76,7 +76,7 @@ // Confirm empty for(Block block : blocks) { - assertTrue(cache.getBlock(block.blockName) == null); + assertTrue(cache.getBlock(block.blockName, true) == null); } // Add blocks @@ -90,7 +90,7 @@ // Check if all blocks are properly cached and retrieved for(Block block : blocks) { - ByteBuffer buf = cache.getBlock(block.blockName); + ByteBuffer buf = cache.getBlock(block.blockName, true); assertTrue(buf != null); assertEquals(buf.capacity(), block.buf.capacity()); } @@ -110,7 +110,7 @@ // Check if all blocks are properly cached and retrieved for(Block block : blocks) { - ByteBuffer buf = cache.getBlock(block.blockName); + ByteBuffer buf = cache.getBlock(block.blockName, true); assertTrue(buf != null); assertEquals(buf.capacity(), block.buf.capacity()); } @@ -154,10 +154,10 @@ (maxSize * LruBlockCache.DEFAULT_ACCEPTABLE_FACTOR)); // All blocks except block 0 and 1 should be in the cache - assertTrue(cache.getBlock(blocks[0].blockName) == null); - assertTrue(cache.getBlock(blocks[1].blockName) == null); + assertTrue(cache.getBlock(blocks[0].blockName, true) == null); + assertTrue(cache.getBlock(blocks[1].blockName, true) == null); for(int i=2;i 0 && + if (this.extendedPeriod > 0 && this.lastUpdate - this.lastExtUpdate >= this.extendedPeriod) { this.lastExtUpdate = this.lastUpdate; this.compactionTime.resetMinMaxAvg(); @@ -208,7 +214,7 @@ this.flushSize.resetMinMaxAvg(); this.resetAllMinMax(); } - + this.stores.pushMetric(this.metricsRecord); this.storefiles.pushMetric(this.metricsRecord); this.storefileIndexSizeMB.pushMetric(this.metricsRecord); @@ -220,6 +226,7 @@ this.blockCacheFree.pushMetric(this.metricsRecord); this.blockCacheCount.pushMetric(this.metricsRecord); this.blockCacheHitRatio.pushMetric(this.metricsRecord); + this.blockCacheHitCachingRatio.pushMetric(this.metricsRecord); // Mix in HFile and HLog metrics // Be careful. Here is code for MTVR from up in hadoop: @@ -265,10 +272,10 @@ public float getRequests() { return this.requests.getPreviousIntervalValue(); } - + /** * @param compact history in - */ + */ public synchronized void addCompaction(final Pair compact) { this.compactionTime.inc(compact.getFirst()); this.compactionSize.inc(compact.getSecond()); @@ -283,7 +290,7 @@ this.flushSize.inc(f.getSecond()); } } - + /** * @param inc How much to add to requests. */ @@ -328,6 +335,8 @@ Long.valueOf(this.blockCacheCount.get())); sb = Strings.appendKeyValue(sb, this.blockCacheHitRatio.getName(), Long.valueOf(this.blockCacheHitRatio.get())); + sb = Strings.appendKeyValue(sb, this.blockCacheHitCachingRatio.getName(), + Long.valueOf(this.blockCacheHitCachingRatio.get())); return sb.toString(); } } Index: src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (revision 1029121) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java (working copy) @@ -279,13 +279,13 @@ * @param blockName block name * @return buffer of specified block name, or null if not in cache */ - public ByteBuffer getBlock(String blockName) { + public ByteBuffer getBlock(String blockName, boolean caching) { CachedBlock cb = map.get(blockName); if(cb == null) { - stats.miss(); + stats.miss(caching); return null; } - stats.hit(); + stats.hit(caching); cb.access(count.incrementAndGet()); return cb.getBuffer(); } @@ -555,7 +555,11 @@ "blocks=" + size() +", " + "accesses=" + stats.getRequestCount() + ", " + "hits=" + stats.getHitCount() + ", " + - "hitRatio=" + StringUtils.formatPercent(stats.getHitRatio(), 2) + "%, " + + "hitRatio=" + StringUtils.formatPercent(stats.getHitRatio(), 2) + "%, "+ + "cachingAccesses=" + stats.getRequestCachingCount() + ", " + + "cachingHits=" + stats.getHitCachingCount() + ", " + + "cachingHitsRatio=" + + StringUtils.formatPercent(stats.getHitCachingRatio(), 2) + "%, " + "evictions=" + stats.getEvictionCount() + ", " + "evicted=" + stats.getEvictedCount() + ", " + "evictedPerRun=" + stats.evictedPerEviction()); @@ -572,20 +576,35 @@ } public static class CacheStats { - private final AtomicLong accessCount = new AtomicLong(0); + /** The number of getBlock requests that were cache hits */ private final AtomicLong hitCount = new AtomicLong(0); + /** + * The number of getBlock requests that were cache hits, but only from + * requests that were set to use the block cache. This is because all reads + * attempt to read from the block cache even if they will not put new blocks + * into the block cache. See HBASE-2253 for more information. + */ + private final AtomicLong hitCachingCount = new AtomicLong(0); + /** The number of getBlock requests that were cache misses */ private final AtomicLong missCount = new AtomicLong(0); + /** + * The number of getBlock requests that were cache misses, but only from + * requests that were set to use the block cache. + */ + private final AtomicLong missCachingCount = new AtomicLong(0); + /** The number of times an eviction has occurred */ private final AtomicLong evictionCount = new AtomicLong(0); + /** The total number of blocks that have been evicted */ private final AtomicLong evictedCount = new AtomicLong(0); - public void miss() { + public void miss(boolean caching) { missCount.incrementAndGet(); - accessCount.incrementAndGet(); + if (caching) missCachingCount.incrementAndGet(); } - public void hit() { + public void hit(boolean caching) { hitCount.incrementAndGet(); - accessCount.incrementAndGet(); + if (caching) hitCachingCount.incrementAndGet(); } public void evict() { @@ -597,17 +616,29 @@ } public long getRequestCount() { - return accessCount.get(); + return getHitCount() + getMissCount(); } + public long getRequestCachingCount() { + return getHitCachingCount() + getMissCachingCount(); + } + public long getMissCount() { return missCount.get(); } + public long getMissCachingCount() { + return missCachingCount.get(); + } + public long getHitCount() { - return hitCount.get(); + return hitCachingCount.get(); } + public long getHitCachingCount() { + return hitCachingCount.get(); + } + public long getEvictionCount() { return evictionCount.get(); } @@ -620,12 +651,20 @@ return ((float)getHitCount()/(float)getRequestCount()); } + public double getHitCachingRatio() { + return ((float)getHitCachingCount()/(float)getRequestCachingCount()); + } + public double getMissRatio() { return ((float)getMissCount()/(float)getRequestCount()); } + public double getMissCachingRatio() { + return ((float)getMissCachingCount()/(float)getRequestCachingCount()); + } + public double evictedPerEviction() { - return (float)((float)getEvictedCount()/(float)getEvictionCount()); + return ((float)getEvictedCount()/(float)getEvictionCount()); } } Index: src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (revision 1029121) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java (working copy) @@ -950,7 +950,8 @@ metaLoads++; // Check cache for block. If found return. if (cache != null) { - ByteBuffer cachedBuf = cache.getBlock(name + "meta" + block); + ByteBuffer cachedBuf = cache.getBlock(name + "meta" + block, + cacheBlock); if (cachedBuf != null) { // Return a distinct 'shallow copy' of the block, // so pos doesnt get messed by the scanner @@ -1009,7 +1010,7 @@ blockLoads++; // Check cache for block. If found return. if (cache != null) { - ByteBuffer cachedBuf = cache.getBlock(name + block); + ByteBuffer cachedBuf = cache.getBlock(name + block, cacheBlock); if (cachedBuf != null) { // Return a distinct 'shallow copy' of the block, // so pos doesnt get messed by the scanner Index: src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java (revision 1029121) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/SimpleBlockCache.java (working copy) @@ -66,7 +66,7 @@ return cache.size(); } - public synchronized ByteBuffer getBlock(String blockName) { + public synchronized ByteBuffer getBlock(String blockName, boolean caching) { processQueue(); // clear out some crap. Ref ref = cache.get(blockName); if (ref == null) Index: src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java =================================================================== --- src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java (revision 1029121) +++ src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCache.java (working copy) @@ -44,9 +44,10 @@ /** * Fetch block from cache. * @param blockName Block number to fetch. + * @param caching Whether this request has caching enabled (used for stats) * @return Block or null if block is not in the cache. */ - public ByteBuffer getBlock(String blockName); + public ByteBuffer getBlock(String blockName, boolean caching); /** * Shutdown the cache.