diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java index 9961d63..1759ab3 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSource.java @@ -207,12 +207,20 @@ public interface MetricsRegionServerSource extends BaseSource { String BLOCK_CACHE_SIZE_DESC = "Size of the block cache."; String BLOCK_CACHE_HIT_COUNT = "blockCacheHitCount"; String BLOCK_CACHE_HIT_COUNT_DESC = "Count of the hit on the block cache."; + String BLOCK_CACHE_PRIMARY_HIT_COUNT = "blockCacheHitCountPrimary"; + String BLOCK_CACHE_PRIMARY_HIT_COUNT_DESC = "Count of hit on primary replica in the block cache."; String BLOCK_CACHE_MISS_COUNT = "blockCacheMissCount"; String BLOCK_COUNT_MISS_COUNT_DESC = "Number of requests for a block that missed the block cache."; + String BLOCK_CACHE_PRIMARY_MISS_COUNT = "blockCacheMissCountPrimary"; + String BLOCK_COUNT_PRIMARY_MISS_COUNT_DESC = + "Number of requests for a block of primary replica that missed the block cache."; String BLOCK_CACHE_EVICTION_COUNT = "blockCacheEvictionCount"; String BLOCK_CACHE_EVICTION_COUNT_DESC = "Count of the number of blocks evicted from the block cache."; + String BLOCK_CACHE_PRIMARY_EVICTION_COUNT = "blockCacheEvictionCountPrimary"; + String BLOCK_CACHE_PRIMARY_EVICTION_COUNT_DESC = + "Count of the number of blocks evicted from primary replica in the block cache."; String BLOCK_CACHE_HIT_PERCENT = "blockCacheCountHitPercent"; String BLOCK_CACHE_HIT_PERCENT_DESC = "Percent of block cache requests that are hits"; diff --git a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java index 1cc7751..885d3a6 100644 --- a/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java +++ b/hbase-hadoop-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapper.java @@ -198,16 +198,32 @@ public interface MetricsRegionServerWrapper { long getBlockCacheHitCount(); /** + * Get the count of hits to primary replica in the block cache + */ + long getBlockCachePrimaryHitCount(); + + /** * Get the count of misses to the block cache. */ long getBlockCacheMissCount(); /** + * Get the count of misses to primary replica in the block cache. + */ + long getBlockCachePrimaryMissCount(); + + /** * Get the number of items evicted from the block cache. */ long getBlockCacheEvictedCount(); /** + * Get the number of items evicted from primary replica in the block cache. + */ + long getBlockCachePrimaryEvictedCount(); + + + /** * Get the percent of all requests that hit the block cache. */ double getBlockCacheHitPercent(); diff --git a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java index de629c8..3d2e0f1 100644 --- a/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java +++ b/hbase-hadoop2-compat/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerSourceImpl.java @@ -236,10 +236,16 @@ public class MetricsRegionServerSourceImpl rsWrap.getBlockCacheSize()) .addCounter(Interns.info(BLOCK_CACHE_HIT_COUNT, BLOCK_CACHE_HIT_COUNT_DESC), rsWrap.getBlockCacheHitCount()) + .addCounter(Interns.info(BLOCK_CACHE_PRIMARY_HIT_COUNT, + BLOCK_CACHE_PRIMARY_HIT_COUNT_DESC), rsWrap.getBlockCachePrimaryHitCount()) .addCounter(Interns.info(BLOCK_CACHE_MISS_COUNT, BLOCK_COUNT_MISS_COUNT_DESC), rsWrap.getBlockCacheMissCount()) + .addCounter(Interns.info(BLOCK_CACHE_PRIMARY_MISS_COUNT, + BLOCK_COUNT_PRIMARY_MISS_COUNT_DESC), rsWrap.getBlockCachePrimaryMissCount()) .addCounter(Interns.info(BLOCK_CACHE_EVICTION_COUNT, BLOCK_CACHE_EVICTION_COUNT_DESC), rsWrap.getBlockCacheEvictedCount()) + .addCounter(Interns.info(BLOCK_CACHE_PRIMARY_EVICTION_COUNT, + BLOCK_CACHE_PRIMARY_EVICTION_COUNT_DESC), rsWrap.getBlockCachePrimaryEvictedCount()) .addGauge(Interns.info(BLOCK_CACHE_HIT_PERCENT, BLOCK_CACHE_HIT_PERCENT_DESC), rsWrap.getBlockCacheHitPercent()) .addGauge(Interns.info(BLOCK_CACHE_EXPRESS_HIT_PERCENT, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java index 8c1e7b9..99af201 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java @@ -58,6 +58,8 @@ public abstract class AbstractHFileReader /** Filled when we read in the trailer. */ protected final Compression.Algorithm compressAlgo; + private boolean isPrimaryReplicaReader; + /** * What kind of data block encoding should be used while reading, writing, * and handling cache. @@ -251,6 +253,15 @@ public abstract class AbstractHFileReader } @Override + public boolean isPrimaryReplicaReader() { + return isPrimaryReplicaReader; + } + + @Override + public void setPrimaryReplicaReader(boolean isPrimaryReplicaReader) { + this.isPrimaryReplicaReader = isPrimaryReplicaReader; + } + public FileInfo loadFileInfo() throws IOException { return fileInfo; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java index f918bdd..180cbb4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheKey.java @@ -30,6 +30,7 @@ public class BlockCacheKey implements HeapSize, java.io.Serializable { private static final long serialVersionUID = -5199992013113130534L; private final String hfileName; private final long offset; + private final boolean isPrimaryReplicaBlock; /** * Construct a new BlockCacheKey @@ -37,6 +38,11 @@ public class BlockCacheKey implements HeapSize, java.io.Serializable { * @param offset Offset of the block into the file */ public BlockCacheKey(String hfileName, long offset) { + this(hfileName, offset, true); + } + + public BlockCacheKey(String hfileName, long offset, boolean isPrimaryReplica) { + this.isPrimaryReplicaBlock = isPrimaryReplica; this.hfileName = hfileName; this.offset = offset; } @@ -63,7 +69,7 @@ public class BlockCacheKey implements HeapSize, java.io.Serializable { return String.format("%s_%d", hfileName, offset); } - public static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT + + public static final long FIXED_OVERHEAD = ClassSize.align(ClassSize.OBJECT +Bytes.SIZEOF_BOOLEAN + ClassSize.REFERENCE + // this.hfileName Bytes.SIZEOF_LONG); // this.offset @@ -85,6 +91,10 @@ public class BlockCacheKey implements HeapSize, java.io.Serializable { return hfileName; } + public boolean isPrimary() { + return isPrimaryReplicaBlock; + } + public long getOffset() { return offset; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java index 4467fee..9301de2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java @@ -43,6 +43,9 @@ public class CacheStats { /** The number of getBlock requests that were cache hits */ private final AtomicLong hitCount = new AtomicLong(0); + /** The number of getBlock requests that were cache hits from primary replica */ + private final AtomicLong primaryHitCount = new AtomicLong(0); + /** * The number of getBlock requests that were cache hits, but only from * requests that were set to use the block cache. This is because all reads @@ -54,6 +57,8 @@ public class CacheStats { /** The number of getBlock requests that were cache misses */ private final AtomicLong missCount = new AtomicLong(0); + /** The number of getBlock requests for primary replica that were cache misses */ + private final AtomicLong primaryMissCount = new AtomicLong(0); /** * The number of getBlock requests that were cache misses, but only from * requests that were set to use the block cache. @@ -66,6 +71,9 @@ public class CacheStats { /** The total number of blocks that have been evicted */ private final AtomicLong evictedBlockCount = new AtomicLong(0); + /** The total number of blocks for primary replica that have been evicted */ + private final AtomicLong primaryEvictedBlockCount = new AtomicLong(0); + /** The number of metrics periods to include in window */ private final int numPeriodsInWindow; /** Hit counts for each period in window */ @@ -112,17 +120,25 @@ public class CacheStats { ", missCount=" + getMissCount() + ", missCachingCount=" + getMissCachingCount() + ", evictionCount=" + getEvictionCount() + ", evictedBlockCount=" + getEvictedCount() + + ", primaryMissCount=" + getPrimaryMissCount() + + ", primaryHitCount=" + getPrimaryHitCount() + ", evictedAgeMean=" + snapshot.getMean() + ", evictedAgeStdDev=" + snapshot.getStdDev(); } - public void miss(boolean caching) { + public void miss(boolean caching, boolean primary) { missCount.incrementAndGet(); + if (primary) primaryMissCount.incrementAndGet(); if (caching) missCachingCount.incrementAndGet(); } public void hit(boolean caching) { + hit(caching, true); + } + + public void hit(boolean caching, boolean primary) { hitCount.incrementAndGet(); + if (primary) primaryHitCount.incrementAndGet(); if (caching) hitCachingCount.incrementAndGet(); } @@ -130,9 +146,12 @@ public class CacheStats { evictionCount.incrementAndGet(); } - public void evicted(final long t) { + public void evicted(final long t, boolean primary) { if (t > this.startTime) this.ageAtEviction.update(t - this.startTime); this.evictedBlockCount.incrementAndGet(); + if (primary) { + primaryEvictedBlockCount.incrementAndGet(); + } } public long getRequestCount() { @@ -147,6 +166,10 @@ public class CacheStats { return missCount.get(); } + public long getPrimaryMissCount() { + return primaryMissCount.get(); + } + public long getMissCachingCount() { return missCachingCount.get(); } @@ -155,6 +178,10 @@ public class CacheStats { return hitCount.get(); } + public long getPrimaryHitCount() { + return primaryHitCount.get(); + } + public long getHitCachingCount() { return hitCachingCount.get(); } @@ -167,6 +194,10 @@ public class CacheStats { return this.evictedBlockCount.get(); } + public long getPrimaryEvictedCount() { + return primaryEvictedBlockCount.get(); + } + public double getHitRatio() { return ((float)getHitCount()/(float)getRequestCount()); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 7725cf9..fbc19a0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -129,7 +129,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize { return lruCache.getBlockCount() + l2Cache.getBlockCount(); } - private static class CombinedCacheStats extends CacheStats { + public static class CombinedCacheStats extends CacheStats { private final CacheStats lruCacheStats; private final CacheStats bucketCacheStats; @@ -157,6 +157,11 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize { } @Override + public long getPrimaryMissCount() { + return lruCacheStats.getPrimaryMissCount() + bucketCacheStats.getPrimaryMissCount(); + } + + @Override public long getMissCachingCount() { return lruCacheStats.getMissCachingCount() + bucketCacheStats.getMissCachingCount(); @@ -168,6 +173,10 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize { } @Override + public long getPrimaryHitCount() { + return lruCacheStats.getPrimaryHitCount() + bucketCacheStats.getPrimaryHitCount(); + } + @Override public long getHitCachingCount() { return lruCacheStats.getHitCachingCount() + bucketCacheStats.getHitCachingCount(); @@ -186,6 +195,12 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize { } @Override + public long getPrimaryEvictedCount() { + return lruCacheStats.getPrimaryEvictedCount() + + bucketCacheStats.getPrimaryEvictedCount(); + } + + @Override public double getHitRatioPastNPeriods() { double ratio = ((double) (lruCacheStats.getSumHitCountsPastNPeriods() + bucketCacheStats .getSumHitCountsPastNPeriods()) / (double) (lruCacheStats diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java index 6741957..0ccd674 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java @@ -449,6 +449,10 @@ public class HFile { * Return the file context of the HFile this reader belongs to */ HFileContext getFileContext(); + + boolean isPrimaryReplicaReader(); + + void setPrimaryReplicaReader(boolean isPrimaryReplicaReader); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java index e1a0e6d..a87f8bb 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java @@ -341,7 +341,8 @@ public class HFileReaderV2 extends AbstractHFileReader { synchronized (metaBlockIndexReader.getRootBlockKey(block)) { // Check cache for block. If found return. long metaBlockOffset = metaBlockIndexReader.getRootBlockOffset(block); - BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset); + BlockCacheKey cacheKey = new BlockCacheKey(name, metaBlockOffset, + this.isPrimaryReplicaReader()); cacheBlock &= cacheConf.shouldCacheDataOnRead(); if (cacheConf.isBlockCacheEnabled()) { @@ -387,7 +388,7 @@ public class HFileReaderV2 extends AbstractHFileReader { // Without a cache, this synchronizing is needless overhead, but really // the other choice is to duplicate work (which the cache would prevent you // from doing). - BlockCacheKey cacheKey = new BlockCacheKey(name, dataBlockOffset); + BlockCacheKey cacheKey = new BlockCacheKey(name, dataBlockOffset,this.isPrimaryReplicaReader()); boolean useLock = false; IdLock.Entry lockEntry = null; TraceScope traceScope = Trace.startSpan("HFileReaderV2.readBlock"); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java index 2bfb821..8ca579d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/LruBlockCache.java @@ -419,7 +419,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { boolean updateCacheMetrics) { LruCachedBlock cb = map.get(cacheKey); if (cb == null) { - if (!repeat && updateCacheMetrics) stats.miss(caching); + if (!repeat && updateCacheMetrics) stats.miss(caching, cacheKey.isPrimary()); // If there is another block cache then try and read there. // However if this is a retry ( second time in double checked locking ) // And it's already a miss then the l2 will also be a miss. @@ -434,7 +434,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { } return null; } - if (updateCacheMetrics) stats.hit(caching); + if (updateCacheMetrics) stats.hit(caching, cacheKey.isPrimary()); cb.access(count.incrementAndGet()); return cb.getBuffer(); } @@ -497,7 +497,7 @@ public class LruBlockCache implements ResizableBlockCache, HeapSize { long size = map.size(); assertCounterSanity(size, val); } - stats.evicted(block.getCachedTime()); + stats.evicted(block.getCachedTime(), block.getCacheKey().isPrimary()); if (evictedByEvictionProcess && victimHandler != null) { if (victimHandler instanceof BucketCache) { boolean wait = getCurrentSize() < acceptableSize(); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java index 57e7f28..ba75542 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/MemcachedBlockCache.java @@ -145,9 +145,9 @@ public class MemcachedBlockCache implements BlockCache { // Update stats if this request doesn't have it turned off 100% of the time if (updateCacheMetrics) { if (result == null) { - cacheStats.miss(caching); + cacheStats.miss(caching, cacheKey.isPrimary()); } else { - cacheStats.hit(caching); + cacheStats.hit(caching, cacheKey.isPrimary()); } } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java index 259afae..da78127 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/bucket/BucketCache.java @@ -401,7 +401,7 @@ public class BucketCache implements BlockCache, HeapSize { RAMQueueEntry re = ramCache.get(key); if (re != null) { if (updateCacheMetrics) { - cacheStats.hit(caching); + cacheStats.hit(caching, key.isPrimary()); } re.access(accessCount.incrementAndGet()); return re.getData(); @@ -427,7 +427,7 @@ public class BucketCache implements BlockCache, HeapSize { Cacheable cachedBlock = deserializer.deserialize(bb, true); long timeTaken = System.nanoTime() - start; if (updateCacheMetrics) { - cacheStats.hit(caching); + cacheStats.hit(caching, key.isPrimary()); cacheStats.ioHit(timeTaken); } bucketEntry.access(accessCount.incrementAndGet()); @@ -446,7 +446,7 @@ public class BucketCache implements BlockCache, HeapSize { } } if (!repeat && updateCacheMetrics) { - cacheStats.miss(caching); + cacheStats.miss(caching, key.isPrimary()); } return null; } @@ -474,7 +474,7 @@ public class BucketCache implements BlockCache, HeapSize { BucketEntry bucketEntry = backingMap.get(cacheKey); if (bucketEntry == null) { if (removedBlock != null) { - cacheStats.evicted(0); + cacheStats.evicted(0, cacheKey.isPrimary()); return true; } else { return false; @@ -496,7 +496,7 @@ public class BucketCache implements BlockCache, HeapSize { offsetLock.releaseLockEntry(lockEntry); } } - cacheStats.evicted(bucketEntry.getCachedTime()); + cacheStats.evicted(bucketEntry.getCachedTime(), cacheKey.isPrimary()); return true; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index e15db38..454b9cc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -599,7 +599,8 @@ public class HStore implements Store { if (newFiles == null) newFiles = new ArrayList(0); - HashMap currentFilesSet = new HashMap(currentFiles.size()); + HashMap currentFilesSet = + new HashMap(currentFiles.size()); for (StoreFile sf : currentFiles) { currentFilesSet.put(sf.getFileInfo(), sf); } @@ -647,7 +648,8 @@ public class HStore implements Store { info.setRegionCoprocessorHost(this.region.getCoprocessorHost()); StoreFile storeFile = new StoreFile(this.getFileSystem(), info, this.conf, this.cacheConf, this.family.getBloomFilterType()); - storeFile.createReader(); + StoreFile.Reader r = storeFile.createReader(); + r.setReplicaStoreFile(isPrimaryReplicaStore()); return storeFile; } @@ -1114,7 +1116,7 @@ public class HStore implements Store { // but now we get them in ascending order, which I think is // actually more correct, since memstore get put at the end. List sfScanners = StoreFileScanner.getScannersForStoreFiles(storeFilesToScan, - cacheBlocks, usePread, isCompaction, false, matcher, readPt); + cacheBlocks, usePread, isCompaction, false, matcher, readPt, isPrimaryReplicaStore()); List scanners = new ArrayList(sfScanners.size()+1); scanners.addAll(sfScanners); @@ -2405,4 +2407,9 @@ public class HStore implements Store { public double getCompactionPressure() { return storeEngine.getStoreFileManager().getCompactionPressure(); } + + @Override + public boolean isPrimaryReplicaStore() { + return getRegionInfo().getReplicaId() == HRegionInfo.DEFAULT_REPLICA_ID; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java index 1bb01da..0a18fef 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperImpl.java @@ -246,6 +246,14 @@ class MetricsRegionServerWrapperImpl } @Override + public long getBlockCachePrimaryHitCount() { + if (this.cacheStats == null) { + return 0; + } + return this.cacheStats.getPrimaryHitCount(); + } + + @Override public long getBlockCacheMissCount() { if (this.cacheStats == null) { return 0; @@ -254,6 +262,14 @@ class MetricsRegionServerWrapperImpl } @Override + public long getBlockCachePrimaryMissCount() { + if (this.cacheStats == null) { + return 0; + } + return this.cacheStats.getPrimaryMissCount(); + } + + @Override public long getBlockCacheEvictedCount() { if (this.cacheStats == null) { return 0; @@ -262,6 +278,14 @@ class MetricsRegionServerWrapperImpl } @Override + public long getBlockCachePrimaryEvictedCount() { + if (this.cacheStats == null) { + return 0; + } + return this.cacheStats.getPrimaryEvictedCount(); + } + + @Override public double getBlockCacheHitPercent() { if (this.cacheStats == null) { return 0; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java index 5a13ba8..092be46 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java @@ -105,7 +105,7 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf byte[] stopRow, long readPt ) throws IOException; - + ScanInfo getScanInfo(); /** @@ -464,4 +464,6 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf void refreshStoreFiles(Collection newFiles) throws IOException; void bulkLoadHFile(StoreFileInfo fileInfo) throws IOException; + + boolean isPrimaryReplicaStore(); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java index acd4233..36e642f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFile.java @@ -1055,6 +1055,13 @@ public class StoreFile { bloomFilterType = BloomType.NONE; } + public void setReplicaStoreFile(boolean isPrimaryReplicaStoreFile) { + reader.setPrimaryReplicaReader(isPrimaryReplicaStoreFile); + } + public boolean isPrimaryReplicaReader() { + return reader.isPrimaryReplicaReader(); + } + /** * ONLY USE DEFAULT CONSTRUCTOR FOR UNIT TESTS */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java index dc22931..fde40e9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/StoreFileScanner.java @@ -81,6 +81,10 @@ public class StoreFileScanner implements KeyValueScanner { this.hasMVCCInfo = hasMVCC; } + boolean isPrimaryReplica() { + return reader.isPrimaryReplicaReader(); + } + /** * Return an array of scanners corresponding to the given * set of store files. @@ -111,11 +115,12 @@ public class StoreFileScanner implements KeyValueScanner { public static List getScannersForStoreFiles( Collection files, boolean cacheBlocks, boolean usePread, boolean isCompaction, boolean canUseDrop, - ScanQueryMatcher matcher, long readPt) throws IOException { + ScanQueryMatcher matcher, long readPt, boolean isPrimaryReplica) throws IOException { List scanners = new ArrayList( files.size()); for (StoreFile file : files) { StoreFile.Reader r = file.createReader(canUseDrop); + r.setReplicaStoreFile(isPrimaryReplica); StoreFileScanner scanner = r.getStoreFileScanner(cacheBlocks, usePread, isCompaction, readPt); scanner.setScanQueryMatcher(matcher); @@ -124,6 +129,14 @@ public class StoreFileScanner implements KeyValueScanner { return scanners; } + public static List getScannersForStoreFiles( + Collection files, boolean cacheBlocks, boolean usePread, + boolean isCompaction, boolean canUseDrop, + ScanQueryMatcher matcher, long readPt) throws IOException { + return getScannersForStoreFiles(files, cacheBlocks, usePread, isCompaction, canUseDrop, + matcher, readPt, true); + } + public String toString() { return "StoreFileScanner[" + hfs.toString() + ", cur=" + cur + "]"; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java index ec8d31d..9f24bb7 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestLruBlockCache.java @@ -618,18 +618,18 @@ public class TestLruBlockCache { // should be (2/4)=0.5 and (1/1)=1 stats.hit(false); stats.hit(true); - stats.miss(false); - stats.miss(false); + stats.miss(false, false); + stats.miss(false, false); stats.rollMetricsPeriod(); assertEquals(0.5, stats.getHitRatioPastNPeriods(), delta); assertEquals(1.0, stats.getHitCachingRatioPastNPeriods(), delta); // period 2, 1 miss caching, 3 miss non-caching // should be (2/8)=0.25 and (1/2)=0.5 - stats.miss(true); - stats.miss(false); - stats.miss(false); - stats.miss(false); + stats.miss(true, false); + stats.miss(false, false); + stats.miss(false, false); + stats.miss(false, false); stats.rollMetricsPeriod(); assertEquals(0.25, stats.getHitRatioPastNPeriods(), delta); assertEquals(0.5, stats.getHitCachingRatioPastNPeriods(), delta); @@ -646,16 +646,16 @@ public class TestLruBlockCache { // period 4, evict period 1, two caching misses // should be (4/10)=0.4 and (2/5)=0.4 - stats.miss(true); - stats.miss(true); + stats.miss(true, false); + stats.miss(true, false); stats.rollMetricsPeriod(); assertEquals(0.4, stats.getHitRatioPastNPeriods(), delta); assertEquals(0.4, stats.getHitCachingRatioPastNPeriods(), delta); // period 5, evict period 2, 2 caching misses, 2 non-caching hit // should be (6/10)=0.6 and (2/6)=1/3 - stats.miss(true); - stats.miss(true); + stats.miss(true, false); + stats.miss(true, false); stats.hit(false); stats.hit(false); stats.rollMetricsPeriod(); @@ -682,8 +682,8 @@ public class TestLruBlockCache { // period 9, one of each // should be (2/4)=0.5 and (1/2)=0.5 - stats.miss(true); - stats.miss(false); + stats.miss(true, false); + stats.miss(false, false); stats.hit(true); stats.hit(false); stats.rollMetricsPeriod(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java index 6b458bc..33fa5c3 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/MetricsRegionServerWrapperStub.java @@ -176,16 +176,31 @@ public class MetricsRegionServerWrapperStub implements MetricsRegionServerWrappe } @Override + public long getBlockCachePrimaryHitCount() { + return 422; + } + + @Override public long getBlockCacheMissCount() { return 417; } @Override + public long getBlockCachePrimaryMissCount() { + return 421; + } + + @Override public long getBlockCacheEvictedCount() { return 418; } @Override + public long getBlockCachePrimaryEvictedCount() { + return 420; + } + + @Override public double getBlockCacheHitPercent() { return 98; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java index 84070d3..9cbde84 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java @@ -522,13 +522,13 @@ public class TestHeapMemoryManager { @Override public boolean evictBlock(BlockCacheKey cacheKey) { - stats.evicted(0); + stats.evicted(0, cacheKey != null ? cacheKey.isPrimary() : true); return false; } @Override public int evictBlocksByHfileName(String hfileName) { - stats.evicted(0); // Just assuming only one block for file here. + stats.evicted(0, true); // Just assuming only one block for file here. return 0; }