From fbb5a4f23aff1ab796e8f44448d8de15ac3e2cd7 Mon Sep 17 00:00:00 2001 From: Biju Nair Date: Fri, 2 Jun 2017 00:42:13 -0400 Subject: [PATCH 1/1] Quick change to modify current LruBlockCache to a named cache of "default" --- .../apache/hadoop/hbase/io/hfile/CacheConfig.java | 17 ++- .../hadoop/hbase/io/hfile/CombinedBlockCache.java | 10 +- .../io/hfile/InclusiveCombinedBlockCache.java | 2 +- .../hadoop/hbase/io/hfile/NamedBlockCache.java | 170 +++++++++++++++++++++ .../hbase/regionserver/HeapMemoryManager.java | 2 +- 5 files changed, 188 insertions(+), 13 deletions(-) create mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NamedBlockCache.java diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index 791445b..4da64ea 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -546,7 +546,7 @@ public class CacheConfig { // Clear this if in tests you'd make more than one block cache instance. @VisibleForTesting static BlockCache GLOBAL_BLOCK_CACHE_INSTANCE; - private static LruBlockCache GLOBAL_L1_CACHE_INSTANCE; + private static BlockCache GLOBAL_L1_CACHE_INSTANCE; /** Boolean whether we have disabled the block cache entirely. */ @VisibleForTesting @@ -556,7 +556,7 @@ public class CacheConfig { * @param c Configuration to use. * @return An L1 instance. Currently an instance of LruBlockCache. */ - public static LruBlockCache getL1(final Configuration c) { + public static BlockCache getL1(final Configuration c) { return getL1Internal(c); } @@ -564,7 +564,7 @@ public class CacheConfig { * @param c Configuration to use. * @return An L1 instance. Currently an instance of LruBlockCache. */ - private synchronized static LruBlockCache getL1Internal(final Configuration c) { + private synchronized static BlockCache getL1Internal(final Configuration c) { if (GLOBAL_L1_CACHE_INSTANCE != null) return GLOBAL_L1_CACHE_INSTANCE; final long lruCacheSize = MemorySizeUtil.getLruCacheSize(c); if (lruCacheSize < 0) { @@ -574,7 +574,10 @@ public class CacheConfig { int blockSize = c.getInt(BLOCKCACHE_BLOCKSIZE_KEY, HConstants.DEFAULT_BLOCKSIZE); LOG.info("Allocating LruBlockCache size=" + StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize)); - GLOBAL_L1_CACHE_INSTANCE = new LruBlockCache(lruCacheSize, blockSize, true, c); + if (c.getBoolean("hbase.named.cache.config",false)) + GLOBAL_L1_CACHE_INSTANCE = new NamedBlockCache(lruCacheSize, blockSize, true, c); + else + GLOBAL_L1_CACHE_INSTANCE = new LruBlockCache(lruCacheSize, blockSize, true, c); return GLOBAL_L1_CACHE_INSTANCE; } @@ -678,7 +681,8 @@ public class CacheConfig { public static synchronized BlockCache instantiateBlockCache(Configuration conf) { if (GLOBAL_BLOCK_CACHE_INSTANCE != null) return GLOBAL_BLOCK_CACHE_INSTANCE; if (blockCacheDisabled) return null; - LruBlockCache l1 = getL1Internal(conf); + BlockCache l1 = getL1Internal(conf); + LOG.info("l1 is not null"+l1.hashCode()); // blockCacheDisabled is set as a side-effect of getL1Internal(), so check it again after the call. if (blockCacheDisabled) return null; BlockCache l2 = getL2(conf); @@ -701,8 +705,9 @@ public class CacheConfig { GLOBAL_BLOCK_CACHE_INSTANCE = l1; } } - l1.setVictimCache(l2); + ((LruBlockCache)l1).setVictimCache(l2); } + LOG.info("GLOBAL_BLOCK_CACHE_INSTANCE is not null"+GLOBAL_BLOCK_CACHE_INSTANCE.hashCode()); return GLOBAL_BLOCK_CACHE_INSTANCE; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java index 4ceda39..76c9a05 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CombinedBlockCache.java @@ -40,11 +40,11 @@ import com.google.common.annotations.VisibleForTesting; */ @InterfaceAudience.Private public class CombinedBlockCache implements ResizableBlockCache, HeapSize { - protected final LruBlockCache lruCache; + protected final BlockCache lruCache; protected final BlockCache l2Cache; protected final CombinedCacheStats combinedCacheStats; - public CombinedBlockCache(LruBlockCache lruCache, BlockCache l2Cache) { + public CombinedBlockCache(BlockCache lruCache, BlockCache l2Cache) { this.lruCache = lruCache; this.l2Cache = l2Cache; this.combinedCacheStats = new CombinedCacheStats(lruCache.getStats(), @@ -57,7 +57,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize { if (l2Cache instanceof HeapSize) { l2size = ((HeapSize) l2Cache).heapSize(); } - return lruCache.heapSize() + l2size; + return ((LruBlockCache)lruCache).heapSize() + l2size; } @Override @@ -81,7 +81,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize { boolean repeat, boolean updateCacheMetrics) { // TODO: is there a hole here, or just awkwardness since in the lruCache getBlock // we end up calling l2Cache.getBlock. - return lruCache.containsBlock(cacheKey)? + return ((LruBlockCache)lruCache).containsBlock(cacheKey)? lruCache.getBlock(cacheKey, caching, repeat, updateCacheMetrics): l2Cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics); } @@ -353,7 +353,7 @@ public class CombinedBlockCache implements ResizableBlockCache, HeapSize { @Override public void setMaxSize(long size) { - this.lruCache.setMaxSize(size); + ((LruBlockCache)this.lruCache).setMaxSize(size); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java index 667e7b4..8bf7c4b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/InclusiveCombinedBlockCache.java @@ -24,7 +24,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.CONFIG) public class InclusiveCombinedBlockCache extends CombinedBlockCache implements BlockCache { - public InclusiveCombinedBlockCache(LruBlockCache l1, BlockCache l2) { + public InclusiveCombinedBlockCache(BlockCache l1, BlockCache l2) { super(l1,l2); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NamedBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NamedBlockCache.java new file mode 100644 index 0000000..59105ee --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/NamedBlockCache.java @@ -0,0 +1,170 @@ +package org.apache.hadoop.hbase.io.hfile; + +import java.util.HashMap; +import java.util.Iterator; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.io.HeapSize; + +public class NamedBlockCache implements ResizableBlockCache, HeapSize { + + private static final Log LOG = LogFactory.getLog(NamedBlockCache.class); + + private HashMap map = new HashMap(); + + public NamedBlockCache(long maxSize, long blockSize, boolean evictionThread, Configuration conf) { + String[] cacheNames = conf.getTrimmedStrings("hbase.named.cache.names"); + int cacheCount = cacheNames.length; + //map = new HashMap(cacheCount); + String[] cacheSize = conf.getTrimmedStrings("hbase.named.cache.size.percent"); + for (int i = 0; i < cacheCount; i++) + LOG.info("Cache Names "+ cacheNames[i] + " "+cacheSize[i]); + int totPercent = 0; + for (int i = 0; i < cacheCount; i++) { + LOG.info("**Cache Names "+ cacheNames[i] + " "+cacheSize[i]+ " "+(int)Math.ceil(maxSize * (Integer.parseInt(cacheSize[i])/100.0))); + map.put(cacheNames[i], new LruBlockCache((int)Math.ceil(maxSize * (Integer.parseInt(cacheSize[i])/100.0)), + blockSize, true, conf)); + totPercent = totPercent + Integer.parseInt(cacheSize[i]); + if (totPercent > 95) + System.out.println("Need to fix the config"); + } + map.put("default", new LruBlockCache(maxSize * (100 - totPercent)/100, + blockSize, true, conf)); + } + + @Override + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory, boolean cacheDataInL1) { + cacheBlock(cacheKey, buf, inMemory, cacheDataInL1, "default"); + } + + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory, boolean cacheDataInL1, String name) { + LruBlockCache cache = map.get(name); + cache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1); + } + + @Override + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { + cacheBlock(cacheKey, buf, "default"); + + } + + public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, String name) { + LruBlockCache cache = map.get(name); + cache.cacheBlock(cacheKey, buf); + } + + @Override + public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, boolean updateCacheMetrics) { + return getBlock(cacheKey, caching, repeat, updateCacheMetrics,"default"); + } + + public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, boolean updateCacheMetrics, String name) { + LruBlockCache cache = map.get(name); + return cache.getBlock(cacheKey, caching, repeat, updateCacheMetrics); + } + + @Override + public boolean evictBlock(BlockCacheKey cacheKey) { + return evictBlock(cacheKey, "default"); + } + + public boolean evictBlock(BlockCacheKey cacheKey, String name) { + LruBlockCache cache = map.get(name); + return cache.evictBlock(cacheKey); + } + + @Override + public int evictBlocksByHfileName(String hfileName) { + return evictBlocksByHfileName(hfileName, "default"); + } + + public int evictBlocksByHfileName(String hfileName, String name) { + LruBlockCache cache = map.get(name); + return cache.evictBlocksByHfileName(hfileName); + } + + @Override + public CacheStats getStats() { + return getStats("default"); + } + + public CacheStats getStats(String name) { + LruBlockCache cache = map.get(name); + return cache.getStats(); + } + + @Override + public void shutdown() { + for (LruBlockCache cache : map.values()) + cache.shutdown(); + } + + @Override + public long size() { + int size = 0; + for (LruBlockCache cache : map.values()) + size += cache.size(); + return size; + } + + @Override + public long getFreeSize() { + int freeSize = 0; + for (LruBlockCache cache : map.values()) + freeSize += cache.getFreeSize(); + return freeSize; + } + + @Override + public long getCurrentSize() { + int currentSize = 0; + for (LruBlockCache cache : map.values()) + currentSize += cache.getCurrentSize(); + return currentSize; + } + + @Override + public long getBlockCount() { + int blockCount = 0; + for (LruBlockCache cache : map.values()) + blockCount += cache.getBlockCount(); + return blockCount; + } + + @Override + public Iterator iterator() { + return map.get("default").iterator(); + // Need more changes to take into account all named caches + } + + @Override + public BlockCache[] getBlockCaches() { + // TODO Auto-generated method stub + return null; + } + + @Override + public void returnBlock(BlockCacheKey cacheKey, Cacheable block) { + returnBlock(cacheKey, block, "default"); + } + + public void returnBlock(BlockCacheKey cacheKey, Cacheable block, String name) { + LruBlockCache cache = map.get(name); + cache.returnBlock(cacheKey, block); + } + + @Override + public long heapSize() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public void setMaxSize(long size) { + // TODO Auto-generated method stub + + } + +} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java index 93e502a..0a0598a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java @@ -108,7 +108,7 @@ public class HeapMemoryManager { public static HeapMemoryManager create(Configuration conf, FlushRequester memStoreFlusher, Server server, RegionServerAccounting regionServerAccounting) { - ResizableBlockCache l1Cache = CacheConfig.getL1(conf); + ResizableBlockCache l1Cache = (ResizableBlockCache)CacheConfig.getL1(conf); if (l1Cache != null) { return new HeapMemoryManager(l1Cache, memStoreFlusher, server, regionServerAccounting); } -- 1.8.5.2 (Apple Git-48)