From 0a687f58c8aa82f4b587479b2abda42b8c4f1362 Mon Sep 17 00:00:00 2001 From: Nick Dimiduk Date: Mon, 30 Jun 2014 19:18:46 -0700 Subject: [PATCH] HBASE-11372 Remove SlabCache --- .../hbase/tmpl/regionserver/BlockCacheTmpl.jamon | 47 +- .../tmpl/regionserver/BlockCacheViewTmpl.jamon | 2 - .../apache/hadoop/hbase/io/hfile/CacheConfig.java | 108 ++--- .../hadoop/hbase/io/hfile/DoubleBlockCache.java | 184 -------- .../hbase/io/hfile/slab/SingleSizeCache.java | 356 -------------- .../apache/hadoop/hbase/io/hfile/slab/Slab.java | 137 ------ .../hadoop/hbase/io/hfile/slab/SlabCache.java | 514 --------------------- .../hbase/io/hfile/slab/SlabItemActionWatcher.java | 49 -- .../hadoop/hbase/mapreduce/TableMapReduceUtil.java | 2 - .../hbase/io/hfile/TestBlockCacheReporting.java | 17 - .../hadoop/hbase/io/hfile/TestCacheConfig.java | 9 - .../hbase/io/hfile/slab/TestSingleSizeCache.java | 83 ---- .../hadoop/hbase/io/hfile/slab/TestSlab.java | 77 --- .../hadoop/hbase/io/hfile/slab/TestSlabCache.java | 111 ----- 14 files changed, 47 insertions(+), 1649 deletions(-) delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/Slab.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java delete mode 100644 hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlab.java delete mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon index 1aed29a..80391b2 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheTmpl.jamon @@ -43,8 +43,6 @@ org.apache.hadoop.hbase.io.hfile.bucket.BucketCacheStats; org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator; org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.Bucket; -org.apache.hadoop.hbase.io.hfile.slab.SlabCache; -org.apache.hadoop.hbase.io.hfile.slab.SingleSizeCache; org.apache.hadoop.util.StringUtils;
@@ -264,17 +262,12 @@ are combined counts. Request count is sum of hits and misses.

org.apache.hadoop.hbase.io.hfile.BlockCacheUtil.getLoadedCachedBlocksByFile(config, bc); AgeSnapshot snapshot = cbsbf.getAgeSnapshot(); - boolean slabCache = bc.getClass().getSimpleName().equals("SlabCache"); - Map sizer = null; - boolean bucketCache = bc.getClass().getSimpleName().equals("BucketCache"); BucketCacheStats bucketCacheStats = null; BucketAllocator bucketAllocator = null; Bucket [] buckets = null; - if (slabCache) { - sizer = ((SlabCache)bc).getSizer(); - } else if (bucketCache) { + if (bucketCache) { bucketCacheStats = (BucketCacheStats)bc.getStats(); bucketAllocator = ((BucketCache)bc).getAllocator(); buckets = bucketAllocator.getBuckets(); @@ -325,8 +318,7 @@ are combined counts. Request count is sum of hits and misses.

Size of DATA Blocks -<%doc>Can't do age of block in cache when slab cache -<%if !slabCache %> + Mean <% String.format("%,d", (long)(snapshot.getMean()/nanosPerSecond)) %> @@ -357,7 +349,7 @@ are combined counts. Request count is sum of hits and misses.

<% String.format("%,d", (long)(snapshot.get99thPercentile()/nanosPerSecond)) %> 99th percentile of age of Blocks in cache (seconds) - + <%if bucketCache %> Hits per Second @@ -392,39 +384,6 @@ are combined counts. Request count is sum of hits and misses.

-<%elseif slabCache %> -

SlabCache does not keep account of block ages so can not show stats on how long blocks have been cached.

-

SlabCache Slabs

- - - - - - - - - - - - - - -<%for Map.Entry e: sizer.entrySet() %> - - - - - - - - - - - - - - -
Block SizeSizeFree SizeCountEvictedEvictionsHitsCachingMissesCachingHit Ratio
<% StringUtils.humanReadableInt(e.getKey()) %><% StringUtils.humanReadableInt(e.getValue().size()) %><% StringUtils.humanReadableInt(e.getValue().getFreeSize()) %><% StringUtils.humanReadableInt(e.getValue().getBlockCount()) %><% StringUtils.humanReadableInt(e.getValue().getStats().getEvictedCount()) %><% StringUtils.humanReadableInt(e.getValue().getStats().getEvictionCount()) %><% StringUtils.humanReadableInt(e.getValue().getStats().getHitCount()) %><% StringUtils.humanReadableInt(e.getValue().getStats().getHitCachingCount()) %><% StringUtils.humanReadableInt(e.getValue().getStats().getMissCount()) %><% StringUtils.humanReadableInt(e.getValue().getStats().getMissCachingCount()) %><% String.format("%,.2f", e.getValue().getStats().getHitRatio() * 100) %><% "%" %>
<%java> cbsbf = null; diff --git a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon index 50da12a..d6a25ad 100644 --- a/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon +++ b/hbase-server/src/main/jamon/org/apache/hadoop/hbase/tmpl/regionserver/BlockCacheViewTmpl.jamon @@ -36,8 +36,6 @@ org.apache.hadoop.hbase.io.hfile.bucket.BucketCacheStats; org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator; org.apache.hadoop.hbase.io.hfile.bucket.BucketAllocator.Bucket; -org.apache.hadoop.hbase.io.hfile.slab.SlabCache; -org.apache.hadoop.hbase.io.hfile.slab.SingleSizeCache; org.apache.hadoop.util.StringUtils; com.yammer.metrics.stats.Snapshot; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java index cf98102..2e9a41d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheConfig.java @@ -29,8 +29,6 @@ import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.io.hfile.BlockType.BlockCategory; import org.apache.hadoop.hbase.io.hfile.bucket.BucketCache; -import org.apache.hadoop.hbase.io.hfile.slab.SlabCache; -import org.apache.hadoop.hbase.util.DirectMemoryUtils; import org.apache.hadoop.util.StringUtils; import com.google.common.annotations.VisibleForTesting; @@ -139,19 +137,6 @@ public class CacheConfig { public static final int DEFAULT_BUCKET_CACHE_WRITER_QUEUE = 64; public static final float DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE = 0.9f; - /** - * Setting this float to a non-null value turns on {@link DoubleBlockCache} - * which makes use of the {@link LruBlockCache} and {@link SlabCache}. - * - * The float value of between 0 and 1 will be multiplied against the setting for - * -XX:MaxDirectMemorySize to figure what size of the offheap allocation to give - * over to slab cache. - * - * Slab cache has been little used and is likely to be deprecated in the near future. - */ - public static final String SLAB_CACHE_OFFHEAP_PERCENTAGE_KEY = - "hbase.offheapcache.percentage"; - /** * Configuration key to prefetch all blocks of a given file into the block cache * when the file is opened. @@ -491,52 +476,31 @@ public class CacheConfig { MemoryUsage mu = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage(); long lruCacheSize = (long) (mu.getMax() * cachePercentage); int blockSize = conf.getInt("hbase.offheapcache.minblocksize", HConstants.DEFAULT_BLOCKSIZE); - long slabCacheOffHeapCacheSize = - conf.getFloat(SLAB_CACHE_OFFHEAP_PERCENTAGE_KEY, 0) == 0? - 0: - (long) (conf.getFloat(SLAB_CACHE_OFFHEAP_PERCENTAGE_KEY, (float) 0) * - DirectMemoryUtils.getDirectMemorySize()); - if (slabCacheOffHeapCacheSize <= 0) { - String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null); - float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F); - // A percentage of max heap size or a absolute value with unit megabytes - long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax() - * bucketCachePercentage : bucketCachePercentage * 1024 * 1024); - - boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY, - DEFAULT_BUCKET_CACHE_COMBINED); - BucketCache bucketCache = null; - if (bucketCacheIOEngineName != null && bucketCacheSize > 0) { - int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY, - DEFAULT_BUCKET_CACHE_WRITER_THREADS); - int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY, - DEFAULT_BUCKET_CACHE_WRITER_QUEUE); - String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY); - float combinedPercentage = conf.getFloat( - BUCKET_CACHE_COMBINED_PERCENTAGE_KEY, - DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE); - String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY); - int[] bucketSizes = null; - if (configuredBucketSizes != null) { - bucketSizes = new int[configuredBucketSizes.length]; - for (int i = 0; i < configuredBucketSizes.length; i++) { - bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i]); - } - } - if (combinedWithLru) { - lruCacheSize = (long) ((1 - combinedPercentage) * bucketCacheSize); - bucketCacheSize = (long) (combinedPercentage * bucketCacheSize); - } - try { - int ioErrorsTolerationDuration = conf.getInt( - "hbase.bucketcache.ioengine.errors.tolerated.duration", - BucketCache.DEFAULT_ERROR_TOLERATION_DURATION); - bucketCache = new BucketCache(bucketCacheIOEngineName, - bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath, - ioErrorsTolerationDuration); - } catch (IOException ioex) { - LOG.error("Can't instantiate bucket cache", ioex); - throw new RuntimeException(ioex); + + String bucketCacheIOEngineName = conf.get(BUCKET_CACHE_IOENGINE_KEY, null); + float bucketCachePercentage = conf.getFloat(BUCKET_CACHE_SIZE_KEY, 0F); + // A percentage of max heap size or a absolute value with unit megabytes + long bucketCacheSize = (long) (bucketCachePercentage < 1 ? mu.getMax() + * bucketCachePercentage : bucketCachePercentage * 1024 * 1024); + + boolean combinedWithLru = conf.getBoolean(BUCKET_CACHE_COMBINED_KEY, + DEFAULT_BUCKET_CACHE_COMBINED); + BucketCache bucketCache = null; + if (bucketCacheIOEngineName != null && bucketCacheSize > 0) { + int writerThreads = conf.getInt(BUCKET_CACHE_WRITER_THREADS_KEY, + DEFAULT_BUCKET_CACHE_WRITER_THREADS); + int writerQueueLen = conf.getInt(BUCKET_CACHE_WRITER_QUEUE_KEY, + DEFAULT_BUCKET_CACHE_WRITER_QUEUE); + String persistentPath = conf.get(BUCKET_CACHE_PERSISTENT_PATH_KEY); + float combinedPercentage = conf.getFloat( + BUCKET_CACHE_COMBINED_PERCENTAGE_KEY, + DEFAULT_BUCKET_CACHE_COMBINED_PERCENTAGE); + String[] configuredBucketSizes = conf.getStrings(BUCKET_CACHE_BUCKETS_KEY); + int[] bucketSizes = null; + if (configuredBucketSizes != null) { + bucketSizes = new int[configuredBucketSizes.length]; + for (int i = 0; i < configuredBucketSizes.length; i++) { + bucketSizes[i] = Integer.parseInt(configuredBucketSizes[i]); } } LOG.info("Allocating LruBlockCache size=" + @@ -548,10 +512,26 @@ public class CacheConfig { } else { GLOBAL_BLOCK_CACHE_INSTANCE = lruCache; } + try { + int ioErrorsTolerationDuration = conf.getInt( + "hbase.bucketcache.ioengine.errors.tolerated.duration", + BucketCache.DEFAULT_ERROR_TOLERATION_DURATION); + bucketCache = new BucketCache(bucketCacheIOEngineName, + bucketCacheSize, blockSize, bucketSizes, writerThreads, writerQueueLen, persistentPath, + ioErrorsTolerationDuration); + } catch (IOException ioex) { + LOG.error("Can't instantiate bucket cache", ioex); + throw new RuntimeException(ioex); + } + } + LOG.info("Allocating LruBlockCache size=" + + StringUtils.byteDesc(lruCacheSize) + ", blockSize=" + StringUtils.byteDesc(blockSize)); + LruBlockCache lruCache = new LruBlockCache(lruCacheSize, blockSize); + lruCache.setVictimCache(bucketCache); + if (bucketCache != null && combinedWithLru) { + GLOBAL_BLOCK_CACHE_INSTANCE = new CombinedBlockCache(lruCache, bucketCache); } else { - LOG.warn("SlabCache is deprecated. Consider BucketCache as a replacement."); - GLOBAL_BLOCK_CACHE_INSTANCE = new DoubleBlockCache( - lruCacheSize, slabCacheOffHeapCacheSize, blockSize, blockSize, conf); + GLOBAL_BLOCK_CACHE_INSTANCE = lruCache; } return GLOBAL_BLOCK_CACHE_INSTANCE; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java deleted file mode 100644 index bf3136b..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/DoubleBlockCache.java +++ /dev/null @@ -1,184 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.io.hfile; - -import java.util.Iterator; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.io.hfile.slab.SlabCache; -import org.apache.hadoop.util.StringUtils; - -/** - * DoubleBlockCache is an abstraction layer that combines two caches, the - * smaller onHeapCache and the larger offHeapCache. CacheBlock attempts to cache - * the block in both caches, while readblock reads first from the faster on heap - * cache before looking for the block in the off heap cache. Metrics are the - * combined size and hits and misses of both caches. - * - * @deprecated As of 1.0, replaced by {@link org.apache.hadoop.hbase.io.hfile.bucket.BucketCache}. - */ -@InterfaceAudience.Private -@Deprecated -public class DoubleBlockCache implements ResizableBlockCache, HeapSize { - - static final Log LOG = LogFactory.getLog(DoubleBlockCache.class.getName()); - - private final LruBlockCache onHeapCache; - private final SlabCache offHeapCache; - private final CacheStats stats; - - /** - * Default constructor. Specify maximum size and expected average block size - * (approximation is fine). - *

- * All other factors will be calculated based on defaults specified in this - * class. - * - * @param onHeapSize maximum size of the onHeapCache, in bytes. - * @param offHeapSize maximum size of the offHeapCache, in bytes. - * @param onHeapBlockSize average block size of the on heap cache. - * @param offHeapBlockSize average block size for the off heap cache - * @param conf configuration file. currently used only by the off heap cache. - */ - public DoubleBlockCache(long onHeapSize, long offHeapSize, - long onHeapBlockSize, long offHeapBlockSize, Configuration conf) { - - LOG.info("Creating on-heap cache of size " - + StringUtils.byteDesc(onHeapSize) - + " with an average block size of " - + StringUtils.byteDesc(onHeapBlockSize)); - onHeapCache = new LruBlockCache(onHeapSize, onHeapBlockSize, conf); - - LOG.info("Creating off-heap cache of size " - + StringUtils.byteDesc(offHeapSize) - + "with an average block size of " - + StringUtils.byteDesc(offHeapBlockSize)); - offHeapCache = new SlabCache(offHeapSize, offHeapBlockSize); - - offHeapCache.addSlabByConf(conf); - this.stats = new CacheStats(); - } - - @Override - public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory, - final boolean cacheDataInL1) { - onHeapCache.cacheBlock(cacheKey, buf, inMemory, cacheDataInL1); - offHeapCache.cacheBlock(cacheKey, buf); - } - - @Override - public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf) { - onHeapCache.cacheBlock(cacheKey, buf); - offHeapCache.cacheBlock(cacheKey, buf); - } - - @Override - public Cacheable getBlock(BlockCacheKey cacheKey, boolean caching, boolean repeat, - boolean updateCacheMetrics) { - Cacheable cachedBlock; - - if ((cachedBlock = onHeapCache.getBlock(cacheKey, caching, repeat, - updateCacheMetrics)) != null) { - if (updateCacheMetrics) stats.hit(caching); - return cachedBlock; - - } else if ((cachedBlock = offHeapCache.getBlock(cacheKey, caching, repeat, - updateCacheMetrics)) != null) { - if (caching) { - onHeapCache.cacheBlock(cacheKey, cachedBlock); - } - if (updateCacheMetrics) stats.hit(caching); - return cachedBlock; - } - - if (!repeat && updateCacheMetrics) stats.miss(caching); - return null; - } - - @Override - public boolean evictBlock(BlockCacheKey cacheKey) { - stats.evict(); - boolean cacheA = onHeapCache.evictBlock(cacheKey); - boolean cacheB = offHeapCache.evictBlock(cacheKey); - boolean evicted = cacheA || cacheB; - if (evicted) { - stats.evicted(); - } - return evicted; - } - - @Override - public CacheStats getStats() { - return this.stats; - } - - @Override - public void shutdown() { - onHeapCache.shutdown(); - offHeapCache.shutdown(); - } - - @Override - public long heapSize() { - return onHeapCache.heapSize() + offHeapCache.heapSize(); - } - - public long size() { - return onHeapCache.size() + offHeapCache.size(); - } - - public long getFreeSize() { - return onHeapCache.getFreeSize() + offHeapCache.getFreeSize(); - } - - public long getCurrentSize() { - return onHeapCache.getCurrentSize() + offHeapCache.getCurrentSize(); - } - - @Override - public int evictBlocksByHfileName(String hfileName) { - onHeapCache.evictBlocksByHfileName(hfileName); - offHeapCache.evictBlocksByHfileName(hfileName); - return 0; - } - - @Override - public long getBlockCount() { - return onHeapCache.getBlockCount() + offHeapCache.getBlockCount(); - } - - @Override - public void setMaxSize(long size) { - this.onHeapCache.setMaxSize(size); - } - - @Override - public Iterator iterator() { - return new BlockCachesIterator(getBlockCaches()); - } - - @Override - public BlockCache[] getBlockCaches() { - return new BlockCache [] {this.onHeapCache, this.offHeapCache}; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java deleted file mode 100644 index 09b82bf..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SingleSizeCache.java +++ /dev/null @@ -1,356 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.io.hfile.slab; - -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.io.hfile.BlockCache; -import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; -import org.apache.hadoop.hbase.io.hfile.CacheStats; -import org.apache.hadoop.hbase.io.hfile.Cacheable; -import org.apache.hadoop.hbase.io.hfile.CacheableDeserializer; -import org.apache.hadoop.hbase.io.hfile.CachedBlock; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.ClassSize; -import org.apache.hadoop.util.StringUtils; - -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.RemovalListener; -import com.google.common.cache.RemovalNotification; - -/** - * SingleSizeCache is a slab allocated cache that caches elements up to a single - * size. It uses a slab allocator (Slab.java) to divide a direct bytebuffer, - * into evenly sized blocks. Any cached data will take up exactly 1 block. An - * exception will be thrown if the cached data cannot fit into the blockSize of - * this SingleSizeCache. - * - * Eviction and LRUness is taken care of by Guava's MapMaker, which creates a - * ConcurrentLinkedHashMap. - * - * @deprecated As of 1.0, replaced by {@link org.apache.hadoop.hbase.io.hfile.bucket.BucketCache}. - **/ -@InterfaceAudience.Private -@Deprecated -public class SingleSizeCache implements BlockCache, HeapSize { - private final Slab backingStore; - private final ConcurrentMap backingMap; - private final int numBlocks; - private final int blockSize; - private final CacheStats stats; - private final SlabItemActionWatcher actionWatcher; - private final AtomicLong size; - private final AtomicLong timeSinceLastAccess; - public final static long CACHE_FIXED_OVERHEAD = ClassSize - .align((2 * Bytes.SIZEOF_INT) + (5 * ClassSize.REFERENCE) - + +ClassSize.OBJECT); - - static final Log LOG = LogFactory.getLog(SingleSizeCache.class); - - /** - * Default constructor. Specify the size of the blocks, number of blocks, and - * the SlabCache this cache will be assigned to. - * - * - * @param blockSize the size of each block, in bytes - * - * @param numBlocks the number of blocks of blockSize this cache will hold. - * - * @param master the SlabCache this SingleSlabCache is assigned to. - */ - public SingleSizeCache(int blockSize, int numBlocks, - SlabItemActionWatcher master) { - this.blockSize = blockSize; - this.numBlocks = numBlocks; - backingStore = new Slab(blockSize, numBlocks); - this.stats = new CacheStats(); - this.actionWatcher = master; - this.size = new AtomicLong(CACHE_FIXED_OVERHEAD + backingStore.heapSize()); - this.timeSinceLastAccess = new AtomicLong(); - - // This evictionListener is called whenever the cache automatically - // evicts something. - RemovalListener listener = - new RemovalListener() { - @Override - public void onRemoval( - RemovalNotification notification) { - if (!notification.wasEvicted()) { - // Only process removals by eviction, not by replacement or - // explicit removal - return; - } - CacheablePair value = notification.getValue(); - timeSinceLastAccess.set(System.nanoTime() - - value.recentlyAccessed.get()); - stats.evict(); - doEviction(notification.getKey(), value); - } - }; - - backingMap = CacheBuilder.newBuilder() - .maximumSize(numBlocks - 1) - .removalListener(listener) - .build() - .asMap(); - } - - @Override - public void cacheBlock(BlockCacheKey blockName, Cacheable toBeCached) { - ByteBuffer storedBlock; - - try { - storedBlock = backingStore.alloc(toBeCached.getSerializedLength()); - } catch (InterruptedException e) { - LOG.warn("SlabAllocator was interrupted while waiting for block to become available"); - LOG.warn(e); - return; - } - - CacheablePair newEntry = new CacheablePair(toBeCached.getDeserializer(), - storedBlock); - toBeCached.serialize(storedBlock); - - synchronized (this) { - CacheablePair alreadyCached = backingMap.putIfAbsent(blockName, newEntry); - - if (alreadyCached != null) { - backingStore.free(storedBlock); - throw new RuntimeException("already cached " + blockName); - } - if (actionWatcher != null) { - actionWatcher.onInsertion(blockName, this); - } - } - newEntry.recentlyAccessed.set(System.nanoTime()); - this.size.addAndGet(newEntry.heapSize()); - } - - @Override - public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat, - boolean updateCacheMetrics) { - CacheablePair contentBlock = backingMap.get(key); - if (contentBlock == null) { - if (!repeat && updateCacheMetrics) stats.miss(caching); - return null; - } - - if (updateCacheMetrics) stats.hit(caching); - // If lock cannot be obtained, that means we're undergoing eviction. - try { - contentBlock.recentlyAccessed.set(System.nanoTime()); - synchronized (contentBlock) { - if (contentBlock.serializedData == null) { - // concurrently evicted - LOG.warn("Concurrent eviction of " + key); - return null; - } - return contentBlock.deserializer - .deserialize(contentBlock.serializedData.asReadOnlyBuffer()); - } - } catch (Throwable t) { - LOG.error("Deserializer threw an exception. This may indicate a bug.", t); - return null; - } - } - - /** - * Evicts the block - * - * @param key the key of the entry we are going to evict - * @return the evicted ByteBuffer - */ - public boolean evictBlock(BlockCacheKey key) { - stats.evict(); - CacheablePair evictedBlock = backingMap.remove(key); - - if (evictedBlock != null) { - doEviction(key, evictedBlock); - } - return evictedBlock != null; - } - - private void doEviction(BlockCacheKey key, CacheablePair evictedBlock) { - long evictedHeap = 0; - synchronized (evictedBlock) { - if (evictedBlock.serializedData == null) { - // someone else already freed - return; - } - evictedHeap = evictedBlock.heapSize(); - ByteBuffer bb = evictedBlock.serializedData; - evictedBlock.serializedData = null; - backingStore.free(bb); - - // We have to do this callback inside the synchronization here. - // Otherwise we can have the following interleaving: - // Thread A calls getBlock(): - // SlabCache directs call to this SingleSizeCache - // It gets the CacheablePair object - // Thread B runs eviction - // doEviction() is called and sets serializedData = null, here. - // Thread A sees the null serializedData, and returns null - // Thread A calls cacheBlock on the same block, and gets - // "already cached" since the block is still in backingStore - - if (actionWatcher != null) { - actionWatcher.onEviction(key, this); - } - } - stats.evicted(); - size.addAndGet(-1 * evictedHeap); - } - - public void logStats() { - - long milliseconds = this.timeSinceLastAccess.get() / 1000000; - - LOG.info("For Slab of size " + this.blockSize + ": " - + this.getOccupiedSize() / this.blockSize - + " occupied, out of a capacity of " + this.numBlocks - + " blocks. HeapSize is " - + StringUtils.humanReadableInt(this.heapSize()) + " bytes." + ", " - + "churnTime=" + StringUtils.formatTime(milliseconds)); - - LOG.info("Slab Stats: " + "accesses=" - + stats.getRequestCount() - + ", " - + "hits=" - + stats.getHitCount() - + ", " - + "hitRatio=" - + (stats.getHitCount() == 0 ? "0" : (StringUtils.formatPercent( - stats.getHitRatio(), 2) + "%, ")) - + "cachingAccesses=" - + stats.getRequestCachingCount() - + ", " - + "cachingHits=" - + stats.getHitCachingCount() - + ", " - + "cachingHitsRatio=" - + (stats.getHitCachingCount() == 0 ? "0" : (StringUtils.formatPercent( - stats.getHitCachingRatio(), 2) + "%, ")) + "evictions=" - + stats.getEvictionCount() + ", " + "evicted=" - + stats.getEvictedCount() + ", " + "evictedPerRun=" - + stats.evictedPerEviction()); - - } - - public void shutdown() { - backingStore.shutdown(); - } - - public long heapSize() { - return this.size.get() + backingStore.heapSize(); - } - - public long size() { - return (long) this.blockSize * (long) this.numBlocks; - } - - public long getFreeSize() { - return (long) backingStore.getBlocksRemaining() * (long) blockSize; - } - - public long getOccupiedSize() { - return (long) (numBlocks - backingStore.getBlocksRemaining()) * (long) blockSize; - } - - public long getEvictedCount() { - return stats.getEvictedCount(); - } - - public CacheStats getStats() { - return this.stats; - } - - @Override - public long getBlockCount() { - return numBlocks - backingStore.getBlocksRemaining(); - } - - /* Since its offheap, it doesn't matter if its in memory or not */ - @Override - public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory, - final boolean cacheDataInL1) { - this.cacheBlock(cacheKey, buf); - } - - /* - * This is never called, as evictions are handled in the SlabCache layer, - * implemented in the event we want to use this as a standalone cache. - */ - @Override - public int evictBlocksByHfileName(String hfileName) { - int evictedCount = 0; - for (BlockCacheKey e : backingMap.keySet()) { - if (e.getHfileName().equals(hfileName)) { - this.evictBlock(e); - } - } - return evictedCount; - } - - @Override - public long getCurrentSize() { - return 0; - } - - /* Just a pair class, holds a reference to the parent cacheable */ - private static class CacheablePair implements HeapSize { - final CacheableDeserializer deserializer; - ByteBuffer serializedData; - AtomicLong recentlyAccessed; - - private CacheablePair(CacheableDeserializer deserializer, - ByteBuffer serializedData) { - this.recentlyAccessed = new AtomicLong(); - this.deserializer = deserializer; - this.serializedData = serializedData; - } - - /* - * Heapsize overhead of this is the default object overhead, the heapsize of - * the serialized object, and the cost of a reference to the bytebuffer, - * which is already accounted for in SingleSizeCache - */ - @Override - public long heapSize() { - return ClassSize.align(ClassSize.OBJECT + ClassSize.REFERENCE * 3 - + ClassSize.ATOMIC_LONG); - } - } - - @Override - public Iterator iterator() { - return null; - } - - @Override - public BlockCache[] getBlockCaches() { - return null; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/Slab.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/Slab.java deleted file mode 100644 index 637890f..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/Slab.java +++ /dev/null @@ -1,137 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.io.hfile.slab; - -import java.nio.ByteBuffer; -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.LinkedBlockingQueue; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.util.ClassSize; -import org.apache.hadoop.hbase.util.DirectMemoryUtils; -import com.google.common.base.Preconditions; - -/** - * Slab is a class which is designed to allocate blocks of a certain size. - * Constructor creates a number of DirectByteBuffers and slices them into the - * requisite size, then puts them all in a buffer. - * - * @deprecated As of 1.0, replaced by {@link org.apache.hadoop.hbase.io.hfile.bucket.BucketCache}. - */ -@InterfaceAudience.Private -@Deprecated -class Slab implements org.apache.hadoop.hbase.io.HeapSize { - static final Log LOG = LogFactory.getLog(Slab.class); - - /** This is where our items, or blocks of the slab, are stored. */ - private LinkedBlockingQueue buffers; - - /** This is where our Slabs are stored */ - private ConcurrentLinkedQueue slabs; - - private final int blockSize; - private final int numBlocks; - private long heapSize; - - Slab(int blockSize, int numBlocks) { - buffers = new LinkedBlockingQueue(); - slabs = new ConcurrentLinkedQueue(); - - this.blockSize = blockSize; - this.numBlocks = numBlocks; - - this.heapSize = ClassSize.estimateBase(this.getClass(), false); - - int maxBlocksPerSlab = Integer.MAX_VALUE / blockSize; - int maxSlabSize = maxBlocksPerSlab * blockSize; - - int numFullSlabs = numBlocks / maxBlocksPerSlab; - int partialSlabSize = (numBlocks % maxBlocksPerSlab) * blockSize; - for (int i = 0; i < numFullSlabs; i++) { - allocateAndSlice(maxSlabSize, blockSize); - } - - if (partialSlabSize > 0) { - allocateAndSlice(partialSlabSize, blockSize); - } - } - - private void allocateAndSlice(int size, int sliceSize) { - ByteBuffer newSlab = ByteBuffer.allocateDirect(size); - slabs.add(newSlab); - for (int j = 0; j < newSlab.capacity(); j += sliceSize) { - newSlab.limit(j + sliceSize).position(j); - ByteBuffer aSlice = newSlab.slice(); - buffers.add(aSlice); - heapSize += ClassSize.estimateBase(aSlice.getClass(), false); - } - } - - /* - * Shutdown deallocates the memory for all the DirectByteBuffers. Each - * DirectByteBuffer has a "cleaner" method, which is similar to a - * deconstructor in C++. - */ - void shutdown() { - for (ByteBuffer aSlab : slabs) { - try { - DirectMemoryUtils.destroyDirectByteBuffer(aSlab); - } catch (Exception e) { - LOG.warn("Unable to deallocate direct memory during shutdown", e); - } - } - } - - int getBlockSize() { - return this.blockSize; - } - - int getBlockCapacity() { - return this.numBlocks; - } - - int getBlocksRemaining() { - return this.buffers.size(); - } - - /* - * Throws an exception if you try to allocate a - * bigger size than the allocator can handle. Alloc will block until a buffer is available. - */ - ByteBuffer alloc(int bufferSize) throws InterruptedException { - int newCapacity = Preconditions.checkPositionIndex(bufferSize, blockSize); - - ByteBuffer returnedBuffer = buffers.take(); - - returnedBuffer.clear().limit(newCapacity); - return returnedBuffer; - } - - void free(ByteBuffer toBeFreed) { - Preconditions.checkArgument(toBeFreed.capacity() == blockSize); - buffers.add(toBeFreed); - } - - @Override - public long heapSize() { - return heapSize; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java deleted file mode 100644 index 88c13fc..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabCache.java +++ /dev/null @@ -1,514 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.io.hfile.slab; - -import java.math.BigDecimal; -import java.util.Iterator; -import java.util.Map; -import java.util.Map.Entry; -import java.util.TreeMap; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.io.HeapSize; -import org.apache.hadoop.hbase.io.hfile.BlockCache; -import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; -import org.apache.hadoop.hbase.io.hfile.BlockPriority; -import org.apache.hadoop.hbase.io.hfile.BlockType; -import org.apache.hadoop.hbase.io.hfile.CacheStats; -import org.apache.hadoop.hbase.io.hfile.Cacheable; -import org.apache.hadoop.hbase.io.hfile.CachedBlock; -import org.apache.hadoop.hbase.io.hfile.BlockCacheUtil; -import org.apache.hadoop.hbase.util.ClassSize; -import org.apache.hadoop.hbase.util.HasThread; -import org.apache.hadoop.util.StringUtils; - -import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.ThreadFactoryBuilder; - -/** - * SlabCache is composed of multiple SingleSizeCaches. It uses a TreeMap in - * order to determine where a given element fits. Redirects gets and puts to the - * correct SingleSizeCache. - * - *

It is configured with a call to {@link #addSlab(int, int)} - * - * @deprecated As of 1.0, replaced by {@link org.apache.hadoop.hbase.io.hfile.bucket.BucketCache}. - */ -@InterfaceAudience.Private -@Deprecated -public class SlabCache implements SlabItemActionWatcher, BlockCache, HeapSize { - private final ConcurrentHashMap backingStore; - private final TreeMap slabs; - static final Log LOG = LogFactory.getLog(SlabCache.class); - static final int STAT_THREAD_PERIOD_SECS = 60 * 5; - - private final ScheduledExecutorService scheduleThreadPool = Executors.newScheduledThreadPool(1, - new ThreadFactoryBuilder().setDaemon(true).setNameFormat("Slab Statistics #%d").build()); - - long size; - private final CacheStats stats; - final SlabStats requestStats; - final SlabStats successfullyCachedStats; - private final long avgBlockSize; - private static final long CACHE_FIXED_OVERHEAD = ClassSize.estimateBase( - SlabCache.class, false); - - /** - * Key used reading from configuration list of the percentage of our total space we allocate to - * the slabs. Defaults: "0.80", "0.20". - * @see #SLAB_CACHE_SIZES_KEY Must have corresponding number of elements. - */ - static final String SLAB_CACHE_PROPORTIONS_KEY = "hbase.offheapcache.slab.proportions"; - - /** - * Configuration key for list of the blocksize of the slabs in bytes. (E.g. the slab holds - * blocks of this size). Defaults: avgBlockSize * 11 / 10, avgBlockSize * 21 / 10 - * @see #SLAB_CACHE_PROPORTIONS_KEY - */ - static final String SLAB_CACHE_SIZES_KEY = "hbase.offheapcache.slab.sizes"; - - /** - * Default constructor, creates an empty SlabCache. - * - * @param size Total size allocated to the SlabCache. (Bytes) - * @param avgBlockSize Average size of a block being cached. - **/ - public SlabCache(long size, long avgBlockSize) { - this.avgBlockSize = avgBlockSize; - this.size = size; - this.stats = new CacheStats(); - this.requestStats = new SlabStats(); - this.successfullyCachedStats = new SlabStats(); - - backingStore = new ConcurrentHashMap(); - slabs = new TreeMap(); - this.scheduleThreadPool.scheduleAtFixedRate(new StatisticsThread(this), - STAT_THREAD_PERIOD_SECS, STAT_THREAD_PERIOD_SECS, TimeUnit.SECONDS); - } - - public Map getSizer() { - return slabs; - } - - /** - * A way of allocating the desired amount of Slabs of each particular size. - * - * This reads two lists from conf, hbase.offheap.slab.proportions and - * hbase.offheap.slab.sizes. - * - * The first list is the percentage of our total space we allocate to the - * slabs. - * - * The second list is blocksize of the slabs in bytes. (E.g. the slab holds - * blocks of this size). - * - * @param conf Configuration file. - */ - public void addSlabByConf(Configuration conf) { - // Proportions we allocate to each slab of the total size. - String[] porportions = conf.getStrings(SLAB_CACHE_PROPORTIONS_KEY, "0.80", "0.20"); - String[] sizes = conf.getStrings(SLAB_CACHE_SIZES_KEY, - Long.valueOf(avgBlockSize * 11 / 10).toString(), - Long.valueOf(avgBlockSize * 21 / 10).toString()); - - if (porportions.length != sizes.length) { - throw new IllegalArgumentException( - "SlabCache conf not " - + "initialized, error in configuration. hbase.offheap.slab.proportions specifies " - + porportions.length - + " slabs while hbase.offheap.slab.sizes specifies " - + sizes.length + " slabs " - + "offheapslabporportions and offheapslabsizes"); - } - /* - * We use BigDecimals instead of floats because float rounding is annoying - */ - - BigDecimal[] parsedProportions = stringArrayToBigDecimalArray(porportions); - BigDecimal[] parsedSizes = stringArrayToBigDecimalArray(sizes); - - BigDecimal sumProportions = new BigDecimal(0); - for (BigDecimal b : parsedProportions) { - /* Make sure all proportions are greater than 0 */ - Preconditions - .checkArgument(b.compareTo(BigDecimal.ZERO) == 1, - "Proportions in hbase.offheap.slab.proportions must be greater than 0!"); - sumProportions = sumProportions.add(b); - } - - /* If the sum is greater than 1 */ - Preconditions - .checkArgument(sumProportions.compareTo(BigDecimal.ONE) != 1, - "Sum of all proportions in hbase.offheap.slab.proportions must be less than 1"); - - /* If the sum of all proportions is less than 0.99 */ - if (sumProportions.compareTo(new BigDecimal("0.99")) == -1) { - LOG.warn("Sum of hbase.offheap.slab.proportions is less than 0.99! Memory is being wasted"); - } - for (int i = 0; i < parsedProportions.length; i++) { - int blockSize = parsedSizes[i].intValue(); - int numBlocks = new BigDecimal(this.size).multiply(parsedProportions[i]) - .divide(parsedSizes[i], BigDecimal.ROUND_DOWN).intValue(); - addSlab(blockSize, numBlocks); - } - } - - /** - * Gets the size of the slab cache a ByteBuffer of this size would be - * allocated to. - * - * @param size Size of the ByteBuffer we are checking. - * - * @return the Slab that the above bytebuffer would be allocated towards. If - * object is too large, returns null. - */ - Entry getHigherBlock(int size) { - return slabs.higherEntry(size - 1); - } - - private BigDecimal[] stringArrayToBigDecimalArray(String[] parsee) { - BigDecimal[] parsed = new BigDecimal[parsee.length]; - for (int i = 0; i < parsee.length; i++) { - parsed[i] = new BigDecimal(parsee[i].trim()); - } - return parsed; - } - - private void addSlab(int blockSize, int numBlocks) { - LOG.info("Creating slab of blockSize " + blockSize + " with " + numBlocks - + " blocks, " + StringUtils.byteDesc(blockSize * (long) numBlocks) + "bytes."); - slabs.put(blockSize, new SingleSizeCache(blockSize, numBlocks, this)); - } - - /** - * Cache the block with the specified key and buffer. First finds what size - * SingleSlabCache it should fit in. If the block doesn't fit in any, it will - * return without doing anything. - *

- * It is assumed this will NEVER be called on an already cached block. If that - * is done, it is assumed that you are reinserting the same exact block due to - * a race condition, and will throw a runtime exception. - * - * @param cacheKey block cache key - * @param cachedItem block buffer - */ - public void cacheBlock(BlockCacheKey cacheKey, Cacheable cachedItem) { - Entry scacheEntry = getHigherBlock(cachedItem - .getSerializedLength()); - - this.requestStats.addin(cachedItem.getSerializedLength()); - - if (scacheEntry == null) { - return; // we can't cache, something too big. - } - - this.successfullyCachedStats.addin(cachedItem.getSerializedLength()); - SingleSizeCache scache = scacheEntry.getValue(); - - /* - * This will throw a runtime exception if we try to cache the same value - * twice - */ - scache.cacheBlock(cacheKey, cachedItem); - } - - /** - * We don't care about whether its in memory or not, so we just pass the call - * through. - */ - public void cacheBlock(BlockCacheKey cacheKey, Cacheable buf, boolean inMemory, - final boolean cacheDataInL1) { - cacheBlock(cacheKey, buf); - } - - public CacheStats getStats() { - return this.stats; - } - - /** - * Get the buffer of the block with the specified name. - * - * @return buffer of specified block name, or null if not in cache - */ - public Cacheable getBlock(BlockCacheKey key, boolean caching, boolean repeat, - boolean updateCacheMetrics) { - SingleSizeCache cachedBlock = backingStore.get(key); - if (cachedBlock == null) { - if (!repeat) stats.miss(caching); - return null; - } - - Cacheable contentBlock = cachedBlock.getBlock(key, caching, false, updateCacheMetrics); - - if (contentBlock != null) { - if (updateCacheMetrics) stats.hit(caching); - } else if (!repeat) { - if (updateCacheMetrics) stats.miss(caching); - } - return contentBlock; - } - - /** - * Evicts a block from the cache. This is public, and thus contributes to the - * the evict counter. - */ - public boolean evictBlock(BlockCacheKey cacheKey) { - SingleSizeCache cacheEntry = backingStore.get(cacheKey); - if (cacheEntry == null) { - return false; - } else { - cacheEntry.evictBlock(cacheKey); - return true; - } - } - - @Override - public void onEviction(BlockCacheKey key, SingleSizeCache notifier) { - stats.evicted(); - backingStore.remove(key); - } - - @Override - public void onInsertion(BlockCacheKey key, SingleSizeCache notifier) { - backingStore.put(key, notifier); - } - - /** - * Sends a shutdown to all SingleSizeCache's contained by this cache. - * - * Also terminates the scheduleThreadPool. - */ - public void shutdown() { - for (SingleSizeCache s : slabs.values()) { - s.shutdown(); - } - this.scheduleThreadPool.shutdown(); - } - - public long heapSize() { - long childCacheSize = 0; - for (SingleSizeCache s : slabs.values()) { - childCacheSize += s.heapSize(); - } - return SlabCache.CACHE_FIXED_OVERHEAD + childCacheSize; - } - - public long size() { - return this.size; - } - - public long getFreeSize() { - long childFreeSize = 0; - for (SingleSizeCache s : slabs.values()) { - childFreeSize += s.getFreeSize(); - } - return childFreeSize; - } - - @Override - public long getBlockCount() { - long count = 0; - for (SingleSizeCache cache : slabs.values()) { - count += cache.getBlockCount(); - } - return count; - } - - public long getCurrentSize() { - return size; - } - - public long getEvictedCount() { - return stats.getEvictedCount(); - } - - /* - * Statistics thread. Periodically prints the cache statistics to the log. - * TODO: Fix. Just emit to metrics. Don't run a thread just to do a log. - */ - static class StatisticsThread extends HasThread { - SlabCache ourcache; - - public StatisticsThread(SlabCache slabCache) { - super("SlabCache.StatisticsThread"); - setDaemon(true); - this.ourcache = slabCache; - } - - @Override - public void run() { - for (SingleSizeCache s : ourcache.slabs.values()) { - s.logStats(); - } - - SlabCache.LOG.info("Current heap size is: " - + StringUtils.humanReadableInt(ourcache.heapSize())); - - LOG.info("Request Stats"); - ourcache.requestStats.logStats(); - LOG.info("Successfully Cached Stats"); - ourcache.successfullyCachedStats.logStats(); - } - - } - - /** - * Just like CacheStats, but more Slab specific. Finely grained profiling of - * sizes we store using logs. - * - */ - static class SlabStats { - // the maximum size somebody will ever try to cache, then we multiply by - // 10 - // so we have finer grained stats. - static final int MULTIPLIER = 10; - final int NUMDIVISIONS = (int) (Math.log(Integer.MAX_VALUE) * MULTIPLIER); - private final AtomicLong[] counts = new AtomicLong[NUMDIVISIONS]; - - public SlabStats() { - for (int i = 0; i < NUMDIVISIONS; i++) { - counts[i] = new AtomicLong(); - } - } - - public void addin(int size) { - int index = (int) (Math.log(size) * MULTIPLIER); - counts[index].incrementAndGet(); - } - - public AtomicLong[] getUsage() { - return counts; - } - - double getUpperBound(int index) { - return Math.pow(Math.E, ((index + 0.5) / MULTIPLIER)); - } - - double getLowerBound(int index) { - return Math.pow(Math.E, ((index - 0.5) / MULTIPLIER)); - } - - public void logStats() { - AtomicLong[] fineGrainedStats = getUsage(); - for (int i = 0; i < fineGrainedStats.length; i++) { - - if (fineGrainedStats[i].get() > 0) { - SlabCache.LOG.info("From " - + StringUtils.humanReadableInt((long) getLowerBound(i)) + "- " - + StringUtils.humanReadableInt((long) getUpperBound(i)) + ": " - + StringUtils.humanReadableInt(fineGrainedStats[i].get()) - + " requests"); - - } - } - } - } - - public int evictBlocksByHfileName(String hfileName) { - int numEvicted = 0; - for (BlockCacheKey key : backingStore.keySet()) { - if (key.getHfileName().equals(hfileName)) { - if (evictBlock(key)) - ++numEvicted; - } - } - return numEvicted; - } - - @Override - public Iterator iterator() { - // Don't bother with ramcache since stuff is in here only a little while. - final Iterator> i = - this.backingStore.entrySet().iterator(); - return new Iterator() { - private final long now = System.nanoTime(); - - @Override - public boolean hasNext() { - return i.hasNext(); - } - - @Override - public CachedBlock next() { - final Map.Entry e = i.next(); - final Cacheable cacheable = e.getValue().getBlock(e.getKey(), false, false, false); - return new CachedBlock() { - @Override - public String toString() { - return BlockCacheUtil.toString(this, now); - } - - @Override - public BlockPriority getBlockPriority() { - return null; - } - - @Override - public BlockType getBlockType() { - return cacheable.getBlockType(); - } - - @Override - public long getOffset() { - return e.getKey().getOffset(); - } - - @Override - public long getSize() { - return cacheable == null? 0: cacheable.getSerializedLength(); - } - - @Override - public long getCachedTime() { - return -1; - } - - @Override - public String getFilename() { - return e.getKey().getHfileName(); - } - - @Override - public int compareTo(CachedBlock other) { - return (int)(this.getOffset() - other.getOffset()); - } - }; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - }; - } - - @Override - public BlockCache[] getBlockCaches() { - return null; - } -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java deleted file mode 100644 index 93b35db..0000000 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/slab/SlabItemActionWatcher.java +++ /dev/null @@ -1,49 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.io.hfile.slab; - -import org.apache.hadoop.classification.InterfaceAudience; -import org.apache.hadoop.hbase.io.hfile.BlockCacheKey; - -/** - * Interface for objects that want to know when actions occur in a SingleSizeCache. - * - * @deprecated As of 1.0, replaced by {@link org.apache.hadoop.hbase.io.hfile.bucket.BucketCache}. - */ -@InterfaceAudience.Private -@Deprecated -interface SlabItemActionWatcher { - - /** - * This is called as a callback when an item is removed from a SingleSizeCache. - * - * @param key the key of the item being evicted - * @param notifier the object notifying the SlabCache of the eviction. - */ - void onEviction(BlockCacheKey key, SingleSizeCache notifier); - - /** - * This is called as a callback when an item is inserted into a SingleSizeCache. - * - * @param key the key of the item being added - * @param notifier the object notifying the SlabCache of the insertion.. - */ - void onInsertion(BlockCacheKey key, SingleSizeCache notifier); -} diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java index 03bc4f0..facef82 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java @@ -281,9 +281,7 @@ public class TableMapReduceUtil { public static void resetCacheConfig(Configuration conf) { conf.setFloat( HConstants.HFILE_BLOCK_CACHE_SIZE_KEY, HConstants.HFILE_BLOCK_CACHE_SIZE_DEFAULT); - conf.setFloat(CacheConfig.SLAB_CACHE_OFFHEAP_PERCENTAGE_KEY, 0f); conf.setFloat(CacheConfig.BUCKET_CACHE_SIZE_KEY, 0f); - conf.setFloat("hbase.offheapcache.percentage", 0f); conf.setFloat("hbase.bucketcache.size", 0f); } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java index 38a72b4..fa1f10f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestBlockCacheReporting.java @@ -70,23 +70,6 @@ public class TestBlockCacheReporting { } @Test - public void testSlabCacheConfig() throws JsonGenerationException, JsonMappingException, IOException { - this.conf.setFloat(CacheConfig.SLAB_CACHE_OFFHEAP_PERCENTAGE_KEY, 0.1f); - CacheConfig cc = new CacheConfig(this.conf); - assertTrue(cc.getBlockCache() instanceof DoubleBlockCache); - logPerBlock(cc.getBlockCache()); - final int count = 3; - addDataAndHits(cc.getBlockCache(), count); - // The below has no asserts. It is just exercising toString and toJSON code. - LOG.info(cc.getBlockCache().getStats()); - BlockCacheUtil.CachedBlocksByFile cbsbf = logPerBlock(cc.getBlockCache()); - LOG.info(cbsbf); - logPerFile(cbsbf); - bucketCacheReport(cc.getBlockCache()); - LOG.info(BlockCacheUtil.toJSON(cbsbf)); - } - - @Test public void testBucketCache() throws JsonGenerationException, JsonMappingException, IOException { this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap"); this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, 100); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java index 0c37b3b..8f7ed2b 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/TestCacheConfig.java @@ -198,15 +198,6 @@ public class TestCacheConfig { } @Test - public void testSlabCacheConfig() { - this.conf.setFloat(CacheConfig.SLAB_CACHE_OFFHEAP_PERCENTAGE_KEY, 0.1f); - CacheConfig cc = new CacheConfig(this.conf); - basicBlockCacheOps(cc, true, true); - assertTrue(cc.getBlockCache() instanceof DoubleBlockCache); - // TODO Assert sizes allocated are right. - } - - @Test public void testBucketCacheConfig() { this.conf.set(CacheConfig.BUCKET_CACHE_IOENGINE_KEY, "offheap"); this.conf.setInt(CacheConfig.BUCKET_CACHE_SIZE_KEY, 100); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java deleted file mode 100644 index 8a0dc16..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSingleSizeCache.java +++ /dev/null @@ -1,83 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.io.hfile.slab; - -import org.apache.hadoop.hbase.MediumTests; -import org.apache.hadoop.hbase.io.hfile.CacheTestUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.experimental.categories.Category; - -/** - * Tests SingleSlabCache. - *

- * - * Tests will ensure that evictions operate when they're supposed to and do what - * they should, and that cached blocks are accessible when expected to be. - */ -// Starts 100 threads, high variability of execution time => Medium -@Category(MediumTests.class) -public class TestSingleSizeCache { - SingleSizeCache cache; - final int CACHE_SIZE = 1000000; - final int NUM_BLOCKS = 100; - final int BLOCK_SIZE = CACHE_SIZE / NUM_BLOCKS; - final int NUM_THREADS = 100; - final int NUM_QUERIES = 10000; - - @Before - public void setup() { - cache = new SingleSizeCache(BLOCK_SIZE, NUM_BLOCKS, null); - } - - @After - public void tearDown() { - cache.shutdown(); - } - - @Test - public void testCacheSimple() throws Exception { - CacheTestUtils.testCacheSimple(cache, BLOCK_SIZE, NUM_QUERIES); - } - - @Test - public void testCacheMultiThreaded() throws Exception { - CacheTestUtils.testCacheMultiThreaded(cache, BLOCK_SIZE, - NUM_THREADS, NUM_QUERIES, 0.80); - } - - @Test - public void testCacheMultiThreadedSingleKey() throws Exception { - CacheTestUtils.hammerSingleKey(cache, BLOCK_SIZE, NUM_THREADS, NUM_QUERIES); - } - - @Test - public void testCacheMultiThreadedEviction() throws Exception { - CacheTestUtils.hammerEviction(cache, BLOCK_SIZE, NUM_THREADS, NUM_QUERIES); - } - - @Test - public void testHeapSizeChanges(){ - CacheTestUtils.testHeapSizeChanges(cache, BLOCK_SIZE); - } - - -} - diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlab.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlab.java deleted file mode 100644 index 71d708a..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlab.java +++ /dev/null @@ -1,77 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hbase.io.hfile.slab; - -import static org.junit.Assert.*; -import java.nio.ByteBuffer; - -import org.apache.hadoop.hbase.SmallTests; -import org.junit.*; -import org.junit.experimental.categories.Category; - -/**Test cases for Slab.java*/ -@Category(SmallTests.class) -public class TestSlab { - static final int BLOCKSIZE = 1000; - static final int NUMBLOCKS = 100; - Slab testSlab; - ByteBuffer[] buffers = new ByteBuffer[NUMBLOCKS]; - - @Before - public void setUp() { - testSlab = new Slab(BLOCKSIZE, NUMBLOCKS); - } - - @After - public void tearDown() { - testSlab.shutdown(); - } - - @Test - public void testBasicFunctionality() throws InterruptedException { - for (int i = 0; i < NUMBLOCKS; i++) { - buffers[i] = testSlab.alloc(BLOCKSIZE); - assertEquals(BLOCKSIZE, buffers[i].limit()); - } - - // write an unique integer to each allocated buffer. - for (int i = 0; i < NUMBLOCKS; i++) { - buffers[i].putInt(i); - } - - // make sure the bytebuffers remain unique (the slab allocator hasn't - // allocated the same chunk of memory twice) - for (int i = 0; i < NUMBLOCKS; i++) { - buffers[i].putInt(i); - } - - for (int i = 0; i < NUMBLOCKS; i++) { - testSlab.free(buffers[i]); // free all the buffers. - } - - for (int i = 0; i < NUMBLOCKS; i++) { - buffers[i] = testSlab.alloc(BLOCKSIZE); - assertEquals(BLOCKSIZE, buffers[i].limit()); - } - } - - -} - diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java deleted file mode 100644 index 41ee705..0000000 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/hfile/slab/TestSlabCache.java +++ /dev/null @@ -1,111 +0,0 @@ -/** - * - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hbase.io.hfile.slab; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.MediumTests; -import org.apache.hadoop.hbase.io.hfile.CacheTestUtils; -import org.apache.hadoop.hbase.io.hfile.slab.SlabCache.SlabStats; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.junit.Ignore; -import org.junit.experimental.categories.Category; - -/** - * Basic test of SlabCache. Puts and gets. - *

- * - * Tests will ensure that blocks that are uncached are identical to the ones - * being cached, and that the cache never exceeds its capacity. Note that its - * fine if the cache evicts before it reaches max capacity - Guava Mapmaker may - * choose to evict at any time. - * - */ -// Starts 50 threads, high variability of execution time => Medium -@Category(MediumTests.class) -public class TestSlabCache { - static final int CACHE_SIZE = 1000000; - static final int NUM_BLOCKS = 101; - static final int BLOCK_SIZE = CACHE_SIZE / NUM_BLOCKS; - static final int NUM_THREADS = 50; - static final int NUM_QUERIES = 10000; - SlabCache cache; - - @Before - public void setup() { - cache = new SlabCache(CACHE_SIZE + BLOCK_SIZE * 2, BLOCK_SIZE); - cache.addSlabByConf(new Configuration()); - } - - @After - public void tearDown() { - cache.shutdown(); - } - - @Test - public void testElementPlacement() { - assertEquals(cache.getHigherBlock(BLOCK_SIZE).getKey().intValue(), - (BLOCK_SIZE * 11 / 10)); - assertEquals(cache.getHigherBlock((BLOCK_SIZE * 2)).getKey() - .intValue(), (BLOCK_SIZE * 21 / 10)); - } - - @Test - public void testCacheSimple() throws Exception { - CacheTestUtils.testCacheSimple(cache, BLOCK_SIZE, NUM_QUERIES); - } - - @Test - public void testCacheMultiThreaded() throws Exception { - CacheTestUtils.testCacheMultiThreaded(cache, BLOCK_SIZE, NUM_THREADS, - NUM_QUERIES, 0.80); - } - - @Test - public void testCacheMultiThreadedSingleKey() throws Exception { - CacheTestUtils.hammerSingleKey(cache, BLOCK_SIZE, NUM_THREADS, NUM_QUERIES); - } - - @Test - public void testCacheMultiThreadedEviction() throws Exception { - CacheTestUtils.hammerEviction(cache, BLOCK_SIZE, 10, NUM_QUERIES); - } - - @Test - /*Just checks if ranges overlap*/ - public void testStatsArithmetic(){ - SlabStats test = cache.requestStats; - for(int i = 0; i < test.NUMDIVISIONS; i++){ - assertTrue("Upper for index " + i + " is " + test.getUpperBound(i) + - " lower " + test.getLowerBound(i + 1), - test.getUpperBound(i) <= test.getLowerBound(i + 1)); - } - } - - @Test - public void testHeapSizeChanges(){ - CacheTestUtils.testHeapSizeChanges(cache, BLOCK_SIZE); - } - -} - -- 1.9.0