From 2524790d2dca5ec6b671c43e11feac149ddc9729 Mon Sep 17 00:00:00 2001 From: gsheffi Date: Tue, 6 Feb 2018 17:37:42 +0200 Subject: [PATCH] for creating patch HBASE-19506-V02.patch --- .../regionserver/CellChunkImmutableSegment.java | 78 ++++-- .../hadoop/hbase/regionserver/ChunkCreator.java | 277 +++++++++++++++------ .../hbase/regionserver/ImmutableMemStoreLAB.java | 4 +- .../hadoop/hbase/regionserver/MemStoreLAB.java | 4 +- .../hadoop/hbase/regionserver/MemStoreLABImpl.java | 61 +++-- 5 files changed, 310 insertions(+), 114 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java index bf9b1915f7..452897ec9a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java @@ -30,7 +30,8 @@ import org.apache.hadoop.hbase.KeyValueUtil; import org.apache.hadoop.hbase.util.ByteBufferUtils; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.yetus.audience.InterfaceAudience; - +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** @@ -43,6 +44,8 @@ public class CellChunkImmutableSegment extends ImmutableSegment { public static final long DEEP_OVERHEAD_CCM = ImmutableSegment.DEEP_OVERHEAD + ClassSize.CELL_CHUNK_MAP; + public static final float INDEX_CHUNK_UNUSED_SPACE_PRECENTAGE = 0.1f; + private static final Logger LOG = LoggerFactory.getLogger(CellChunkImmutableSegment.class); ///////////////////// CONSTRUCTORS ///////////////////// /**------------------------------------------------------------------------ @@ -96,20 +99,12 @@ public class CellChunkImmutableSegment extends ImmutableSegment { private void initializeCellSet(int numOfCells, MemStoreSegmentsIterator iterator, MemStoreCompactionStrategy.Action action) { - // calculate how many chunks we will need for index - int chunkSize = ChunkCreator.getInstance().getChunkSize(); - int numOfCellsInChunk = CellChunkMap.NUM_OF_CELL_REPS_IN_CHUNK; - int numberOfChunks = calculateNumberOfChunks(numOfCells, numOfCellsInChunk); int numOfCellsAfterCompaction = 0; int currentChunkIdx = 0; int offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER; int numUniqueKeys=0; Cell prev = null; - // all index Chunks are allocated from ChunkCreator - Chunk[] chunks = new Chunk[numberOfChunks]; - for (int i=0; i < numberOfChunks; i++) { - chunks[i] = this.getMemStoreLAB().getNewExternalChunk(); - } + Chunk[] chunks = allocIndexChunks(numOfCells); while (iterator.hasNext()) { // the iterator hides the elimination logic for compaction boolean alreadyCopied = false; Cell c = iterator.next(); @@ -122,7 +117,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment { c = copyCellIntoMSLAB(c); alreadyCopied = true; } - if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkSize) { + if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunks[currentChunkIdx].size) { currentChunkIdx++; // continue to the next index chunk offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER; } @@ -168,15 +163,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment { int numOfCells, KeyValueScanner segmentScanner, CellSet oldCellSet, MemStoreCompactionStrategy.Action action) { Cell curCell; - // calculate how many chunks we will need for metadata - int chunkSize = ChunkCreator.getInstance().getChunkSize(); - int numOfCellsInChunk = CellChunkMap.NUM_OF_CELL_REPS_IN_CHUNK; - int numberOfChunks = calculateNumberOfChunks(numOfCells, numOfCellsInChunk); - // all index Chunks are allocated from ChunkCreator - Chunk[] chunks = new Chunk[numberOfChunks]; - for (int i=0; i < numberOfChunks; i++) { - chunks[i] = this.getMemStoreLAB().getNewExternalChunk(); - } + Chunk[] chunks = allocIndexChunks(numOfCells); int currentChunkIdx = 0; int offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER; @@ -192,7 +179,7 @@ public class CellChunkImmutableSegment extends ImmutableSegment { // are copied into MSLAB here. curCell = copyCellIntoMSLAB(curCell); } - if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkSize) { + if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunks[currentChunkIdx].size) { // continue to the next metadata chunk currentChunkIdx++; offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER; @@ -240,14 +227,57 @@ public class CellChunkImmutableSegment extends ImmutableSegment { return offset; } - private int calculateNumberOfChunks(int numOfCells, int numOfCellsInChunk) { - int numberOfChunks = numOfCells/numOfCellsInChunk; - if(numOfCells%numOfCellsInChunk!=0) { // if cells cannot be divided evenly between chunks + private int calculateNumberOfChunks(int numOfCells, int chunkSize) { + int numOfCellsInChunk = calcNumOfCellsInChunk(chunkSize); + int numberOfChunks = numOfCells / numOfCellsInChunk; + if(numOfCells % numOfCellsInChunk != 0) { // if cells cannot be divided evenly between chunks numberOfChunks++; // add one additional chunk } return numberOfChunks; } + // Assuming we are going to use index chunks of a regular chunk size, + // we check here how much free space will remain in the last allocated chunk + // (the least occupied one). + // If the percentage of its remaining free space is above the INDEX_CHUNK_UNUSED_SPACE + // threshold, then we will use small chunks as index chunks. + private boolean useSmallIndexChunks(int numOfCells) { + int regularChunkSize = ChunkCreator.getInstance().getChunkSize(); + int numOfCellsInChunk = calcNumOfCellsInChunk(regularChunkSize); + int cellsInLastChunk = numOfCells % numOfCellsInChunk; + if (cellsInLastChunk == 0) { // There is no free space in the last chunk and thus, + return false; // no need to use small chunks. + } else { + int chunkSpace = regularChunkSize - ChunkCreator.SIZEOF_CHUNK_HEADER; + int freeSpaceInLastChunk = chunkSpace - cellsInLastChunk * ClassSize.CELL_CHUNK_MAP_ENTRY; + if (freeSpaceInLastChunk > INDEX_CHUNK_UNUSED_SPACE_PRECENTAGE * chunkSpace) { + return true; + } + return false; + } + } + + private int calcNumOfCellsInChunk(int chunkSize) { + int chunkSpace = chunkSize - ChunkCreator.SIZEOF_CHUNK_HEADER; + int numOfCellsInChunk = chunkSpace / ClassSize.CELL_CHUNK_MAP_ENTRY; + return numOfCellsInChunk; + } + + private Chunk[] allocIndexChunks(int numOfCells) { + // Decide whether to use regular or small chunks and then + // calculate how many chunks we will need for index + boolean smallIndexChunks = useSmallIndexChunks(numOfCells); + int chunkSize = ChunkCreator.getInstance().getChunkSize(smallIndexChunks); + int numberOfChunks = calculateNumberOfChunks(numOfCells, chunkSize); + // all index Chunks are allocated from ChunkCreator + Chunk[] chunks = new Chunk[numberOfChunks]; + // all index Chunks are allocated from ChunkCreator + for (int i=0; i < numberOfChunks; i++) { + chunks[i] = this.getMemStoreLAB().getNewExternalChunk(smallIndexChunks); + } + return chunks; + } + private Cell copyCellIntoMSLAB(Cell cell) { // Take care for a special case when a cell is copied from on-heap to (probably off-heap) MSLAB. // The cell allocated as an on-heap JVM object (byte array) occupies slightly different diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java index b2d4ba8807..bb1a8abb37 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java @@ -59,24 +59,77 @@ public class ChunkCreator { // mapping from chunk IDs to chunks private Map chunkIdMap = new ConcurrentHashMap(); - private final int chunkSize; private final boolean offheap; @VisibleForTesting static ChunkCreator INSTANCE; @VisibleForTesting static boolean chunkPoolDisabled = false; - private MemStoreChunkPool pool; + private MemStoreChunkPool chunksPool; + private MemStoreChunkPool smallChunksPool; @VisibleForTesting ChunkCreator(int chunkSize, boolean offheap, long globalMemStoreSize, float poolSizePercentage, - float initialCountPercentage, HeapMemoryManager heapMemoryManager) { - this.chunkSize = chunkSize; + float initialCountPercentage, HeapMemoryManager heapMemoryManager, + float smallChunkSizePercentage) { this.offheap = offheap; - this.pool = initializePool(globalMemStoreSize, poolSizePercentage, initialCountPercentage); - if (heapMemoryManager != null && this.pool != null) { + this.chunksPool = initializePool(globalMemStoreSize, + (1 - smallChunkSizePercentage) * poolSizePercentage, + initialCountPercentage, chunkSize); + if (heapMemoryManager != null && this.chunksPool != null) { // Register with Heap Memory manager - heapMemoryManager.registerTuneObserver(this.pool); + heapMemoryManager.registerTuneObserver(this.chunksPool); } + this.smallChunksPool = initializePool(globalMemStoreSize, + smallChunkSizePercentage * poolSizePercentage, + initialCountPercentage, (int) (smallChunkSizePercentage * chunkSize)); + if (heapMemoryManager != null && this.smallChunksPool != null) { + // Register with Heap Memory manager + heapMemoryManager.registerTuneObserver(this.smallChunksPool); + } + } + + @VisibleForTesting + ChunkCreator(int chunkSize, boolean offheap, long globalMemStoreSize, float poolSizePercentage, + float initialCountPercentage, HeapMemoryManager heapMemoryManager) { + this.offheap = offheap; + float smallChunkSizePercentage = MemStoreLABImpl.SMALL_CHUNK_SIZE_DEFAULT; + this.chunksPool = initializePool(globalMemStoreSize, + (1 - smallChunkSizePercentage) * poolSizePercentage, + initialCountPercentage, chunkSize); + if (heapMemoryManager != null && this.chunksPool != null) { + // Register with Heap Memory manager + heapMemoryManager.registerTuneObserver(this.chunksPool); + } + this.smallChunksPool = initializePool(globalMemStoreSize, + smallChunkSizePercentage * poolSizePercentage, + initialCountPercentage, (int) smallChunkSizePercentage * chunkSize); + if (heapMemoryManager != null && this.smallChunksPool != null) { + // Register with Heap Memory manager + heapMemoryManager.registerTuneObserver(this.smallChunksPool); + } + } + + /** + * Initializes the instance of ChunkCreator + * @param chunkSize the chunkSize + * @param offheap indicates if the chunk is to be created offheap or not + * @param globalMemStoreSize the global memstore size + * @param poolSizePercentage pool size percentage + * @param initialCountPercentage the initial count of the chunk pool if any + * @param heapMemoryManager the heapmemory manager + * @param smallChunkSizePercentage small chunk pool percentage + * @return singleton MSLABChunkCreator + */ + @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "LI_LAZY_INIT_STATIC", + justification = "Method is called by single thread at the starting of RS") + @VisibleForTesting + public static ChunkCreator initialize(int chunkSize, boolean offheap, long globalMemStoreSize, + float poolSizePercentage, float initialCountPercentage, HeapMemoryManager heapMemoryManager, + float smallChunkSizePercentage) { + if (INSTANCE != null) return INSTANCE; + INSTANCE = new ChunkCreator(chunkSize, offheap, globalMemStoreSize, poolSizePercentage, + initialCountPercentage, heapMemoryManager, smallChunkSizePercentage); + return INSTANCE; } /** @@ -90,13 +143,14 @@ public class ChunkCreator { * @return singleton MSLABChunkCreator */ @edu.umd.cs.findbugs.annotations.SuppressWarnings(value = "LI_LAZY_INIT_STATIC", - justification = "Method is called by single thread at the starting of RS") + justification = "Method is called by single thread at the starting of RS") @VisibleForTesting public static ChunkCreator initialize(int chunkSize, boolean offheap, long globalMemStoreSize, - float poolSizePercentage, float initialCountPercentage, HeapMemoryManager heapMemoryManager) { + float poolSizePercentage, float initialCountPercentage, + HeapMemoryManager heapMemoryManager) { if (INSTANCE != null) return INSTANCE; INSTANCE = new ChunkCreator(chunkSize, offheap, globalMemStoreSize, poolSizePercentage, - initialCountPercentage, heapMemoryManager); + initialCountPercentage, heapMemoryManager, MemStoreLABImpl.SMALL_CHUNK_SIZE_DEFAULT); return INSTANCE; } @@ -108,8 +162,32 @@ public class ChunkCreator { * Creates and inits a chunk. The default implementation. * @return the chunk that was initialized */ + Chunk getChunk(boolean small) { + if (small) { + return getChunk(CompactingMemStore.IndexType.ARRAY_MAP, smallChunksPool.getChunkSize()); + } else { + return getChunk(CompactingMemStore.IndexType.ARRAY_MAP, chunksPool.getChunkSize()); + } + } + + /** + * Creates and inits a chunk. The default implementation. + * @return the chunk that was initialized + */ Chunk getChunk() { - return getChunk(CompactingMemStore.IndexType.ARRAY_MAP, chunkSize); + return getChunk(false); + } + + /** + * Creates and inits a chunk. The default implementation for specific index type. + * @return the chunk that was initialized + */ + Chunk getChunk(CompactingMemStore.IndexType chunkIndexType, boolean small) { + if (small) { + return getChunk(chunkIndexType, smallChunksPool.getChunkSize()); + } else { + return getChunk(chunkIndexType, chunksPool.getChunkSize()); + } } /** @@ -117,7 +195,7 @@ public class ChunkCreator { * @return the chunk that was initialized */ Chunk getChunk(CompactingMemStore.IndexType chunkIndexType) { - return getChunk(chunkIndexType, chunkSize); + return getChunk(chunkIndexType, getChunkSize()); } /** @@ -128,18 +206,28 @@ public class ChunkCreator { */ Chunk getChunk(CompactingMemStore.IndexType chunkIndexType, int size) { Chunk chunk = null; - // if we have pool and this is not jumbo chunk (when size != chunkSize this is jumbo chunk) - if ((pool != null) && (size == chunkSize)) { + MemStoreChunkPool pool = null; + + // if the size is suitable for one of the pools + if (size == chunksPool.getChunkSize()) { + pool = chunksPool; + } else if (size == smallChunksPool.getChunkSize()) { + pool = smallChunksPool; + } + + // if we have a pool + if (pool != null) { // the pool creates the chunk internally. The chunk#init() call happens here - chunk = this.pool.getChunk(); + chunk = pool.getChunk(); // the pool has run out of maxCount if (chunk == null) { if (LOG.isTraceEnabled()) { - LOG.trace("Chunk pool full (maxCount={}); creating chunk offheap.", - this.pool.getMaxCount()); + LOG.trace("The chunk pool is full. Reached maxCount= " + pool.getMaxCount() + + ". Creating chunk onheap."); } } } + if (chunk == null) { // the second parameter explains whether CellChunkMap index is requested, // in that case, put allocated on demand chunk mapping into chunkIdMap @@ -160,14 +248,15 @@ public class ChunkCreator { * @param jumboSize the special size to be used */ Chunk getJumboChunk(CompactingMemStore.IndexType chunkIndexType, int jumboSize) { - if (jumboSize <= chunkSize) { - LOG.warn("Jumbo chunk size=" + jumboSize + " must be more than regular chunk size=" - + chunkSize + "; converting to regular chunk."); - return getChunk(chunkIndexType,chunkSize); + int allocSize = jumboSize + SIZEOF_CHUNK_HEADER; + if (allocSize <= chunksPool.getChunkSize()) { + LOG.warn("Jumbo chunk size " + jumboSize + " must be more than regular chunk size " + + chunksPool.getChunkSize() + ". Converting to regular chunk."); + return getChunk(chunkIndexType); } // the size of the allocation includes // both the size requested and a place for the Chunk's header - return getChunk(chunkIndexType, jumboSize + SIZEOF_CHUNK_HEADER); + return getChunk(chunkIndexType, allocSize); } /** @@ -196,19 +285,29 @@ public class ChunkCreator { // Chunks from pool are created covered with strong references anyway // TODO: change to CHUNK_MAP if it is generally defined - private Chunk createChunkForPool() { - return createChunk(true, CompactingMemStore.IndexType.ARRAY_MAP, chunkSize); + private Chunk createChunkForPool(boolean small) { + if (small) { + return createChunk(true, CompactingMemStore.IndexType.ARRAY_MAP, + smallChunksPool.getChunkSize()); + } else { + return createChunk(true, CompactingMemStore.IndexType.ARRAY_MAP, + chunksPool.getChunkSize()); + } } @VisibleForTesting - // Used to translate the ChunkID into a chunk ref + // Used to translate the ChunkID into a chunk ref Chunk getChunk(int id) { // can return null if chunk was never mapped return chunkIdMap.get(id); } - int getChunkSize() { - return this.chunkSize; + int getChunkSize(boolean small) { + if (small) { + return smallChunksPool.getChunkSize(); + } else { + return chunksPool.getChunkSize(); + } } boolean isOffheap() { @@ -224,8 +323,8 @@ public class ChunkCreator { } @VisibleForTesting - // the chunks in the chunkIdMap may already be released so we shouldn't relay - // on this counting for strong correctness. This method is used only in testing. + // the chunks in the chunkIdMap may already be released so we shouldn't relay + // on this counting for strong correctness. This method is used only in testing. int numberOfMappedChunks() { return this.chunkIdMap.size(); } @@ -243,6 +342,7 @@ public class ChunkCreator { * collection on JVM. */ private class MemStoreChunkPool implements HeapMemoryTuneObserver { + private final int poolChunkSize; private int maxCount; // A queue of reclaimed chunks @@ -256,21 +356,22 @@ public class ChunkCreator { private final AtomicLong chunkCount = new AtomicLong(); private final LongAdder reusedChunkCount = new LongAdder(); - MemStoreChunkPool(int maxCount, int initialCount, float poolSizePercentage) { + MemStoreChunkPool(int poolChunkSize, int maxCount, int initialCount, float poolSizePercentage) { + this.poolChunkSize = poolChunkSize; this.maxCount = maxCount; this.poolSizePercentage = poolSizePercentage; this.reclaimedChunks = new LinkedBlockingQueue<>(); for (int i = 0; i < initialCount; i++) { - Chunk chunk = createChunkForPool(); + Chunk chunk = createChunk(true, CompactingMemStore.IndexType.ARRAY_MAP, poolChunkSize); chunk.init(); reclaimedChunks.add(chunk); } chunkCount.set(initialCount); final String n = Thread.currentThread().getName(); scheduleThreadPool = Executors.newScheduledThreadPool(1, new ThreadFactoryBuilder() - .setNameFormat(n + "-MemStoreChunkPool Statistics").setDaemon(true).build()); + .setNameFormat(n + "-MemStoreChunkPool Statistics").setDaemon(true).build()); this.scheduleThreadPool.scheduleAtFixedRate(new StatisticsThread(), statThreadPeriod, - statThreadPeriod, TimeUnit.SECONDS); + statThreadPeriod, TimeUnit.SECONDS); } /** @@ -283,6 +384,10 @@ public class ChunkCreator { * @see #putbackChunks(Chunk) */ Chunk getChunk() { + return getChunk(CompactingMemStore.IndexType.ARRAY_MAP); + } + + Chunk getChunk(CompactingMemStore.IndexType chunkIndexType) { Chunk chunk = reclaimedChunks.poll(); if (chunk != null) { chunk.reset(); @@ -293,7 +398,7 @@ public class ChunkCreator { long created = this.chunkCount.get(); if (created < this.maxCount) { if (this.chunkCount.compareAndSet(created, created + 1)) { - chunk = createChunkForPool(); + chunk = createChunk(true, chunkIndexType, poolChunkSize); break; } } else { @@ -304,6 +409,10 @@ public class ChunkCreator { return chunk; } + int getChunkSize() { + return poolChunkSize; + } + /** * Add the chunks to the pool, when the pool achieves the max size, it will skip the remaining * chunks @@ -311,7 +420,7 @@ public class ChunkCreator { */ private void putbackChunks(Chunk c) { int toAdd = this.maxCount - reclaimedChunks.size(); - if (c.isFromPool() && toAdd > 0) { + if (c.isFromPool() && c.size == poolChunkSize && toAdd > 0) { reclaimedChunks.add(c); } else { // remove the chunk (that is not going to pool) @@ -336,10 +445,11 @@ public class ChunkCreator { long created = chunkCount.get(); long reused = reusedChunkCount.sum(); long total = created + reused; - LOG.debug("Stats: current pool size=" + reclaimedChunks.size() - + ",created chunk count=" + created - + ",reused chunk count=" + reused - + ",reuseRatio=" + (total == 0 ? "0" : StringUtils.formatPercent( + LOG.debug("Stats (chunk size=" + poolChunkSize + "): " + + "current pool size=" + reclaimedChunks.size() + + ",created chunk count=" + created + + ",reused chunk count=" + reused + + ",reuseRatio=" + (total == 0 ? "0" : StringUtils.formatPercent( (float) reused / (float) total, 2))); } } @@ -356,7 +466,7 @@ public class ChunkCreator { return; } int newMaxCount = - (int) (newMemstoreSize * poolSizePercentage / getChunkSize()); + (int) (newMemstoreSize * poolSizePercentage / getChunkSize()); if (newMaxCount != this.maxCount) { // We need an adjustment in the chunks numbers if (newMaxCount > this.maxCount) { @@ -387,7 +497,7 @@ public class ChunkCreator { } private MemStoreChunkPool initializePool(long globalMemStoreSize, float poolSizePercentage, - float initialCountPercentage) { + float initialCountPercentage, int poolChunkSize) { if (poolSizePercentage <= 0) { LOG.info("PoolSizePercentage is less than 0. So not using pool"); return null; @@ -397,47 +507,63 @@ public class ChunkCreator { } if (poolSizePercentage > 1.0) { throw new IllegalArgumentException( - MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY + " must be between 0.0 and 1.0"); + MemStoreLAB.CHUNK_POOL_MAXSIZE_KEY + " must be between 0.0 and 1.0"); } - int maxCount = (int) (globalMemStoreSize * poolSizePercentage / getChunkSize()); + int maxCount = (int) (globalMemStoreSize * poolSizePercentage / poolChunkSize); if (initialCountPercentage > 1.0 || initialCountPercentage < 0) { throw new IllegalArgumentException( - MemStoreLAB.CHUNK_POOL_INITIALSIZE_KEY + " must be between 0.0 and 1.0"); + MemStoreLAB.CHUNK_POOL_INITIALSIZE_KEY + " must be between 0.0 and 1.0"); } int initialCount = (int) (initialCountPercentage * maxCount); - LOG.info("Allocating MemStoreChunkPool with chunk size=" - + StringUtils.byteDesc(getChunkSize()) + ", max count=" + maxCount - + ", initial count=" + initialCount); - return new MemStoreChunkPool(maxCount, initialCount, poolSizePercentage); + LOG.info("Allocating MemStoreChunkPool with chunk size " + + StringUtils.byteDesc(poolChunkSize) + ", max count " + maxCount + + ", initial count " + initialCount); + return new MemStoreChunkPool(poolChunkSize, maxCount, initialCount, poolSizePercentage); } @VisibleForTesting - int getMaxCount() { - if (pool != null) { - return pool.getMaxCount(); + int getMaxCount(boolean small) { + if (small && smallChunksPool != null) { + return smallChunksPool.getMaxCount(); + } else if (!small && chunksPool != null) { + return chunksPool.getMaxCount(); } return 0; } @VisibleForTesting - int getPoolSize() { - if (pool != null) { - return pool.reclaimedChunks.size(); + int getMaxCount() { + return getMaxCount(false); + } + + @VisibleForTesting + int getPoolSize(boolean small) { + if (small && smallChunksPool != null) { + return smallChunksPool.reclaimedChunks.size(); + } else if (!small && chunksPool != null) { + return chunksPool.reclaimedChunks.size(); } return 0; } @VisibleForTesting + int getPoolSize() { + return getPoolSize(false); + } + + @VisibleForTesting boolean isChunkInPool(int chunkId) { - if (pool != null) { - // chunks that are from pool will return true chunk reference not null - Chunk c = getChunk(chunkId); - if (c==null) { - return false; - } - return pool.reclaimedChunks.contains(c); + Chunk c = getChunk(chunkId); + if (c==null) { + return false; } + // chunks that are from pool will return true chunk reference not null + if (chunksPool != null && chunksPool.reclaimedChunks.contains(c)) { + return true; + } else if (smallChunksPool != null && smallChunksPool.reclaimedChunks.contains(c)) { + return true; + } return false; } @@ -446,31 +572,39 @@ public class ChunkCreator { */ @VisibleForTesting void clearChunksInPool() { - if (pool != null) { - pool.reclaimedChunks.clear(); + if (chunksPool != null) { + chunksPool.reclaimedChunks.clear(); + } + if (smallChunksPool != null) { + smallChunksPool.reclaimedChunks.clear(); } } + int getChunkSize () { + return chunksPool.getChunkSize(); + } + synchronized void putbackChunks(Set chunks) { // if there is no pool just try to clear the chunkIdMap in case there is something - if ( pool == null ) { + if (chunksPool == null && smallChunksPool == null) { this.removeChunks(chunks); return; } - // if there is pool, go over all chunk IDs that came back, the chunks may be from pool or not + // if there is a pool, go over all chunk IDs that came back, the chunks may be from pool or not for (int chunkID : chunks) { // translate chunk ID to chunk, if chunk initially wasn't in pool // this translation will (most likely) return null Chunk chunk = ChunkCreator.this.getChunk(chunkID); if (chunk != null) { - // Jumbo chunks are covered with chunkIdMap, but are not from pool, so such a chunk should - // be released here without going to pool. - // Removing them from chunkIdMap will cause their removal by the GC. - if (chunk.isJumbo()) { - this.removeChunk(chunkID); + if (chunk.isFromPool() && chunk.size == smallChunksPool.getChunkSize()) { + smallChunksPool.putbackChunks(chunk); + } else if (chunk.isFromPool() && chunk.size == chunksPool.getChunkSize()) { + chunksPool.putbackChunks(chunk); } else { - pool.putbackChunks(chunk); + // chunks which are not from one of the pools should be released without going to the pools. + // Removing them from chunkIdMap will cause their removal by the GC. + this.removeChunk(chunkID); } } // if chunk is null, it was never covered by the chunkIdMap (and so wasn't in pool also), @@ -480,3 +614,4 @@ public class ChunkCreator { } } + diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java index 93d2685e37..5d913be5ec 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java @@ -57,9 +57,9 @@ public class ImmutableMemStoreLAB implements MemStoreLAB { ** The interface is only for external callers */ @Override - public Chunk getNewExternalChunk() { + public Chunk getNewExternalChunk(boolean small) { MemStoreLAB mslab = this.mslabs.get(0); - return mslab.getNewExternalChunk(); + return mslab.getNewExternalChunk(small); } /* Creating chunk to be used as data chunk in CellChunkMap. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java index 6bc8886296..1b17acca59 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java @@ -52,6 +52,8 @@ public interface MemStoreLAB { String CHUNK_SIZE_KEY = "hbase.hregion.memstore.mslab.chunksize"; int CHUNK_SIZE_DEFAULT = 2048 * 1024; + String SMALL_CHUNK_SIZE_KEY = "hbase.hregion.memstore.mslab.chunksize.small"; + float SMALL_CHUNK_SIZE_DEFAULT = 0.1f; String MAX_ALLOC_KEY = "hbase.hregion.memstore.mslab.max.allocation"; int MAX_ALLOC_DEFAULT = 256 * 1024; // allocs bigger than this don't go through // allocator @@ -100,7 +102,7 @@ public interface MemStoreLAB { ** The space on this chunk will be allocated externally. ** The interface is only for external callers */ - Chunk getNewExternalChunk(); + Chunk getNewExternalChunk(boolean small); /* Creating chunk to be used as data chunk in CellChunkMap. ** This chunk is bigger than normal constant chunk size, and thus called JumboChunk it is used for diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java index f7728acb2f..1ee8c7ef38 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java @@ -67,14 +67,16 @@ public class MemStoreLABImpl implements MemStoreLAB { static final Logger LOG = LoggerFactory.getLogger(MemStoreLABImpl.class); - private AtomicReference curChunk = new AtomicReference<>(); + private AtomicReference currChunk = new AtomicReference<>(); + private AtomicReference currSmallChunk = new AtomicReference<>(); // Lock to manage multiple handlers requesting for a chunk private ReentrantLock lock = new ReentrantLock(); // A set of chunks contained by this memstore LAB @VisibleForTesting Set chunks = new ConcurrentSkipListSet(); - private final int chunkSize; + private final int regularChunkSize; + private final int smallChunkSize; private final int maxAlloc; private final ChunkCreator chunkCreator; private final CompactingMemStore.IndexType idxType; // what index is used for corresponding segment @@ -94,11 +96,12 @@ public class MemStoreLABImpl implements MemStoreLAB { } public MemStoreLABImpl(Configuration conf) { - chunkSize = conf.getInt(CHUNK_SIZE_KEY, CHUNK_SIZE_DEFAULT); + regularChunkSize = conf.getInt(CHUNK_SIZE_KEY, CHUNK_SIZE_DEFAULT); + smallChunkSize = (int) conf.getFloat(SMALL_CHUNK_SIZE_KEY, SMALL_CHUNK_SIZE_DEFAULT) * regularChunkSize; maxAlloc = conf.getInt(MAX_ALLOC_KEY, MAX_ALLOC_DEFAULT); this.chunkCreator = ChunkCreator.getInstance(); // if we don't exclude allocations >CHUNK_SIZE, we'd infiniteloop on one! - Preconditions.checkArgument(maxAlloc <= chunkSize, + Preconditions.checkArgument(maxAlloc <= regularChunkSize, MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY); // if user requested to work with MSLABs (whether on- or off-heap), then the @@ -122,9 +125,9 @@ public class MemStoreLABImpl implements MemStoreLAB { public Cell forceCopyOfBigCellInto(Cell cell) { int size = KeyValueUtil.length(cell); Preconditions.checkArgument(size >= 0, "negative size"); - if (size <= chunkSize) { + if (size <= regularChunkSize) { //TODO: add a smallChunkSize possibility // Using copyCellInto for cells which are bigger than the original maxAlloc - Cell newCell = copyCellInto(cell, chunkSize); + Cell newCell = copyCellInto(cell, regularChunkSize); return newCell; } else { Chunk c = getNewExternalJumboChunk(size); @@ -145,7 +148,7 @@ public class MemStoreLABImpl implements MemStoreLAB { int allocOffset = 0; while (true) { // Try to get the chunk - c = getOrMakeChunk(); + c = getOrMakeChunk(false); //TODO: add a smallChunkSize possibility // we may get null because the some other thread succeeded in getting the lock // and so the current thread has to try again to make its chunk or grab the chunk // that the other thread created @@ -240,7 +243,8 @@ public class MemStoreLABImpl implements MemStoreLAB { * @return true if we won the race to retire the chunk */ private void tryRetireChunk(Chunk c) { - curChunk.compareAndSet(c, null); + currChunk.compareAndSet(c, null); + currSmallChunk.compareAndSet(c, null); // If the CAS succeeds, that means that we won the race // to retire the chunk. We could use this opportunity to // update metrics on external fragmentation. @@ -253,9 +257,17 @@ public class MemStoreLABImpl implements MemStoreLAB { * Get the current chunk, or, if there is no current chunk, * allocate a new one from the JVM. */ - private Chunk getOrMakeChunk() { + private Chunk getOrMakeChunk(boolean small) { // Try to get the chunk - Chunk c = curChunk.get(); + Chunk c; + int chunkSize; + if (small) { + c = currSmallChunk.get(); + chunkSize = smallChunkSize; + } else { + c = currChunk.get(); + chunkSize = regularChunkSize; + } if (c != null) { return c; } @@ -265,14 +277,22 @@ public class MemStoreLABImpl implements MemStoreLAB { if (lock.tryLock()) { try { // once again check inside the lock - c = curChunk.get(); + if (small) { + c = currSmallChunk.get(); + } else { + c = currChunk.get(); + } if (c != null) { return c; } - c = this.chunkCreator.getChunk(idxType); + c = this.chunkCreator.getChunk(idxType, small); if (c != null) { // set the curChunk. No need of CAS as only one thread will be here - curChunk.set(c); + if (small) { + currSmallChunk.set(c); + } else { + currChunk.set(c); + } chunks.add(c.getId()); return c; } @@ -290,9 +310,9 @@ public class MemStoreLABImpl implements MemStoreLAB { ** The interface is only for external callers */ @Override - public Chunk getNewExternalChunk() { + public Chunk getNewExternalChunk(boolean small) { // the new chunk is going to be part of the chunk array and will always be referenced - Chunk c = this.chunkCreator.getChunk(); + Chunk c = this.chunkCreator.getChunk(small); chunks.add(c.getId()); return c; } @@ -318,8 +338,17 @@ public class MemStoreLABImpl implements MemStoreLAB { } @VisibleForTesting + Chunk getCurrentChunk(boolean small) { + if (small) { + return currSmallChunk.get(); + } else { + return currChunk.get(); + } + } + + @VisibleForTesting Chunk getCurrentChunk() { - return this.curChunk.get(); + return getCurrentChunk(false); } @VisibleForTesting -- 2.12.2