From f8e8c2c25ac9be064e5fab1d72f3c8bda9558ddd Mon Sep 17 00:00:00 2001 From: gsheffi Date: Sun, 10 Dec 2017 14:15:05 +0200 Subject: [PATCH] for creating patch HBASE-19133.02.patch --- .../hbase/regionserver/AbstractMemStore.java | 17 ++- .../regionserver/CellArrayImmutableSegment.java | 2 +- .../regionserver/CellChunkImmutableSegment.java | 25 +++- .../hadoop/hbase/regionserver/ChunkCreator.java | 3 +- .../regionserver/CompositeImmutableSegment.java | 2 +- .../hbase/regionserver/ImmutableMemStoreLAB.java | 5 + .../hadoop/hbase/regionserver/MemStoreLAB.java | 13 +- .../hadoop/hbase/regionserver/MemStoreLABImpl.java | 25 +++- .../apache/hadoop/hbase/regionserver/Segment.java | 12 +- .../hbase/regionserver/TestCompactingMemStore.java | 21 +++ .../TestCompactingToCellFlatMapMemStore.java | 150 ++++++++++++++++++++- 11 files changed, 259 insertions(+), 16 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java index ee480be1e2..a902bdad85 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java @@ -104,7 +104,7 @@ public abstract class AbstractMemStore implements MemStore { @Override public void add(Cell cell, MemStoreSizing memstoreSizing) { - Cell toAdd = maybeCloneWithAllocator(cell); + Cell toAdd = maybeCloneWithAllocator(cell, false); boolean mslabUsed = (toAdd != cell); // This cell data is backed by the same byte[] where we read request in RPC(See HBASE-15180). By // default MSLAB is ON and we might have copied cell to MSLAB area. If not we must do below deep @@ -268,8 +268,19 @@ public abstract class AbstractMemStore implements MemStore { return result; } - private Cell maybeCloneWithAllocator(Cell cell) { - return active.maybeCloneWithAllocator(cell); + /** + * If the segment has a memory allocator the cell is being cloned to this space, and returned; + * Otherwise the given cell is returned + * + * When a cell's size is too big (bigger than maxAlloc), it is not allocated on MSLAB. + * Since the process of flattening to CellChunkMap assumes that all cells are allocated on MSLAB, during this process, the input parameter forceCloneOfBigCell is set to 'true' and the cell is copied into MSLAB. + * + * @param cell the cell to clone + * @param forceCloneOfBigCell true only during the process of flattening to CellChunkMap. + * @return either the given cell or its clone + */ + private Cell maybeCloneWithAllocator(Cell cell, boolean forceCloneOfBigCell) { + return active.maybeCloneWithAllocator(cell, forceCloneOfBigCell); } /* diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java index 0e80b1d0ed..aeded7a0e3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellArrayImmutableSegment.java @@ -101,7 +101,7 @@ public class CellArrayImmutableSegment extends ImmutableSegment { cells[i] = c; } else { // now we just copy it to the new segment (also MSLAB copy) - cells[i] = maybeCloneWithAllocator(c); + cells[i] = maybeCloneWithAllocator(c, false); } // second parameter true, because in compaction/merge the addition of the cell to new segment // is always successful diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java index 7db00a0c48..8664a64e7d 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CellChunkImmutableSegment.java @@ -111,13 +111,17 @@ public class CellChunkImmutableSegment extends ImmutableSegment { while (iterator.hasNext()) { // the iterator hides the elimination logic for compaction Cell c = iterator.next(); numOfCellsAfterCompaction++; - assert (c instanceof ByteBufferKeyValue); // shouldn't get here anything but ByteBufferKeyValue + if (c instanceof ByteBufferKeyValue == false) { + // CellChunkMap assumes all cells are allocated on MSLAB. + // Therefore, cells which are not allocated on MSLAB initially, are copied into MSLAB here. + c = copyCellIntoMSLAB(c); + } if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkSize) { currentChunkIdx++; // continue to the next index chunk offsetInCurentChunk = ChunkCreator.SIZEOF_CHUNK_HEADER; } if (action == MemStoreCompactionStrategy.Action.COMPACT) { - c = maybeCloneWithAllocator(c); // for compaction copy cell to the new segment (MSLAB copy) + c = maybeCloneWithAllocator(c, false); // for compaction copy cell to the new segment (MSLAB copy) } offsetInCurentChunk = // add the Cell reference to the index chunk createCellReference((ByteBufferKeyValue)c, chunks[currentChunkIdx].getData(), @@ -153,7 +157,6 @@ public class CellChunkImmutableSegment extends ImmutableSegment { // Create CellSet based on CellChunkMap from current ConcurrentSkipListMap based CellSet // (without compacting iterator) // This is a service for not-flat immutable segments - // Assumption: cells do not exceed chunk size! private void reinitializeCellSet( int numOfCells, KeyValueScanner segmentScanner, CellSet oldCellSet, MemStoreCompactionStrategy.Action action) { @@ -175,7 +178,11 @@ public class CellChunkImmutableSegment extends ImmutableSegment { Cell prev = null; try { while ((curCell = segmentScanner.next()) != null) { - assert (curCell instanceof ByteBufferKeyValue); // shouldn't get here anything but ByteBufferKeyValue + if (curCell instanceof ByteBufferKeyValue == false) { + // CellChunkMap assumes all cells are allocated on MSLAB. + // Therefore, cells which are not allocated on MSLAB initially, are copied into MSLAB here. + curCell = copyCellIntoMSLAB(curCell); + } if (offsetInCurentChunk + ClassSize.CELL_CHUNK_MAP_ENTRY > chunkSize) { // continue to the next metadata chunk currentChunkIdx++; @@ -231,4 +238,14 @@ public class CellChunkImmutableSegment extends ImmutableSegment { } return numberOfChunks; } + + private Cell copyCellIntoMSLAB(Cell cell) { + long oldHeapSize = heapSizeChange(cell, true); + cell = maybeCloneWithAllocator(cell, true); + long newHeapSize = heapSizeChange(cell, true); + long heapOverhead = newHeapSize - oldHeapSize; + // The dataSize and heapSize are updated because the cell object's size and the size of its serialization on heap are different. + incSize(0, heapOverhead); //TODO: maybe need to update the dataSize of the region + return cell; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java index faf517bac5..beae1e81bf 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ChunkCreator.java @@ -166,7 +166,8 @@ public class ChunkCreator { + chunkSize + ". Converting to regular chunk."); getChunk(chunkIndexType,chunkSize); } - return getChunk(chunkIndexType, jumboSize); + // the size of the allocation includes both the size requested and a place for the Chunk's header + return getChunk(chunkIndexType, jumboSize + SIZEOF_CHUNK_HEADER); } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java index 93658193c1..df5c4f6f6b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java @@ -112,7 +112,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { * @return either the given cell or its clone */ @Override - public Cell maybeCloneWithAllocator(Cell cell) { + public Cell maybeCloneWithAllocator(Cell cell, boolean forceCloneOfBigCell) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java index b2c91300f1..93d2685e37 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java @@ -45,6 +45,11 @@ public class ImmutableMemStoreLAB implements MemStoreLAB { throw new IllegalStateException("This is an Immutable MemStoreLAB."); } + @Override + public Cell forceCopyOfBigCellInto(Cell cell) { + throw new IllegalStateException("This is an Immutable MemStoreLAB."); + } + /* Creating chunk to be used as index chunk in CellChunkMap, part of the chunks array. ** Returning a new chunk, without replacing current chunk, ** meaning MSLABImpl does not make the returned chunk as CurChunk. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java index 1982b4fb4e..6bc8886296 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java @@ -69,6 +69,17 @@ public interface MemStoreLAB { Cell copyCellInto(Cell cell); /** + * Allocates slice in this LAB and copy the passed Cell into this area. Returns new Cell instance + * over the copied the data. When this MemStoreLAB can not copy this Cell, it returns null. + * + * Since the process of flattening to CellChunkMap assumes all cells are allocated on MSLAB, + * and since copyCellInto does not copy big cells (for whom size > maxAlloc) into MSLAB, + * this method is called while the process of flattening to CellChunkMap is running, + * for forcing the allocation of big cells on this MSLAB. + */ + Cell forceCopyOfBigCellInto(Cell cell); + + /** * Close instance since it won't be used any more, try to put the chunks back to pool */ void close(); @@ -92,7 +103,7 @@ public interface MemStoreLAB { Chunk getNewExternalChunk(); /* Creating chunk to be used as data chunk in CellChunkMap. - ** This chunk is bigger the normal constant chunk size, and thus called JumboChunk it is used for + ** This chunk is bigger than normal constant chunk size, and thus called JumboChunk it is used for ** jumbo cells (which size is bigger than normal chunks). ** Jumbo Chunks are needed only for CCM and thus are created only in ** CompactingMemStore.IndexType.CHUNK_MAP type. diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java index 08588d26e8..ce3ae9d864 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java @@ -76,7 +76,7 @@ public class MemStoreLABImpl implements MemStoreLAB { @VisibleForTesting Set chunks = new ConcurrentSkipListSet(); private final int chunkSize; - private final int maxAlloc; + private int maxAlloc; private final ChunkCreator chunkCreator; private final CompactingMemStore.IndexType idxType; // what index is used for corresponding segment @@ -140,6 +140,29 @@ public class MemStoreLABImpl implements MemStoreLAB { } /** + * When a cell's size is too big (bigger than maxAlloc), copyCellInto does not allocate it on MSLAB. + * Since the process of flattening to CellChunkMap assumes that all cells are allocated on MSLAB, during this process, + * the big cells are copied into MSLAB using this method. + */ + @Override + public Cell forceCopyOfBigCellInto(Cell cell) { + int size = KeyValueUtil.length(cell); + Preconditions.checkArgument(size >= 0, "negative size"); + if (size <= chunkSize) { + // Changing maxAlloc temporarily in order to use copyCellInto for cells which are bigger than the original maxAlloc + int tmp = maxAlloc; + maxAlloc = chunkSize; + Cell newCell = copyCellInto(cell); + maxAlloc = tmp; + return newCell; + } else { + Chunk c = getNewExternalJumboChunk(size); + int allocOffset = c.alloc(size); + return copyToChunkCell(cell, c.getData(), allocOffset, size); + } + } + + /** * Clone the passed cell by copying its data into the passed buf and create a cell with a chunkid * out of it */ diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java index c054666395..46e9d3ef33 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java @@ -158,14 +158,22 @@ public abstract class Segment { /** * If the segment has a memory allocator the cell is being cloned to this space, and returned; * otherwise the given cell is returned + * + * When a cell's size is too big (bigger than maxAlloc), it is not allocated on MSLAB. + * Since the process of flattening to CellChunkMap assumes that all cells are allocated on MSLAB, during this process, the input parameter forceCloneOfBigCell is set to 'true' and the cell is copied into MSLAB. + * * @return either the given cell or its clone */ - public Cell maybeCloneWithAllocator(Cell cell) { + public Cell maybeCloneWithAllocator(Cell cell, boolean forceCloneOfBigCell) { if (this.memStoreLAB == null) { return cell; } - Cell cellFromMslab = this.memStoreLAB.copyCellInto(cell); + Cell cellFromMslab = null; + if (forceCloneOfBigCell == false) + cellFromMslab = this.memStoreLAB.copyCellInto(cell); + else + cellFromMslab = this.memStoreLAB.forceCopyOfBigCellInto(cell); return (cellFromMslab != null) ? cellFromMslab : cell; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java index 0f18deeab4..444da83f8f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java @@ -836,6 +836,27 @@ public class TestCompactingMemStore extends TestDefaultMemStore { return totalLen; } + // for controlling the val size when adding a new cell + protected int addRowsByKeys(final AbstractMemStore hmc, String[] keys, byte[] val) { + byte[] fam = Bytes.toBytes("testfamily"); + byte[] qf = Bytes.toBytes("testqualifier"); + long size = hmc.getActive().keySize(); + long heapOverhead = hmc.getActive().heapSize(); + int totalLen = 0; + for (int i = 0; i < keys.length; i++) { + long timestamp = System.currentTimeMillis(); + Threads.sleep(1); // to make sure each kv gets a different ts + byte[] row = Bytes.toBytes(keys[i]); + KeyValue kv = new KeyValue(row, fam, qf, timestamp, val); + totalLen += kv.getLength(); + hmc.add(kv, null); + LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); + } + regionServicesForStores.addMemStoreSize(new MemStoreSize(hmc.getActive().keySize() - size, + hmc.getActive().heapSize() - heapOverhead)); + return totalLen; + } + private class EnvironmentEdgeForMemstoreTest implements EnvironmentEdge { long t = 1234; diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java index b4cf4ec342..d5dfa6aa7f 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java @@ -641,8 +641,6 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and flatten assertEquals(0, memstore.getSnapshot().getCellsCount()); - // One cell is duplicated, but it shouldn't be compacted because we are in BASIC mode. - // totalCellsLen should remain the same long oneCellOnCCMHeapSize = ClassSize.CELL_CHUNK_MAP_ENTRY + ClassSize.align(KeyValueUtil.length(kv)); totalHeapSize = MutableSegment.DEEP_OVERHEAD + CellChunkImmutableSegment.DEEP_OVERHEAD_CCM @@ -661,6 +659,154 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore memstore.clearSnapshot(snapshot.getId()); } + /** + * CellChunkMap Segment index requires all cell data to be written in the MSLAB Chunks. + * Even though MSLAB is enabled, cells bigger than maxAlloc (even if smaller than the size of a chunk) are not written in the MSLAB Chunks. + * If such cells are found in the process of flattening into CellChunkMap (in-memory-flush) they need to be copied into MSLAB. + * testFlatteningToBigCellChunkMap checks that the process of flattening into CellChunkMap succeeds, even when such big cells are allocated. + */ + @Test + public void testFlatteningToBigCellChunkMap() throws IOException { + + if (toCellChunkMap == false) + return; + // set memstore to flat into CellChunkMap + MemoryCompactionPolicy compactionType = MemoryCompactionPolicy.BASIC; + memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, + String.valueOf(compactionType)); + ((MyCompactingMemStore)memstore).initiateType(compactionType, memstore.getConfiguration()); + memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, + String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); + ((CompactingMemStore)memstore).setIndexType(); + int numOfCells = 4; + char[] chars = new char[MemStoreLAB.MAX_ALLOC_DEFAULT]; + for (int i = 0; i < chars.length; i++) + chars[i] = 'A'; + String bigVal = new String(chars); + String[] keys1 = { "A", "B", "C", "D"}; + + // make one cell + byte[] row = Bytes.toBytes(keys1[0]); + byte[] val = Bytes.toBytes(bigVal); + KeyValue kv = + new KeyValue(row, Bytes.toBytes("testfamily"), Bytes.toBytes("testqualifier"), + System.currentTimeMillis(), val); + + // test 1 bucket + int totalCellsLen = addRowsByKeys(memstore, keys1, val); + + long oneCellOnCSLMHeapSize = + ClassSize.align( + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + kv.heapSize()); + + long totalHeapSize = numOfCells * oneCellOnCSLMHeapSize + MutableSegment.DEEP_OVERHEAD; + assertEquals(totalCellsLen, regionServicesForStores.getMemStoreSize()); + assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); + + ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and flatten + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + // One cell is duplicated, but it shouldn't be compacted because we are in BASIC mode. + // totalCellsLen should remain the same + long oneCellOnCCMHeapSize = + ClassSize.CELL_CHUNK_MAP_ENTRY + ClassSize.align(KeyValueUtil.length(kv)); + totalHeapSize = MutableSegment.DEEP_OVERHEAD + CellChunkImmutableSegment.DEEP_OVERHEAD_CCM + + numOfCells * oneCellOnCCMHeapSize; + + assertEquals(totalCellsLen, regionServicesForStores.getMemStoreSize()); + assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); + + MemStoreSize size = memstore.getFlushableSize(); + MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot + region.decrMemStoreSize(size); // simulate flusher + ImmutableSegment s = memstore.getSnapshot(); + assertEquals(numOfCells, s.getCellsCount()); + assertEquals(0, regionServicesForStores.getMemStoreSize()); + + memstore.clearSnapshot(snapshot.getId()); + } + + /** + * CellChunkMap Segment index requires all cell data to be written in the MSLAB Chunks. + * Even though MSLAB is enabled, cells bigger than the size of a chunk are not written in the MSLAB Chunks. + * If such cells are found in the process of flattening into CellChunkMap (in-memory-flush) they need to be copied into MSLAB. + * testFlatteningToJumboCellChunkMap checks that the process of flattening into CellChunkMap succeeds, even when such big cells are allocated. + */ + @Test + public void testFlatteningToJumboCellChunkMap() throws IOException { + + if (toCellChunkMap == false) + return; + // set memstore to flat into CellChunkMap + MemoryCompactionPolicy compactionType = MemoryCompactionPolicy.BASIC; + memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, + String.valueOf(compactionType)); + ((MyCompactingMemStore)memstore).initiateType(compactionType, memstore.getConfiguration()); + memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, + String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); + ((CompactingMemStore)memstore).setIndexType(); + int numOfCells = 1; + char[] chars = new char[MemStoreLAB.CHUNK_SIZE_DEFAULT]; + for (int i = 0; i < chars.length; i++) + chars[i] = 'A'; + String bigVal = new String(chars); + String[] keys1 = { "A"}; + + // make one cell + byte[] row = Bytes.toBytes(keys1[0]); + byte[] val = Bytes.toBytes(bigVal); + KeyValue kv = + new KeyValue(row, Bytes.toBytes("testfamily"), Bytes.toBytes("testqualifier"), + System.currentTimeMillis(), val); + + // test 1 bucket + int totalCellsLen = addRowsByKeys(memstore, keys1, val); + + long oneCellOnCSLMHeapSize = + ClassSize.align( + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY + kv.heapSize()); + + long totalHeapSize = numOfCells * oneCellOnCSLMHeapSize + MutableSegment.DEEP_OVERHEAD; + assertEquals(totalCellsLen, regionServicesForStores.getMemStoreSize()); + assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); + + ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and flatten + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + assertEquals(0, memstore.getSnapshot().getCellsCount()); + + // One cell is duplicated, but it shouldn't be compacted because we are in BASIC mode. + // totalCellsLen should remain the same + long oneCellOnCCMHeapSize = + ClassSize.CELL_CHUNK_MAP_ENTRY + ClassSize.align(KeyValueUtil.length(kv)); + totalHeapSize = MutableSegment.DEEP_OVERHEAD + CellChunkImmutableSegment.DEEP_OVERHEAD_CCM + + numOfCells * oneCellOnCCMHeapSize; + + assertEquals(totalCellsLen, regionServicesForStores.getMemStoreSize()); + assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); + + MemStoreSize size = memstore.getFlushableSize(); + MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot + region.decrMemStoreSize(size); // simulate flusher + ImmutableSegment s = memstore.getSnapshot(); + assertEquals(numOfCells, s.getCellsCount()); + assertEquals(0, regionServicesForStores.getMemStoreSize()); + + memstore.clearSnapshot(snapshot.getId()); + + String[] keys2 = { "C", "D", "E"}; + addRowsByKeys(memstore, keys2, val); + while (((CompactingMemStore) memstore).isMemStoreFlushingInMemory()) { + Threads.sleep(10); + } + totalHeapSize = 1 * oneCellOnCSLMHeapSize + MutableSegment.DEEP_OVERHEAD + CellChunkImmutableSegment.DEEP_OVERHEAD_CCM + + 2 * oneCellOnCCMHeapSize; + assertEquals(totalHeapSize, ((CompactingMemStore) memstore).heapSize()); + } + private long addRowsByKeysDataSize(final AbstractMemStore hmc, String[] keys) { byte[] fam = Bytes.toBytes("testfamily"); -- 2.12.2