From 5789865020ad06a833fcbc1184b418e2109c5daa Mon Sep 17 00:00:00 2001 From: anastas Date: Thu, 28 Dec 2017 16:01:41 +0200 Subject: [PATCH] HBASE-19282: Making CellChunkMap the default index (CellSet delegatee) for ImmutableSegments, when MSLAB is used. In order to avoid additional user settings. If no MSLAB is requested the index is going to be CellArrayMap --- .../hbase/regionserver/CompactingMemStore.java | 48 ++++++++++++++-------- .../hadoop/hbase/regionserver/MemStoreLABImpl.java | 7 ++-- .../hbase/regionserver/TestCompactingMemStore.java | 1 + .../TestCompactingToCellFlatMapMemStore.java | 35 +++++++--------- .../TestWalAndCompactingMemStoreFlush.java | 9 +++- 5 files changed, 56 insertions(+), 44 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index bb03117..3182cfd 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -58,16 +58,13 @@ public class CompactingMemStore extends AbstractMemStore { "hbase.hregion.compacting.memstore.type"; public static final String COMPACTING_MEMSTORE_TYPE_DEFAULT = String.valueOf(MemoryCompactionPolicy.BASIC); - // The external setting of the compacting MemStore behaviour - public static final String COMPACTING_MEMSTORE_INDEX_KEY = - "hbase.hregion.compacting.memstore.index"; - // usage of CellArrayMap is default, later it will be decided how to use CellChunkMap - public static final String COMPACTING_MEMSTORE_INDEX_DEFAULT = - String.valueOf(IndexType.ARRAY_MAP); // Default fraction of in-memory-flush size w.r.t. flush-to-disk size public static final String IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY = "hbase.memstore.inmemoryflush.threshold.factor"; - private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT = 0.02; + private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_CAM_DEFAULT = 0.02; + // working CellChunkMap index requires less frequent in-memory-flushes + // so we have more cells to fill into CCM's index chunk + private static final double IN_MEMORY_FLUSH_THRESHOLD_FACTOR_CCM_DEFAULT = 0.1; private static final Logger LOG = LoggerFactory.getLogger(CompactingMemStore.class); private HStore store; @@ -114,9 +111,16 @@ public class CompactingMemStore extends AbstractMemStore { this.regionServices = regionServices; this.pipeline = new CompactionPipeline(getRegionServices()); this.compactor = createMemStoreCompactor(compactionPolicy); + if (conf.getBoolean(MemStoreLAB.USEMSLAB_KEY, MemStoreLAB.USEMSLAB_DEFAULT)) { + // if user requested to work with MSLABs (whether on- or off-heap), then the + // immutable segments are going to use CellChunkMap as their index + indexType = IndexType.CHUNK_MAP; + } else { + indexType = IndexType.ARRAY_MAP; + } + // initialization of the flush size should happen after initialization of the index type + // so do not transfer the following method initInmemoryFlushSize(conf); - indexType = IndexType.valueOf(conf.get(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - CompactingMemStore.COMPACTING_MEMSTORE_INDEX_DEFAULT)); } @VisibleForTesting @@ -126,6 +130,7 @@ public class CompactingMemStore extends AbstractMemStore { } private void initInmemoryFlushSize(Configuration conf) { + double factor = 0; long memstoreFlushSize = getRegionServices().getMemStoreFlushSize(); int numStores = getRegionServices().getNumStores(); if (numStores <= 1) { @@ -133,11 +138,17 @@ public class CompactingMemStore extends AbstractMemStore { numStores = 1; } inmemoryFlushSize = memstoreFlushSize / numStores; - // multiply by a factor - double factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, - IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT); + // multiply by a factor (different factors for different index types) + if (indexType == IndexType.ARRAY_MAP) { + factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, + IN_MEMORY_FLUSH_THRESHOLD_FACTOR_CAM_DEFAULT); + } else { + factor = conf.getDouble(IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, + IN_MEMORY_FLUSH_THRESHOLD_FACTOR_CCM_DEFAULT); + } inmemoryFlushSize *= factor; - LOG.info("Setting in-memory flush size threshold to " + inmemoryFlushSize); + LOG.info("Setting in-memory flush size threshold to " + inmemoryFlushSize + + " and immutable segments index to be of type " + indexType); } /** @@ -318,10 +329,11 @@ public class CompactingMemStore extends AbstractMemStore { // setter is used only for testability @VisibleForTesting - public void setIndexType() { - indexType = IndexType.valueOf(getConfiguration().get( - CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - CompactingMemStore.COMPACTING_MEMSTORE_INDEX_DEFAULT)); + void setIndexType(IndexType type) { + indexType = type; + // Because this functionality is for testing only and tests are setting in-memory flush size + // according to their need, there is no setting of in-memory flush size, here. + // If it is needed, please change in-memory flush size explicitly } public IndexType getIndexType() { @@ -572,7 +584,7 @@ public class CompactingMemStore extends AbstractMemStore { // debug method public void debug() { String msg = "active size=" + this.active.keySize(); - msg += " threshold="+IN_MEMORY_FLUSH_THRESHOLD_FACTOR_DEFAULT* inmemoryFlushSize; + msg += " in-memory flush size is "+ inmemoryFlushSize; msg += " allow compaction is "+ (allowCompaction.get() ? "true" : "false"); msg += " inMemoryFlushInProgress is "+ (inMemoryFlushInProgress.get() ? "true" : "false"); LOG.debug(msg); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java index 41a6cf8..33e3974 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java @@ -100,9 +100,10 @@ public class MemStoreLABImpl implements MemStoreLAB { // if we don't exclude allocations >CHUNK_SIZE, we'd infiniteloop on one! Preconditions.checkArgument(maxAlloc <= chunkSize, MAX_ALLOC_KEY + " must be less than " + CHUNK_SIZE_KEY); - idxType = CompactingMemStore.IndexType.valueOf(conf.get( - CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - CompactingMemStore.COMPACTING_MEMSTORE_INDEX_DEFAULT)); + + // if user requested to work with MSLABs (whether on- or off-heap), then the + // immutable segments are going to use CellChunkMap as their index + idxType = CompactingMemStore.IndexType.CHUNK_MAP; } @Override diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java index 6fbf99a..220c42d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java @@ -89,6 +89,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { compactingSetUp(); this.memstore = new MyCompactingMemStore(HBaseConfiguration.create(), CellComparator.getInstance(), store, regionServicesForStores, MemoryCompactionPolicy.EAGER); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP); } protected void compactingSetUp() throws Exception { diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java index 673acdd..e0107ce 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellFlatMapMemStore.java @@ -94,9 +94,9 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore String[] keys1 = { "A", "A", "B", "C" }; //A1, A2, B3, C4 if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); + } else { + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP); } // test 1 bucket @@ -140,9 +140,9 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore public void testCompaction2Buckets() throws IOException { if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); + } else { + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP); } String[] keys1 = { "A", "A", "B", "C" }; String[] keys2 = { "A", "B", "D" }; @@ -202,9 +202,10 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore public void testCompaction3Buckets() throws IOException { if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); + } else { + // set to CellArrayMap as CCM is configured by default due to MSLAB usage + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.ARRAY_MAP); } String[] keys1 = { "A", "A", "B", "C" }; String[] keys2 = { "A", "B", "D" }; @@ -290,9 +291,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore public void testMerging() throws IOException { if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); } String[] keys1 = { "A", "A", "B", "C", "F", "H"}; String[] keys2 = { "A", "B", "D", "G", "I", "J"}; @@ -362,9 +361,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore public void testTimeRangeAfterCompaction() throws IOException { if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); } testTimeRange(true); } @@ -373,9 +370,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore public void testTimeRangeAfterMerge() throws IOException { if (toCellChunkMap) { // set memstore to flat into CellChunkMap - conf.set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); } MemoryCompactionPolicy compactionType = MemoryCompactionPolicy.BASIC; memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, @@ -620,9 +615,7 @@ public class TestCompactingToCellFlatMapMemStore extends TestCompactingMemStore memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, String.valueOf(compactionType)); ((MyCompactingMemStore)memstore).initiateType(compactionType, memstore.getConfiguration()); - memstore.getConfiguration().set(CompactingMemStore.COMPACTING_MEMSTORE_INDEX_KEY, - String.valueOf(CompactingMemStore.IndexType.CHUNK_MAP)); - ((CompactingMemStore)memstore).setIndexType(); + ((CompactingMemStore)memstore).setIndexType(CompactingMemStore.IndexType.CHUNK_MAP); int numOfCells = 8; String[] keys1 = { "A", "A", "B", "C", "D", "D", "E", "F" }; //A1, A2, B3, C4, D5, D6, E7, F8 diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java index 6a64796..f3bd7ee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java @@ -768,8 +768,7 @@ public class TestWalAndCompactingMemStoreFlush { conf.setLong(HConstants.HREGION_MEMSTORE_FLUSH_SIZE, 300 * 1024); conf.set(FlushPolicyFactory.HBASE_FLUSH_POLICY_KEY, FlushNonSloppyStoresFirstPolicy.class.getName()); - conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN, - 75 * 1024); + conf.setLong(FlushLargeStoresPolicy.HREGION_COLUMNFAMILY_FLUSH_SIZE_LOWER_BOUND_MIN, 75 * 1024); conf.setDouble(CompactingMemStore.IN_MEMORY_FLUSH_THRESHOLD_FACTOR_KEY, 0.8); // set memstore to do index compaction with merge conf.set(CompactingMemStore.COMPACTING_MEMSTORE_TYPE_KEY, @@ -797,6 +796,12 @@ public class TestWalAndCompactingMemStoreFlush { long totalMemstoreSize = region.getMemStoreSize(); + // test in-memory flashing into CAM here + ((CompactingMemStore) ((HStore)region.getStore(FAMILY1)).memstore).setIndexType( + CompactingMemStore.IndexType.ARRAY_MAP); + ((CompactingMemStore) ((HStore)region.getStore(FAMILY3)).memstore).setIndexType( + CompactingMemStore.IndexType.ARRAY_MAP); + // Find the sizes of the memstores of each CF. MemStoreSize cf1MemstoreSizePhaseI = region.getStore(FAMILY1).getMemStoreSize(); MemStoreSize cf2MemstoreSizePhaseI = region.getStore(FAMILY2).getMemStoreSize(); -- 1.8.5.2 (Apple Git-48)