From c67f06d837183fb26635d77ae59f62c1566984ed Mon Sep 17 00:00:00 2001 From: Andrew Purtell Date: Tue, 22 Jan 2019 17:27:20 -0800 Subject: [PATCH] HBASE-21748 Port HBASE-21738 (Remove all the CLSM#size operation in our memstore because it's an quite time consuming) to branch-1 --- .../hbase/regionserver/DefaultMemStore.java | 19 +++++++++++++++---- .../apache/hadoop/hbase/io/TestHeapSize.java | 4 +++- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java index 299c9989b2..aae166fa01 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java @@ -27,6 +27,7 @@ import java.util.Iterator; import java.util.List; import java.util.NavigableSet; import java.util.SortedSet; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import org.apache.commons.logging.Log; @@ -132,7 +133,7 @@ public class DefaultMemStore implements MemStore { } } MemStoreSnapshot memStoreSnapshot = new MemStoreSnapshot(this.snapshotId, - snapshotSection.getCellSkipListSet().size(), snapshotSection.getHeapSize().get(), + snapshotSection.getCellCount().get(), snapshotSection.getHeapSize().get(), snapshotSection.getTimeRangeTracker(), new CollectionBackedScanner(snapshotSection.getCellSkipListSet(), this.comparator), this.tagsPresent); @@ -244,6 +245,7 @@ public class DefaultMemStore implements MemStore { } activeSection.getTimeRangeTracker().includeTimestamp(toAdd); activeSection.getHeapSize().addAndGet(s); + activeSection.getCellCount().incrementAndGet(); return s; } @@ -294,6 +296,7 @@ public class DefaultMemStore implements MemStore { snapshotSection.getCellSkipListSet().remove(cell); long sz = heapSizeChange(cell, true); snapshotSection.getHeapSize().addAndGet(-sz); + snapshotSection.getCellCount().decrementAndGet(); } // If the key is in the memstore, delete it. Update this.size. @@ -302,6 +305,7 @@ public class DefaultMemStore implements MemStore { removeFromCellSet(found); long sz = heapSizeChange(found, true); activeSection.getHeapSize().addAndGet(-sz); + activeSection.getCellCount().decrementAndGet(); } } @@ -589,6 +593,7 @@ public class DefaultMemStore implements MemStore { long delta = heapSizeChange(cur, true); addedSize -= delta; activeSection.getHeapSize().addAndGet(-delta); + activeSection.getCellCount().decrementAndGet(); if (removedCells != null) { removedCells.add(cur); } @@ -1013,7 +1018,7 @@ public class DefaultMemStore implements MemStore { @Override public synchronized boolean seekToLastRow() { Cell first = activeAtCreation.getCellSkipListSet().isEmpty() ? null - : activeAtCreation.getCellSkipListSet().last(); + : activeAtCreation.getCellSkipListSet().last(); Cell second = snapshotAtCreation.getCellSkipListSet().isEmpty() ? null : snapshotAtCreation.getCellSkipListSet().last(); Cell higherCell = getHighest(first, second); @@ -1036,7 +1041,8 @@ public class DefaultMemStore implements MemStore { public final static long DEEP_OVERHEAD = ClassSize.align(FIXED_OVERHEAD + (2 * ClassSize.ATOMIC_LONG) + (2 * ClassSize.TIMERANGE_TRACKER) + - (2 * ClassSize.CELL_SKIPLIST_SET) + (2 * ClassSize.CONCURRENT_SKIPLISTMAP)); + (2 * ClassSize.CELL_SKIPLIST_SET) + (2 * ClassSize.CONCURRENT_SKIPLISTMAP) + + ClassSize.ATOMIC_INTEGER); /* * Calculate how the MemStore size has changed. Includes overhead of the @@ -1128,6 +1134,7 @@ public class DefaultMemStore implements MemStore { * Used to track own heapSize. */ private final AtomicLong heapSize; + private final AtomicInteger cellCount; private final MemStoreLAB allocator; static Section newSnapshotSection(final KeyValue.KVComparator c) { @@ -1143,6 +1150,7 @@ public class DefaultMemStore implements MemStore { final Configuration conf, long initHeapSize) { this.cellSet = new CellSkipListSet(c); this.heapSize = new AtomicLong(initHeapSize); + this.cellCount = new AtomicInteger(0); if (conf != null && conf.getBoolean(USEMSLAB_KEY, USEMSLAB_DEFAULT)) { String className = conf.get(MSLAB_CLASS_NAME, HeapMemStoreLAB.class.getName()); this.allocator = ReflectionUtils.instantiateWithCustomCtor(className, @@ -1164,9 +1172,12 @@ public class DefaultMemStore implements MemStore { return heapSize; } + AtomicInteger getCellCount() { + return cellCount; + } + MemStoreLAB getMemStoreLAB() { return allocator; } - } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java index 12559e7fb9..170633ecee 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java @@ -305,13 +305,15 @@ public class TestHeapSize { actual = DefaultMemStore.DEEP_OVERHEAD; expected = ClassSize.estimateBase(cl, false); expected += (2 * ClassSize.estimateBase(AtomicLong.class, false)); + expected += ClassSize.estimateBase(AtomicInteger.class, false); expected += (2 * ClassSize.estimateBase(CellSkipListSet.class, false)); expected += (2 * ClassSize.estimateBase(ConcurrentSkipListMap.class, false)); expected += (2 * ClassSize.estimateBase(TimeRangeTracker.class, false)); - if(expected != actual) { + if (expected != actual) { ClassSize.estimateBase(cl, true); ClassSize.estimateBase(AtomicLong.class, true); ClassSize.estimateBase(AtomicLong.class, true); + ClassSize.estimateBase(AtomicInteger.class, true); ClassSize.estimateBase(CellSkipListSet.class, true); ClassSize.estimateBase(CellSkipListSet.class, true); ClassSize.estimateBase(ConcurrentSkipListMap.class, true); -- 2.20.1