diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java index 8564045..57374a2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AbstractMemStore.java @@ -152,7 +152,7 @@ public abstract class AbstractMemStore implements MemStore { @Override public MemstoreSize getSnapshotSize() { - return new MemstoreSize(this.snapshot.keySize(), this.snapshot.heapOverhead()); + return new MemstoreSize(this.snapshot.keySize(), this.snapshot.heapSize()); } @Override @@ -299,10 +299,10 @@ public abstract class AbstractMemStore implements MemStore { protected abstract long keySize(); /** - * @return The total heap overhead of cells in this memstore. We will not consider cells in the + * @return The total heap size of this memstore. We will not consider cells in the * snapshot */ - protected abstract long heapOverhead(); + protected abstract long heapSize(); protected CellComparator getComparator() { return comparator; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java index 1cd30dd..ad673b0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactingMemStore.java @@ -29,7 +29,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.HColumnDescriptor; import org.apache.hadoop.hbase.classification.InterfaceAudience; @@ -116,9 +115,9 @@ public class CompactingMemStore extends AbstractMemStore { @Override public MemstoreSize size() { MemstoreSize memstoreSize = new MemstoreSize(); - memstoreSize.incMemstoreSize(this.active.keySize(), this.active.heapOverhead()); + memstoreSize.incMemstoreSize(this.active.keySize(), this.active.heapSize()); for (Segment item : pipeline.getSegments()) { - memstoreSize.incMemstoreSize(item.keySize(), item.heapOverhead()); + memstoreSize.incMemstoreSize(item.keySize(), item.heapSize()); } return memstoreSize; } @@ -182,13 +181,13 @@ public class CompactingMemStore extends AbstractMemStore { // if snapshot is empty the tail of the pipeline (or everything in the memstore) is flushed if (compositeSnapshot) { snapshotSize = pipeline.getPipelineSize(); - snapshotSize.incMemstoreSize(this.active.keySize(), this.active.heapOverhead()); + snapshotSize.incMemstoreSize(this.active.keySize(), this.active.heapSize()); } else { snapshotSize = pipeline.getTailSize(); } } return snapshotSize.getDataSize() > 0 ? snapshotSize - : new MemstoreSize(this.active.keySize(), this.active.heapOverhead()); + : new MemstoreSize(this.active.keySize(), this.active.heapSize()); } @Override @@ -202,11 +201,11 @@ public class CompactingMemStore extends AbstractMemStore { } @Override - protected long heapOverhead() { + protected long heapSize() { // Need to consider heapOverhead of all segments in pipeline and active - long h = this.active.heapOverhead(); + long h = this.active.heapSize(); for (Segment segment : this.pipeline.getSegments()) { - h += segment.heapOverhead(); + h += segment.heapSize(); } return h; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java index 2fd2a14..dac9dc3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactionPipeline.java @@ -132,23 +132,23 @@ public class CompactionPipeline { long suffixDataSize = getSegmentsKeySize(suffix); long newDataSize = segment.keySize(); long dataSizeDelta = suffixDataSize - newDataSize; - long suffixHeapOverhead = getSegmentsHeapOverhead(suffix); - long newHeapOverhead = segment.heapOverhead(); - long heapOverheadDelta = suffixHeapOverhead - newHeapOverhead; - region.addMemstoreSize(new MemstoreSize(-dataSizeDelta, -heapOverheadDelta)); + long suffixHeapSize = getSegmentsHeapSize(suffix); + long newHeapSize = segment.heapSize(); + long heapSizeDelta = suffixHeapSize - newHeapSize; + region.addMemstoreSize(new MemstoreSize(-dataSizeDelta, -heapSizeDelta)); if (LOG.isDebugEnabled()) { LOG.debug("Suffix data size: " + suffixDataSize + " compacted item data size: " - + newDataSize + ". Suffix heap overhead: " + suffixHeapOverhead - + " compacted item heap overhead: " + newHeapOverhead); + + newDataSize + ". Suffix heap size: " + suffixHeapSize + + " compacted item heap size: " + newHeapSize); } } return true; } - private static long getSegmentsHeapOverhead(List list) { + private static long getSegmentsHeapSize(List list) { long res = 0; for (Segment segment : list) { - res += segment.heapOverhead(); + res += segment.heapSize(); } return res; } @@ -237,12 +237,12 @@ public class CompactionPipeline { public MemstoreSize getTailSize() { if (isEmpty()) return MemstoreSize.EMPTY_SIZE; - return new MemstoreSize(pipeline.peekLast().keySize(), pipeline.peekLast().heapOverhead()); + return new MemstoreSize(pipeline.peekLast().keySize(), pipeline.peekLast().heapSize()); } public MemstoreSize getPipelineSize() { if (isEmpty()) return MemstoreSize.EMPTY_SIZE; - return new MemstoreSize(getSegmentsKeySize(pipeline), getSegmentsHeapOverhead(pipeline)); + return new MemstoreSize(getSegmentsKeySize(pipeline), getSegmentsHeapSize(pipeline)); } private void swapSuffix(List suffix, ImmutableSegment segment, diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java index 4fdd2d0..d7b988c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompositeImmutableSegment.java @@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting; import org.apache.commons.logging.Log; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; -import org.apache.hadoop.hbase.CellScanner; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Scan; @@ -43,7 +42,6 @@ import java.util.SortedSet; public class CompositeImmutableSegment extends ImmutableSegment { private final List segments; - private final CellComparator comparator; // CompositeImmutableSegment is used for snapshots and snapshot should // support getTimeRangeTracker() interface. // Thus we hold a constant TRT build in the construction time from TRT of the given segments. @@ -58,7 +56,6 @@ public class CompositeImmutableSegment extends ImmutableSegment { public CompositeImmutableSegment(CellComparator comparator, List segments) { super(comparator); - this.comparator = comparator; this.segments = segments; this.timeRangeTracker = new TimeRangeTracker(); for (ImmutableSegment s : segments) { @@ -69,10 +66,12 @@ public class CompositeImmutableSegment extends ImmutableSegment { } @VisibleForTesting + @Override public List getAllSegments() { return new LinkedList(segments); } + @Override public long getNumOfSegments() { return segments.size(); } @@ -82,6 +81,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { * general segment scanner. * @return a special scanner for the MemStoreSnapshot object */ + @Override public KeyValueScanner getKeyValueScanner() { KeyValueScanner scanner; List list = new ArrayList(segments.size()); @@ -115,6 +115,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { /** * @return whether the segment has any cells */ + @Override public boolean isEmpty() { for (ImmutableSegment s : segments) { if (!s.isEmpty()) return false; @@ -125,6 +126,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { /** * @return number of cells in segment */ + @Override public int getCellsCount() { int result = 0; for (ImmutableSegment s : segments) { @@ -136,6 +138,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { /** * @return the first cell in the segment that has equal or greater key than the given cell */ + @Override public Cell getFirstAfter(Cell cell) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } @@ -143,6 +146,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { /** * Closing a segment before it is being discarded */ + @Override public void close() { if (flushingScanner != null) { flushingScanner.close(); @@ -158,14 +162,17 @@ public class CompositeImmutableSegment extends ImmutableSegment { * otherwise the given cell is returned * @return either the given cell or its clone */ + @Override public Cell maybeCloneWithAllocator(Cell cell) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } + @Override public boolean shouldSeek(Scan scan, long oldestUnexpiredTS){ throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } + @Override public long getMinTimestamp(){ throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } @@ -174,6 +181,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { * Creates the scanner for the given read point * @return a scanner for the given read point */ + @Override public KeyValueScanner getScanner(long readPoint) { KeyValueScanner resultScanner; List list = new ArrayList(segments.size()); @@ -194,6 +202,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { * Creates the scanner for the given read point, and a specific order in a list * @return a scanner for the given read point */ + @Override public KeyValueScanner getScanner(long readPoint, long order) { KeyValueScanner resultScanner; List list = new ArrayList(segments.size()); @@ -210,6 +219,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { return resultScanner; } + @Override public boolean isTagsPresent() { for (ImmutableSegment s : segments) { if (s.isTagsPresent()) return true; @@ -217,10 +227,12 @@ public class CompositeImmutableSegment extends ImmutableSegment { return false; } + @Override public void incScannerCount() { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } + @Override public void decScannerCount() { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } @@ -230,7 +242,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { * immutable CellSet after its creation in immutable segment constructor * @return this object */ - + @Override protected CompositeImmutableSegment setCellSet(CellSet cellSetOld, CellSet cellSetNew) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } @@ -238,6 +250,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { /** * @return Sum of all cell's size. */ + @Override public long keySize() { return this.keySize; } @@ -245,10 +258,11 @@ public class CompositeImmutableSegment extends ImmutableSegment { /** * @return The heap overhead of this segment. */ - public long heapOverhead() { + @Override + public long heapSize() { long result = 0; for (ImmutableSegment s : segments) { - result += s.heapOverhead(); + result += s.heapSize(); } return result; } @@ -256,39 +270,44 @@ public class CompositeImmutableSegment extends ImmutableSegment { /** * Updates the heap size counter of the segment by the given delta */ - protected void incSize(long delta, long heapOverhead) { - throw new IllegalStateException("Not supported by CompositeImmutableScanner"); - } - - protected void incHeapOverheadSize(long delta) { + @Override + protected void incSize(long dataSizeDelta, long heapOverhead, boolean mslabUsed, + MemstoreSize memstoreSize) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } + @Override public long getMinSequenceId() { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } + @Override public TimeRangeTracker getTimeRangeTracker() { return this.timeRangeTracker; } //*** Methods for SegmentsScanner + @Override public Cell last() { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } + @Override public Iterator iterator() { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } + @Override public SortedSet headSet(Cell firstKeyOnRow) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } + @Override public int compare(Cell left, Cell right) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } + @Override public int compareRows(Cell left, Cell right) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } @@ -296,27 +315,23 @@ public class CompositeImmutableSegment extends ImmutableSegment { /** * @return a set of all cells in the segment */ + @Override protected CellSet getCellSet() { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } - /** - * Returns the Cell comparator used by this segment - * @return the Cell comparator used by this segment - */ - protected CellComparator getComparator() { - return comparator; - } - + @Override protected void internalAdd(Cell cell, boolean mslabUsed, MemstoreSize memstoreSize) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } + @Override protected void updateMetaInfo(Cell cellToAdd, boolean succ, boolean mslabUsed, MemstoreSize memstoreSize) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } + @Override protected long heapOverheadChange(Cell cell, boolean succ) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } @@ -326,6 +341,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { * @param firstCell a cell in the segment * @return a subset of the segment cell set, which starts with the given cell */ + @Override protected SortedSet tailSet(Cell firstCell) { throw new IllegalStateException("Not supported by CompositeImmutableScanner"); } @@ -334,6 +350,7 @@ public class CompositeImmutableSegment extends ImmutableSegment { /** * Dumps all cells of the segment into the given log */ + @Override void dump(Log log) { for (ImmutableSegment s : segments) { s.dump(log); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java index 76442e1..a3cd85c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/DefaultMemStore.java @@ -108,7 +108,7 @@ public class DefaultMemStore extends AbstractMemStore { public MemstoreSize getFlushableSize() { MemstoreSize snapshotSize = getSnapshotSize(); return snapshotSize.getDataSize() > 0 ? snapshotSize - : new MemstoreSize(keySize(), heapOverhead()); + : new MemstoreSize(keySize(), heapSize()); } @Override @@ -117,8 +117,8 @@ public class DefaultMemStore extends AbstractMemStore { } @Override - protected long heapOverhead() { - return this.active.heapOverhead(); + protected long heapSize() { + return this.active.heapSize(); } @Override @@ -147,7 +147,7 @@ public class DefaultMemStore extends AbstractMemStore { @Override public MemstoreSize size() { - return new MemstoreSize(this.active.keySize(), this.active.heapOverhead()); + return new MemstoreSize(this.active.keySize(), this.active.heapSize()); } /** @@ -190,21 +190,18 @@ public class DefaultMemStore extends AbstractMemStore { // Give each its own ts memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memstoreSize); } - LOG.info("memstore1 estimated size=" - + (memstoreSize.getDataSize() + memstoreSize.getHeapOverhead())); + LOG.info("memstore1 estimated size=" + (memstoreSize.getDataSize())); for (int i = 0; i < count; i++) { memstore1.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, empty), memstoreSize); } - LOG.info("memstore1 estimated size (2nd loading of same data)=" - + (memstoreSize.getDataSize() + memstoreSize.getHeapOverhead())); + LOG.info("memstore1 estimated size(2nd loading of same data)=" + (memstoreSize.getDataSize())); // Make a variably sized memstore. DefaultMemStore memstore2 = new DefaultMemStore(); memstoreSize = new MemstoreSize(); for (int i = 0; i < count; i++) { memstore2.add(new KeyValue(Bytes.toBytes(i), fam, qf, i, new byte[i]), memstoreSize); } - LOG.info("memstore2 estimated size=" - + (memstoreSize.getDataSize() + memstoreSize.getHeapOverhead())); + LOG.info("memstore2 estimated size=" + (memstoreSize.getDataSize())); final int seconds = 30; LOG.info("Waiting " + seconds + " seconds while heap dump is taken"); LOG.info("Exiting."); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index b664a4a..b971d62 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -1689,7 +1689,7 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi this.closed.set(true); if (!canFlush) { - this.decrMemstoreSize(new MemstoreSize(memstoreDataSize.get(), getMemstoreHeapOverhead())); + this.decrMemstoreSize(new MemstoreSize(memstoreDataSize.get(), getMemstoreHeapSize())); } else if (memstoreDataSize.get() != 0) { LOG.error("Memstore size is " + memstoreDataSize.get()); } @@ -1714,12 +1714,12 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi } } - private long getMemstoreHeapOverhead() { - long overhead = 0; + private long getMemstoreHeapSize() { + long size = 0; for (Store s : this.stores.values()) { - overhead += s.getSizeOfMemStore().getHeapOverhead(); + size += s.getSizeOfMemStore().getHeapSize(); } - return overhead; + return size; } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java index 007c28b..5e93d7f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HStore.java @@ -377,7 +377,7 @@ public class HStore implements Store { @Deprecated public long getFlushableSize() { MemstoreSize size = getSizeToFlush(); - return size.getDataSize() + size.getHeapOverhead(); + return size.getHeapSize(); } @Override @@ -389,7 +389,7 @@ public class HStore implements Store { @Deprecated public long getSnapshotSize() { MemstoreSize size = getSizeOfSnapshot(); - return size.getDataSize() + size.getHeapOverhead(); + return size.getHeapSize(); } @Override @@ -2051,7 +2051,7 @@ public class HStore implements Store { @Deprecated public long getMemStoreSize() { MemstoreSize size = getSizeOfMemStore(); - return size.getDataSize() + size.getHeapOverhead(); + return size.getHeapSize(); } @Override @@ -2282,7 +2282,7 @@ public class HStore implements Store { @Override public long heapSize() { MemstoreSize memstoreSize = this.memstore.size(); - return DEEP_OVERHEAD + memstoreSize.getDataSize() + memstoreSize.getHeapOverhead(); + return DEEP_OVERHEAD + memstoreSize.getHeapSize(); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java index 96c9052..1e93af2 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HeapMemoryManager.java @@ -21,7 +21,6 @@ package org.apache.hadoop.hbase.regionserver; import static org.apache.hadoop.hbase.HConstants.HFILE_BLOCK_CACHE_SIZE_KEY; import java.lang.management.ManagementFactory; -import java.lang.management.MemoryType; import java.lang.management.MemoryUsage; import java.util.ArrayList; import java.util.List; @@ -305,7 +304,7 @@ public class HeapMemoryManager { // TODO : add support for offheap metrics tunerContext.setCurBlockCacheUsed((float) blockCache.getCurrentSize() / maxHeapSize); metricsHeapMemoryManager.setCurBlockCacheSizeGauge(blockCache.getCurrentSize()); - long globalMemstoreHeapSize = regionServerAccounting.getGlobalMemstoreSize(); + long globalMemstoreHeapSize = regionServerAccounting.getGlobalMemstoreHeapSize(); tunerContext.setCurMemStoreUsed((float) globalMemstoreHeapSize / maxHeapSize); metricsHeapMemoryManager.setCurMemStoreSizeGauge(globalMemstoreHeapSize); tunerContext.setCurBlockCacheSize(blockCachePercent); diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java index 430b642..401c728 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableMemStoreLAB.java @@ -37,6 +37,8 @@ public class ImmutableMemStoreLAB implements MemStoreLAB { private final List mslabs; public ImmutableMemStoreLAB(List mslabs) { + assert mslabs != null; + assert mslabs.size() > 0; this.mslabs = mslabs; } @@ -89,4 +91,9 @@ public class ImmutableMemStoreLAB implements MemStoreLAB { checkAndCloseMSLABs(count); } } + + @Override + public boolean isOffheap() { + return mslabs.get(0).isOffheap(); + } } \ No newline at end of file diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java index 547d332..9ae51f3 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/ImmutableSegment.java @@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.util.CollectionBackedScanner; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; -import java.util.LinkedList; import java.util.List; /** @@ -121,14 +120,13 @@ public class ImmutableSegment extends Segment { super(new CellSet(comparator), // initiailize the CellSet with empty CellSet comparator, memStoreLAB); type = Type.SKIPLIST_MAP_BASED; - MemstoreSize memstoreSize = new MemstoreSize(); while (iterator.hasNext()) { Cell c = iterator.next(); // The scanner is doing all the elimination logic // now we just copy it to the new segment Cell newKV = maybeCloneWithAllocator(c); boolean usedMSLAB = (newKV != c); - internalAdd(newKV, usedMSLAB, memstoreSize); + internalAdd(newKV, usedMSLAB, null); } this.timeRange = this.timeRangeTracker == null ? null : this.timeRangeTracker.toTimeRange(); } @@ -189,11 +187,7 @@ public class ImmutableSegment extends Segment { long newSegmentSizeDelta = -(numOfCells * ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); // add size of CellArrayMap and meta-data overhead per Cell newSegmentSizeDelta = newSegmentSizeDelta + numOfCells * ClassSize.CELL_ARRAY_MAP_ENTRY; - incSize(0, newSegmentSizeDelta); - if (memstoreSize != null) { - memstoreSize.incMemstoreSize(0, newSegmentSizeDelta); - } - + incSize(0, newSegmentSizeDelta, false, memstoreSize); return true; } @@ -202,7 +196,6 @@ public class ImmutableSegment extends Segment { // Create CellSet based on CellArrayMap from compacting iterator private CellSet createCellArrayMapSet(int numOfCells, MemStoreSegmentsIterator iterator, boolean merge) { - Cell[] cells = new Cell[numOfCells]; // build the Cell Array int i = 0; while (iterator.hasNext()) { @@ -248,7 +241,6 @@ public class ImmutableSegment extends Segment { // Create CellSet based on CellArrayMap from current ConcurrentSkipListMap based CellSet // (without compacting iterator) private CellSet recreateCellArrayMapSet(int numOfCells) { - Cell[] cells = new Cell[numOfCells]; // build the Cell Array Cell curCell; int idx = 0; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java index b7ac212..02b28b8 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreChunkPool.java @@ -262,4 +262,8 @@ public class MemStoreChunkPool implements HeapMemoryTuneObserver { } } } + + public boolean isOffheap() { + return this.offheap; + } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java index 2ee099c..e899491 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreFlusher.java @@ -24,7 +24,6 @@ import com.google.common.base.Preconditions; import java.io.IOException; import java.lang.Thread.UncaughtExceptionHandler; -import java.lang.management.MemoryType; import java.util.ArrayList; import java.util.ConcurrentModificationException; import java.util.HashMap; @@ -188,11 +187,11 @@ class MemStoreFlusher implements FlushRequester { (bestRegionReplica.getMemstoreSize() > secondaryMultiplier * regionToFlush.getMemstoreSize()))) { LOG.info("Refreshing storefiles of region " + bestRegionReplica - + " due to global heap pressure. Total memstore size=" + + " due to global heap pressure. Total memstore datasize=" + StringUtils .humanReadableInt(server.getRegionServerAccounting().getGlobalMemstoreDataSize()) - + " memstore heap overhead=" + StringUtils.humanReadableInt( - server.getRegionServerAccounting().getGlobalMemstoreHeapOverhead())); + + " memstore heap size=" + StringUtils.humanReadableInt( + server.getRegionServerAccounting().getGlobalMemstoreHeapSize())); flushedOne = refreshStoreFilesAndReclaimMemory(bestRegionReplica); if (!flushedOne) { LOG.info("Excluding secondary region " + bestRegionReplica + @@ -588,19 +587,25 @@ class MemStoreFlusher implements FlushRequester { while (flushType != FlushType.NORMAL && !server.isStopped()) { if (!blocked) { startTime = EnvironmentEdgeManager.currentTime(); - if (!server.getRegionServerAccounting().isOffheap()) { - logMsg("the global memstore size", "global memstore heap overhead"); - } else { + if (server.getRegionServerAccounting().isOffheap()) { switch (flushType) { case ABOVE_OFFHEAP_HIGHER_MARK: - logMsg("the global offheap memstore size", "global memstore heap overhead"); + logMsg("the global offheap memstore datasize", + server.getRegionServerAccounting().getGlobalMemstoreDataSize(), + server.getRegionServerAccounting().getGlobalMemstoreLimit()); break; case ABOVE_ONHEAP_HIGHER_MARK: - logMsg("global memstore heap overhead", ""); + logMsg("global memstore heap overhead", + server.getRegionServerAccounting().getGlobalMemstoreHeapSize(), + server.getRegionServerAccounting().getGlobalOnHeapMemstoreLimit()); break; default: break; } + } else { + logMsg("global memstore heapsize", + server.getRegionServerAccounting().getGlobalMemstoreHeapSize(), + server.getRegionServerAccounting().getGlobalMemstoreLimit()); } } blocked = true; @@ -637,16 +642,10 @@ class MemStoreFlusher implements FlushRequester { scope.close(); } - private void logMsg(String string1, String string2) { + private void logMsg(String string1, long val, long max) { LOG.info("Blocking updates on " + server.toString() + ": " + string1 + " " - + TraditionalBinaryPrefix - .long2String(server.getRegionServerAccounting().getGlobalMemstoreDataSize(), "", 1) - + " + " + string2 + " " - + TraditionalBinaryPrefix - .long2String(server.getRegionServerAccounting().getGlobalMemstoreHeapOverhead(), "", 1) - + " is >= than blocking " + TraditionalBinaryPrefix.long2String( - server.getRegionServerAccounting().getGlobalMemstoreLimit(), "", 1) - + " size"); + + TraditionalBinaryPrefix.long2String(val, "", 1) + " is >= than blocking " + + TraditionalBinaryPrefix.long2String(max, "", 1) + " size"); } @Override diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java index f6d1607..daf95ca 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java @@ -83,6 +83,11 @@ public interface MemStoreLAB { */ void decScannerCount(); + /** + * @return Whether off heap based MSLAB in place. + */ + boolean isOffheap(); + public static MemStoreLAB newInstance(Configuration conf) { MemStoreLAB memStoreLAB = null; if (isEnabled(conf)) { diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java index 30e4311..6791839 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLABImpl.java @@ -168,6 +168,11 @@ public class MemStoreLABImpl implements MemStoreLAB { } } + @Override + public boolean isOffheap() { + return chunkPool != null && chunkPool.isOffheap(); + } + /** * Try to retire the current chunk if it is still * c. Postcondition is that curChunk.get() diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java index 74d1e17..819e42c 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreSnapshot.java @@ -29,7 +29,7 @@ public class MemStoreSnapshot { private final long id; private final int cellsCount; private final long dataSize; - private final long heapOverhead; + private final long heapSize; private final TimeRangeTracker timeRangeTracker; private final KeyValueScanner scanner; private final boolean tagsPresent; @@ -38,7 +38,7 @@ public class MemStoreSnapshot { this.id = id; this.cellsCount = snapshot.getCellsCount(); this.dataSize = snapshot.keySize(); - this.heapOverhead = snapshot.heapOverhead(); + this.heapSize = snapshot.heapSize(); this.timeRangeTracker = snapshot.getTimeRangeTracker(); this.scanner = snapshot.getKeyValueScanner(); this.tagsPresent = snapshot.isTagsPresent(); @@ -65,8 +65,8 @@ public class MemStoreSnapshot { return dataSize; } - public long getHeapOverhead() { - return this.heapOverhead; + public long getHeapSize() { + return this.heapSize; } /** diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreSize.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreSize.java index fa7c342..947cfd9 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreSize.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MemstoreSize.java @@ -20,26 +20,26 @@ package org.apache.hadoop.hbase.regionserver; import org.apache.hadoop.hbase.classification.InterfaceAudience; /** - * Wraps the data size part and heap overhead of the memstore. + * Wraps the data size part and total heap space occupied by the memstore. */ @InterfaceAudience.Private public class MemstoreSize { private long dataSize; - private long heapOverhead; + private long heapSize; final private boolean isEmpty; static final MemstoreSize EMPTY_SIZE = new MemstoreSize(true); public MemstoreSize() { dataSize = 0; - heapOverhead = 0; + heapSize = 0; isEmpty = false; } public MemstoreSize(boolean isEmpty) { dataSize = 0; - heapOverhead = 0; + heapSize = 0; this.isEmpty = isEmpty; } @@ -47,40 +47,38 @@ public class MemstoreSize { return isEmpty; } - public MemstoreSize(long dataSize, long heapOverhead) { + public MemstoreSize(long dataSize, long heapSize) { this.dataSize = dataSize; - this.heapOverhead = heapOverhead; + this.heapSize = heapSize; this.isEmpty = false; } - public void incMemstoreSize(long dataSize, long heapOverhead) { - this.dataSize += dataSize; - this.heapOverhead += heapOverhead; + public void incMemstoreSize(long dataSizeDelta, long heapSizeDelta) { + this.dataSize += dataSizeDelta; + this.heapSize += heapSizeDelta; } - public void incMemstoreSize(MemstoreSize size) { - this.dataSize += size.dataSize; - this.heapOverhead += size.heapOverhead; + public void incMemstoreSize(MemstoreSize delta) { + this.dataSize += delta.dataSize; + this.heapSize += delta.heapSize; } - public void decMemstoreSize(long dataSize, long heapOverhead) { - this.dataSize -= dataSize; - this.heapOverhead -= heapOverhead; + public void decMemstoreSize(long dataSizeDelta, long heapSizeDelta) { + this.dataSize -= dataSizeDelta; + this.heapSize -= heapSizeDelta; } - public void decMemstoreSize(MemstoreSize size) { - this.dataSize -= size.dataSize; - this.heapOverhead -= size.heapOverhead; + public void decMemstoreSize(MemstoreSize delta) { + this.dataSize -= delta.dataSize; + this.heapSize -= delta.heapSize; } public long getDataSize() { - return isEmpty ? 0 : dataSize; } - public long getHeapOverhead() { - - return isEmpty ? 0 : heapOverhead; + public long getHeapSize() { + return isEmpty ? 0 : heapSize; } @Override @@ -89,18 +87,18 @@ public class MemstoreSize { return false; } MemstoreSize other = (MemstoreSize) obj; - return getDataSize() == other.dataSize && getHeapOverhead() == other.heapOverhead; + return this.dataSize == other.dataSize && this.heapSize == other.heapSize; } @Override public int hashCode() { long h = 13 * this.dataSize; - h = h + 14 * this.heapOverhead; + h = h + 14 * this.heapSize; return (int) h; } @Override public String toString() { - return "dataSize=" + this.dataSize + " , heapOverhead=" + this.heapOverhead; + return "dataSize=" + this.dataSize + " , heapSize=" + this.heapSize; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java index 3dbd7ad..e66720f 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/MutableSegment.java @@ -86,10 +86,7 @@ public class MutableSegment extends Segment { // removed cell is from MSLAB or not. Will do once HBASE-16438 is in int cellLen = getCellLength(cur); long heapOverheadDelta = heapOverheadChange(cur, true); - this.incSize(-cellLen, -heapOverheadDelta); - if (memstoreSize != null) { - memstoreSize.decMemstoreSize(cellLen, heapOverheadDelta); - } + this.incSize(-cellLen, -heapOverheadDelta, false, memstoreSize); it.remove(); } else { versionsVisible++; diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java index 3904393..b90befc 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerAccounting.java @@ -39,30 +39,29 @@ public class RegionServerAccounting { // memstore data size private final AtomicLong globalMemstoreDataSize = new AtomicLong(0); - // memstore heap over head size - private final AtomicLong globalMemstoreHeapOverhead = new AtomicLong(0); + // memstore heap size. When off heap MSLAB in place, this will be only heap overhead of the Cell + // POJOs and entry overhead of them onto memstore. When on heap MSLAB, this will be include heap + // overhead as well as the cell data size. Ya cell data is in on heap area only then. + private final AtomicLong globalMemstoreHeapSize = new AtomicLong(0); // Store the edits size during replaying WAL. Use this to roll back the // global memstore size once a region opening failed. private final ConcurrentMap replayEditsPerRegion = new ConcurrentSkipListMap(Bytes.BYTES_COMPARATOR); - private final Configuration conf; - private long globalMemStoreLimit; private final float globalMemStoreLimitLowMarkPercent; private long globalMemStoreLimitLowMark; - private final MemoryType memType; + private final boolean offheap; private long globalOnHeapMemstoreLimit; private long globalOnHeapMemstoreLimitLowMark; public RegionServerAccounting(Configuration conf) { - this.conf = conf; Pair globalMemstoreSizePair = MemorySizeUtil.getGlobalMemstoreSize(conf); this.globalMemStoreLimit = globalMemstoreSizePair.getFirst(); - this.memType = globalMemstoreSizePair.getSecond(); - this.globalMemStoreLimitLowMarkPercent = - MemorySizeUtil.getGlobalMemStoreHeapLowerMark(conf, this.memType == MemoryType.HEAP); + this.offheap = globalMemstoreSizePair.getSecond() == MemoryType.NON_HEAP; + this.globalMemStoreLimitLowMarkPercent = MemorySizeUtil.getGlobalMemStoreHeapLowerMark(conf, + !offheap); // When off heap memstore in use we configure the global off heap space for memstore as bytes // not as % of max memory size. In such case, the lower water mark should be specified using the // key "hbase.regionserver.global.memstore.size.lower.limit" which says % of the global upper @@ -79,63 +78,47 @@ public class RegionServerAccounting { (long) (this.globalOnHeapMemstoreLimit * this.globalMemStoreLimitLowMarkPercent); } - public long getGlobalMemstoreLimit() { + long getGlobalMemstoreLimit() { return this.globalMemStoreLimit; } - public long getOnheapGlobalMemstoreLimit() { + long getGlobalOnHeapMemstoreLimit() { return this.globalOnHeapMemstoreLimit; } // Called by the tuners. - public void setGlobalMemstoreLimits(long newGlobalMemstoreLimit) { - if (this.memType == MemoryType.HEAP) { - this.globalMemStoreLimit = newGlobalMemstoreLimit; - this.globalMemStoreLimitLowMark = - (long) (this.globalMemStoreLimit * this.globalMemStoreLimitLowMarkPercent); - } else { + void setGlobalMemstoreLimits(long newGlobalMemstoreLimit) { + if (this.offheap) { this.globalOnHeapMemstoreLimit = newGlobalMemstoreLimit; - this.globalOnHeapMemstoreLimitLowMark = - (long) (this.globalOnHeapMemstoreLimit * this.globalMemStoreLimitLowMarkPercent); + this.globalOnHeapMemstoreLimitLowMark = (long) (this.globalOnHeapMemstoreLimit + * this.globalMemStoreLimitLowMarkPercent); + } else { + this.globalMemStoreLimit = newGlobalMemstoreLimit; + this.globalMemStoreLimitLowMark = (long) (this.globalMemStoreLimit + * this.globalMemStoreLimitLowMarkPercent); } } - public boolean isOffheap() { - return this.memType == MemoryType.NON_HEAP; + boolean isOffheap() { + return this.offheap; } - public long getGlobalMemstoreLimitLowMark() { + long getGlobalMemstoreLimitLowMark() { return this.globalMemStoreLimitLowMark; } - public float getGlobalMemstoreLimitLowMarkPercent() { - return this.globalMemStoreLimitLowMarkPercent; - } - /** * @return the global Memstore data size in the RegionServer */ public long getGlobalMemstoreDataSize() { return globalMemstoreDataSize.get(); } - /** - * @return the global memstore heap overhead size in the RegionServer - */ - public long getGlobalMemstoreHeapOverhead() { - return this.globalMemstoreHeapOverhead.get(); - } /** - * @return the global memstore data size and heap overhead size for an onheap memstore - * whereas return the heap overhead size for an offheap memstore + * @return the global memstore heap size in the RegionServer */ - public long getGlobalMemstoreSize() { - if (isOffheap()) { - // get only the heap overhead for offheap memstore - return getGlobalMemstoreHeapOverhead(); - } else { - return getGlobalMemstoreDataSize() + getGlobalMemstoreHeapOverhead(); - } + public long getGlobalMemstoreHeapSize() { + return this.globalMemstoreHeapSize.get(); } /** @@ -144,12 +127,12 @@ public class RegionServerAccounting { */ public void incGlobalMemstoreSize(MemstoreSize memStoreSize) { globalMemstoreDataSize.addAndGet(memStoreSize.getDataSize()); - globalMemstoreHeapOverhead.addAndGet(memStoreSize.getHeapOverhead()); + globalMemstoreHeapSize.addAndGet(memStoreSize.getHeapSize()); } public void decGlobalMemstoreSize(MemstoreSize memStoreSize) { globalMemstoreDataSize.addAndGet(-memStoreSize.getDataSize()); - globalMemstoreHeapOverhead.addAndGet(-memStoreSize.getHeapOverhead()); + globalMemstoreHeapSize.addAndGet(-memStoreSize.getHeapSize()); } /** @@ -159,11 +142,7 @@ public class RegionServerAccounting { public FlushType isAboveHighWaterMark() { // for onheap memstore we check if the global memstore size and the // global heap overhead is greater than the global memstore limit - if (memType == MemoryType.HEAP) { - if (getGlobalMemstoreDataSize() + getGlobalMemstoreHeapOverhead() >= globalMemStoreLimit) { - return FlushType.ABOVE_ONHEAP_HIGHER_MARK; - } - } else { + if (this.offheap) { // If the configured memstore is offheap, check for two things // 1) If the global memstore data size is greater than the configured // 'hbase.regionserver.offheap.global.memstore.size' @@ -175,11 +154,15 @@ public class RegionServerAccounting { // Indicates that global memstore size is above the configured // 'hbase.regionserver.offheap.global.memstore.size' return FlushType.ABOVE_OFFHEAP_HIGHER_MARK; - } else if (getGlobalMemstoreHeapOverhead() >= this.globalOnHeapMemstoreLimit) { + } else if (getGlobalMemstoreHeapSize() >= this.globalOnHeapMemstoreLimit) { // Indicates that the offheap memstore's heap overhead is greater than the // configured 'hbase.regionserver.global.memstore.size'. return FlushType.ABOVE_ONHEAP_HIGHER_MARK; } + } else { + if (getGlobalMemstoreHeapSize() >= globalMemStoreLimit) { + return FlushType.ABOVE_ONHEAP_HIGHER_MARK; + } } return FlushType.NORMAL; } @@ -190,20 +173,20 @@ public class RegionServerAccounting { public FlushType isAboveLowWaterMark() { // for onheap memstore we check if the global memstore size and the // global heap overhead is greater than the global memstore lower mark limit - if (memType == MemoryType.HEAP) { - if (getGlobalMemstoreDataSize() + getGlobalMemstoreHeapOverhead() >= globalMemStoreLimitLowMark) { - return FlushType.ABOVE_ONHEAP_LOWER_MARK; - } - } else { + if (this.offheap) { if (getGlobalMemstoreDataSize() >= globalMemStoreLimitLowMark) { // Indicates that the offheap memstore's data size is greater than the global memstore // lower limit return FlushType.ABOVE_OFFHEAP_LOWER_MARK; - } else if (getGlobalMemstoreHeapOverhead() >= globalOnHeapMemstoreLimitLowMark) { + } else if (getGlobalMemstoreHeapSize() >= globalOnHeapMemstoreLimitLowMark) { // Indicates that the offheap memstore's heap overhead is greater than the global memstore // onheap lower limit return FlushType.ABOVE_ONHEAP_LOWER_MARK; } + } else { + if (getGlobalMemstoreHeapSize() >= globalMemStoreLimitLowMark) { + return FlushType.ABOVE_ONHEAP_LOWER_MARK; + } } return FlushType.NORMAL; } @@ -214,12 +197,11 @@ public class RegionServerAccounting { * global memstore size already exceeds lower limit. */ public double getFlushPressure() { - if (memType == MemoryType.HEAP) { - return (getGlobalMemstoreDataSize() + getGlobalMemstoreHeapOverhead()) * 1.0 - / globalMemStoreLimitLowMark; - } else { + if (this.offheap) { return Math.max(getGlobalMemstoreDataSize() * 1.0 / globalMemStoreLimitLowMark, - getGlobalMemstoreHeapOverhead() * 1.0 / globalOnHeapMemstoreLimitLowMark); + getGlobalMemstoreHeapSize() * 1.0 / globalOnHeapMemstoreLimitLowMark); + } else { + return getGlobalMemstoreHeapSize() * 1.0 / globalMemStoreLimitLowMark; } } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java index 8581517..05748a4 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Segment.java @@ -62,7 +62,7 @@ public abstract class Segment { // Sum of sizes of all Cells added to this Segment. Cell's heapSize is considered. This is not // including the heap overhead of this class. protected final AtomicLong dataSize; - protected final AtomicLong heapOverhead; + protected final AtomicLong heapSize; protected final TimeRangeTracker timeRangeTracker; protected volatile boolean tagsPresent; @@ -71,7 +71,7 @@ public abstract class Segment { protected Segment(CellComparator comparator) { this.comparator = comparator; this.dataSize = new AtomicLong(0); - this.heapOverhead = new AtomicLong(0); + this.heapSize = new AtomicLong(0); this.timeRangeTracker = new TimeRangeTracker(); } @@ -82,7 +82,7 @@ public abstract class Segment { this.minSequenceId = Long.MAX_VALUE; this.memStoreLAB = memStoreLAB; this.dataSize = new AtomicLong(0); - this.heapOverhead = new AtomicLong(0); + this.heapSize = new AtomicLong(0); this.tagsPresent = false; this.timeRangeTracker = new TimeRangeTracker(); } @@ -93,7 +93,7 @@ public abstract class Segment { this.minSequenceId = segment.getMinSequenceId(); this.memStoreLAB = segment.getMemStoreLAB(); this.dataSize = new AtomicLong(segment.keySize()); - this.heapOverhead = new AtomicLong(segment.heapOverhead.get()); + this.heapSize = new AtomicLong(segment.heapSize.get()); this.tagsPresent = segment.isTagsPresent(); this.timeRangeTracker = segment.getTimeRangeTracker(); } @@ -217,22 +217,29 @@ public abstract class Segment { } /** - * @return The heap overhead of this segment. + * @return The heap size of this segment. */ - public long heapOverhead() { - return this.heapOverhead.get(); + public long heapSize() { + return this.heapSize.get(); } /** - * Updates the heap size counter of the segment by the given delta + * Updates the size counters of the segment by the given delta */ - protected void incSize(long delta, long heapOverhead) { - this.dataSize.addAndGet(delta); - this.heapOverhead.addAndGet(heapOverhead); - } - - protected void incHeapOverheadSize(long delta) { - this.heapOverhead.addAndGet(delta); + protected void incSize(long dataSizeDelta, long heapOverhead, boolean mslabUsed, + MemstoreSize memstoreSize) { + long heapSizeDelta = heapOverhead; + // When MSLAB is not at all in place or it is on heap or when the current cell addition did not + // use MSLAB, we have the Cell data also in the heap memory area. Means the heap size includes + // the cell data size also and we have to consider that as well for the below size accounting. + if (!mslabUsed || this.memStoreLAB == null || !this.memStoreLAB.isOffheap()) { + heapSizeDelta += dataSizeDelta; + } + this.dataSize.addAndGet(dataSizeDelta); + this.heapSize.addAndGet(heapSizeDelta); + if (memstoreSize != null) { + memstoreSize.incMemstoreSize(dataSizeDelta, heapSizeDelta); + } } public long getMinSequenceId() { @@ -294,10 +301,7 @@ public abstract class Segment { cellSize = getCellLength(cellToAdd); } long overhead = heapOverheadChange(cellToAdd, succ); - incSize(cellSize, overhead); - if (memstoreSize != null) { - memstoreSize.incMemstoreSize(cellSize, overhead); - } + incSize(cellSize, overhead, mslabUsed, memstoreSize); getTimeRangeTracker().includeTimestamp(cellToAdd); minSequenceId = Math.min(minSequenceId, cellToAdd.getSequenceId()); // In no tags case this NoTagsKeyValue.getTagsLength() is a cheap call. @@ -309,6 +313,10 @@ public abstract class Segment { } } + /** + * @return The heap overheav which this cell addition causes. This includes this cell POJO's java + * overhead itself and additional overhead because of addition on to CSLM. + */ protected long heapOverheadChange(Cell cell, boolean succ) { if (succ) { if (cell instanceof ExtendedCell) { @@ -350,9 +358,9 @@ public abstract class Segment { public String toString() { String res = "Store segment of type "+this.getClass().getName()+"; "; res += "isEmpty "+(isEmpty()?"yes":"no")+"; "; - res += "cellCount "+getCellsCount()+"; "; + res += "cellsCount "+getCellsCount()+"; "; res += "cellsSize "+keySize()+"; "; - res += "heapOverhead "+heapOverhead()+"; "; + res += "totalHeapSize "+heapSize()+"; "; res += "Min ts "+getMinTimestamp()+"; "; return res; } diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java index 30e6a74..eb148a0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/Store.java @@ -256,6 +256,9 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf /** * @return The size of this store's memstore, in bytes * @deprecated Since 2.0 and will be removed in 3.0. Use {@link #getSizeOfMemStore()} instead. + *

+ * Note: When using off heap MSLAB feature, this will not account the cell data bytes size which + * is in off heap MSLAB area. */ @Deprecated long getMemStoreSize(); @@ -270,6 +273,9 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf * {@link #getMemStoreSize()} unless we are carrying snapshots and then it will be the size of * outstanding snapshots. * @deprecated Since 2.0 and will be removed in 3.0. Use {@link #getSizeToFlush()} instead. + *

+ * Note: When using off heap MSLAB feature, this will not account the cell data bytes size which + * is in off heap MSLAB area. */ @Deprecated long getFlushableSize(); @@ -285,6 +291,9 @@ public interface Store extends HeapSize, StoreConfigInformation, PropagatingConf * Returns the memstore snapshot size * @return size of the memstore snapshot * @deprecated Since 2.0 and will be removed in 3.0. Use {@link #getSizeOfSnapshot()} instead. + *

+ * Note: When using off heap MSLAB feature, this will not account the cell data bytes size which + * is in off heap MSLAB area. */ @Deprecated long getSnapshotSize(); diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java index 0c1880c..c97e6fb 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingMemStore.java @@ -569,7 +569,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { int totalCellsLen = addRowsByKeys(memstore, keys1); long totalHeapOverhead = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize()); - assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); + assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapSize()); MemstoreSize size = memstore.getFlushableSize(); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact @@ -579,7 +579,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { totalCellsLen = (totalCellsLen * 3) / 4; totalHeapOverhead = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize()); - assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); + assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapSize()); size = memstore.getFlushableSize(); MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot @@ -606,7 +606,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { long totalHeapOverhead = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); - assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); + assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapSize()); MemstoreSize size = memstore.getFlushableSize(); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact @@ -621,12 +621,12 @@ public class TestCompactingMemStore extends TestDefaultMemStore { totalCellsLen1 = (totalCellsLen1 * 3) / 4; assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); totalHeapOverhead = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); - assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); + assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapSize()); int totalCellsLen2 = addRowsByKeys(memstore, keys2); totalHeapOverhead += 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); - assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); + assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapSize()); size = memstore.getFlushableSize(); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact @@ -634,7 +634,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { totalCellsLen2 = totalCellsLen2 / 3;// 2 cells duplicated in set 2 assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); totalHeapOverhead = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); - assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); + assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapSize()); size = memstore.getFlushableSize(); MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot @@ -661,7 +661,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { int totalCellsLen1 = addRowsByKeys(memstore, keys1);// Adding 4 cells. assertEquals(totalCellsLen1, region.getMemstoreSize()); long totalHeapOverhead = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); - assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); + assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapSize()); MemstoreSize size = memstore.getFlushableSize(); ((CompactingMemStore)memstore).flushInMemory(); // push keys to pipeline and compact @@ -673,7 +673,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); // In memory flush to make a CellArrayMap instead of CSLM. See the overhead diff. totalHeapOverhead = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); - assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); + assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapSize()); int totalCellsLen2 = addRowsByKeys(memstore, keys2);// Adding 3 more cells. long totalHeapOverhead2 = 3 @@ -681,7 +681,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalHeapOverhead + totalHeapOverhead2, - ((CompactingMemStore) memstore).heapOverhead()); + ((CompactingMemStore) memstore).heapSize()); ((CompactingMemStore) memstore).disableCompaction(); size = memstore.getFlushableSize(); @@ -690,7 +690,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { // No change in the cells data size. ie. memstore size. as there is no compaction. assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalHeapOverhead + totalHeapOverhead2, - ((CompactingMemStore) memstore).heapOverhead()); + ((CompactingMemStore) memstore).heapSize()); int totalCellsLen3 = addRowsByKeys(memstore, keys3);// 3 more cells added assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3, @@ -698,7 +698,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { long totalHeapOverhead3 = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); assertEquals(totalHeapOverhead + totalHeapOverhead2 + totalHeapOverhead3, - ((CompactingMemStore) memstore).heapOverhead()); + ((CompactingMemStore) memstore).heapSize()); ((CompactingMemStore)memstore).enableCompaction(); size = memstore.getFlushableSize(); @@ -715,7 +715,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { regionServicesForStores.getMemstoreSize()); // Only 4 unique cells left assertEquals(4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY), - ((CompactingMemStore) memstore).heapOverhead()); + ((CompactingMemStore) memstore).heapSize()); size = memstore.getFlushableSize(); MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot @@ -733,7 +733,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { byte[] fam = Bytes.toBytes("testfamily"); byte[] qf = Bytes.toBytes("testqualifier"); long size = hmc.getActive().keySize(); - long heapOverhead = hmc.getActive().heapOverhead(); + long heapOverhead = hmc.getActive().heapSize(); int totalLen = 0; for (int i = 0; i < keys.length; i++) { long timestamp = System.currentTimeMillis(); @@ -746,7 +746,7 @@ public class TestCompactingMemStore extends TestDefaultMemStore { LOG.debug("added kv: " + kv.getKeyString() + ", timestamp:" + kv.getTimestamp()); } regionServicesForStores.addMemstoreSize(new MemstoreSize(hmc.getActive().keySize() - size, - hmc.getActive().heapOverhead() - heapOverhead)); + hmc.getActive().heapSize() - heapOverhead)); return totalLen; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java index 56ae72e..c59fd7d 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestCompactingToCellArrayMapMemStore.java @@ -82,7 +82,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore long totalCellsLen = addRowsByKeys(memstore, keys1); long totalHeapOverhead = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize()); - assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); + assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapSize()); assertEquals(4, memstore.getActive().getCellsCount()); MemstoreSize size = memstore.getFlushableSize(); @@ -96,7 +96,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore totalCellsLen = (totalCellsLen * 3) / 4; assertEquals(totalCellsLen, regionServicesForStores.getMemstoreSize()); totalHeapOverhead = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); - assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapOverhead()); + assertEquals(totalHeapOverhead, ((CompactingMemStore)memstore).heapSize()); for ( Segment s : memstore.getSegments()) { counter += s.getCellsCount(); } @@ -120,7 +120,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore long totalHeapOverhead1 = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); - assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapOverhead()); + assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapSize()); MemstoreSize size = memstore.getFlushableSize(); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact @@ -138,14 +138,14 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore totalCellsLen1 = (totalCellsLen1 * 3) / 4; totalHeapOverhead1 = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); - assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapOverhead()); + assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapSize()); long totalCellsLen2 = addRowsByKeys(memstore, keys2); long totalHeapOverhead2 = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalHeapOverhead1 + totalHeapOverhead2, - ((CompactingMemStore) memstore).heapOverhead()); + ((CompactingMemStore) memstore).heapSize()); size = memstore.getFlushableSize(); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact @@ -167,7 +167,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); totalHeapOverhead2 = 1 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); assertEquals(totalHeapOverhead1 + totalHeapOverhead2, - ((CompactingMemStore) memstore).heapOverhead()); + ((CompactingMemStore) memstore).heapSize()); size = memstore.getFlushableSize(); MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot @@ -189,7 +189,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore long totalHeapOverhead1 = 4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CONCURRENT_SKIPLISTMAP_ENTRY); assertEquals(totalCellsLen1, region.getMemstoreSize()); - assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapOverhead()); + assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapSize()); MemstoreSize size = memstore.getFlushableSize(); ((CompactingMemStore) memstore).flushInMemory(); // push keys to pipeline and compact @@ -203,7 +203,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore totalCellsLen1 = (totalCellsLen1 * 3) / 4; totalHeapOverhead1 = 3 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY); assertEquals(totalCellsLen1, regionServicesForStores.getMemstoreSize()); - assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapOverhead()); + assertEquals(totalHeapOverhead1, ((CompactingMemStore) memstore).heapSize()); long totalCellsLen2 = addRowsByKeys(memstore, keys2); long totalHeapOverhead2 = 3 @@ -211,7 +211,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalHeapOverhead1 + totalHeapOverhead2, - ((CompactingMemStore) memstore).heapOverhead()); + ((CompactingMemStore) memstore).heapSize()); ((CompactingMemStore) memstore).disableCompaction(); size = memstore.getFlushableSize(); @@ -219,7 +219,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore assertEquals(0, memstore.getSnapshot().getCellsCount()); assertEquals(totalCellsLen1 + totalCellsLen2, regionServicesForStores.getMemstoreSize()); assertEquals(totalHeapOverhead1 + totalHeapOverhead2, - ((CompactingMemStore) memstore).heapOverhead()); + ((CompactingMemStore) memstore).heapSize()); long totalCellsLen3 = addRowsByKeys(memstore, keys3); long totalHeapOverhead3 = 3 @@ -227,7 +227,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore assertEquals(totalCellsLen1 + totalCellsLen2 + totalCellsLen3, regionServicesForStores.getMemstoreSize()); assertEquals(totalHeapOverhead1 + totalHeapOverhead2 + totalHeapOverhead3, - ((CompactingMemStore) memstore).heapOverhead()); + ((CompactingMemStore) memstore).heapSize()); ((CompactingMemStore) memstore).enableCompaction(); size = memstore.getFlushableSize(); @@ -244,7 +244,7 @@ public class TestCompactingToCellArrayMapMemStore extends TestCompactingMemStore regionServicesForStores.getMemstoreSize()); // Only 4 unique cells left assertEquals(4 * (KeyValue.FIXED_OVERHEAD + ClassSize.CELL_ARRAY_MAP_ENTRY), - ((CompactingMemStore) memstore).heapOverhead()); + ((CompactingMemStore) memstore).heapSize()); size = memstore.getFlushableSize(); MemStoreSnapshot snapshot = memstore.snapshot(); // push keys to snapshot diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java index 93d28d5..9b84551 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestDefaultMemStore.java @@ -123,7 +123,7 @@ public class TestDefaultMemStore { // make sure memstore size increase won't double-count MSLAB chunk size assertEquals(Segment.getCellLength(kv), sizeChangeForFirstCell.getDataSize()); assertEquals(this.memstore.active.heapOverheadChange(kv, true), - sizeChangeForFirstCell.getHeapOverhead()); + sizeChangeForFirstCell.getHeapSize()); Segment segment = this.memstore.getActive(); MemStoreLAB msLab = segment.getMemStoreLAB(); if (msLab != null) { @@ -137,7 +137,7 @@ public class TestDefaultMemStore { } else { // make sure no memstore size change w/o MSLAB assertEquals(0, sizeChangeForSecondCell.getDataSize()); - assertEquals(0, sizeChangeForSecondCell.getHeapOverhead()); + assertEquals(0, sizeChangeForSecondCell.getHeapSize()); } } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java index 5a09274..0d20148 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestHeapMemoryManager.java @@ -928,7 +928,7 @@ public class TestHeapMemoryManager { } @Override - public long getGlobalMemstoreHeapOverhead() { + public long getGlobalMemstoreHeapSize() { return testMemstoreSize; } diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java index 332a125..d635878 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestWalAndCompactingMemStoreFlush.java @@ -243,7 +243,7 @@ public class TestWalAndCompactingMemStoreFlush { // CF1 was flushed to memory, but there is nothing to compact, and CF1 was flattened assertTrue(cf1MemstoreSizePhaseII.getDataSize() == cf1MemstoreSizePhaseI.getDataSize()); - assertTrue(cf1MemstoreSizePhaseII.getHeapOverhead() < cf1MemstoreSizePhaseI.getHeapOverhead()); + assertTrue(cf1MemstoreSizePhaseII.getHeapSize() < cf1MemstoreSizePhaseI.getHeapSize()); // CF2 should become empty assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseII); @@ -251,7 +251,7 @@ public class TestWalAndCompactingMemStoreFlush { // verify that CF3 was flushed to memory and was compacted (this is approximation check) assertTrue(cf3MemstoreSizePhaseI.getDataSize() > cf3MemstoreSizePhaseII.getDataSize()); assertTrue( - cf3MemstoreSizePhaseI.getHeapOverhead() / 2 > cf3MemstoreSizePhaseII.getHeapOverhead()); + cf3MemstoreSizePhaseI.getHeapSize() / 2 > cf3MemstoreSizePhaseII.getHeapSize()); // Now the smallest LSN in the region should be the same as the smallest // LSN in the memstore of CF1. @@ -485,14 +485,14 @@ public class TestWalAndCompactingMemStoreFlush { /* PHASE II - validation */ // CF1 was flushed to memory, should be flattened and take less space assertEquals(cf1MemstoreSizePhaseII.getDataSize(), cf1MemstoreSizePhaseI.getDataSize()); - assertTrue(cf1MemstoreSizePhaseII.getHeapOverhead() < cf1MemstoreSizePhaseI.getHeapOverhead()); + assertTrue(cf1MemstoreSizePhaseII.getHeapSize() < cf1MemstoreSizePhaseI.getHeapSize()); // CF2 should become empty assertEquals(MemstoreSize.EMPTY_SIZE, cf2MemstoreSizePhaseII); // verify that CF3 was flushed to memory and was not compacted (this is an approximation check) // if compacted CF# should be at least twice less because its every key was duplicated assertEquals(cf3MemstoreSizePhaseII.getDataSize(), cf3MemstoreSizePhaseI.getDataSize()); assertTrue( - cf3MemstoreSizePhaseI.getHeapOverhead() / 2 < cf3MemstoreSizePhaseII.getHeapOverhead()); + cf3MemstoreSizePhaseI.getHeapSize() / 2 < cf3MemstoreSizePhaseII.getHeapSize()); // Now the smallest LSN in the region should be the same as the smallest // LSN in the memstore of CF1.