commit 68272a04a165995f9ed4425e4c1b558e651ce853 Author: Todd Lipcon Date: Thu Oct 20 17:32:56 2011 -0700 HBASE-3680. Publish metrics about mslab diff --git src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java index 8e9ba6b..31f1ab9 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java @@ -1286,12 +1286,14 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, new HDFSBlocksDistribution(); long totalStaticIndexSize = 0; long totalStaticBloomSize = 0; + long totalMslabWaste = 0; long tmpfiles; long tmpindex; long tmpfilesize; long tmpbloomsize; long tmpstaticsize; + long tmpMslabWaste; String cfname; // Note that this is a map of Doubles instead of Longs. This is because we @@ -1315,6 +1317,7 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, tmpfilesize = store.getStorefilesSize(); tmpbloomsize = store.getTotalStaticBloomSize(); tmpstaticsize = store.getTotalStaticIndexSize(); + tmpMslabWaste = store.memstore.getMslabWaste(); // Note that there is only one store per CF so setting is safe cfname = "cf." + store.toString(); @@ -1329,11 +1332,14 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, (store.getMemStoreSize() / (1024.0 * 1024))); this.incrMap(tempVals, cfname + ".staticIndexSizeKB", tmpstaticsize / 1024.0); + this.incrMap(tempVals, cfname + ".mslabWasteKB", + tmpMslabWaste / 1024.0); storefiles += tmpfiles; storefileIndexSize += tmpindex; totalStaticIndexSize += tmpstaticsize; totalStaticBloomSize += tmpbloomsize; + totalMslabWaste += tmpMslabWaste; } } @@ -1353,6 +1359,8 @@ public class HRegionServer implements HRegionInterface, HBaseRPCErrorHandler, (int) (totalStaticIndexSize / 1024)); this.metrics.totalStaticBloomSizeKB.set( (int) (totalStaticBloomSize / 1024)); + this.metrics.totalMslabWasteKB.set( + (int) (totalMslabWaste / 1024)); this.metrics.readRequestsCount.set(readRequestsCount); this.metrics.writeRequestsCount.set(writeRequestsCount); diff --git src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java index 34263e4..254c3dd 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/MemStore.java @@ -124,6 +124,18 @@ public class MemStore implements HeapSize { } } + /** + * @return the number of bytes "wasted" by external fragmentation + * in the MSLAB, if configured. + */ + long getMslabWaste() { + if (allocator != null) { + return allocator.getWastedBytes(); + } else { + return 0; + } + } + void dump() { for (KeyValue kv: this.kvset) { LOG.info(kv); diff --git src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java index cbb76e8..d5d1fd3 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/MemStoreLAB.java @@ -20,6 +20,7 @@ package org.apache.hadoop.hbase.regionserver; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import org.apache.hadoop.conf.Configuration; @@ -56,7 +57,9 @@ public class MemStoreLAB { final static String MAX_ALLOC_KEY = "hbase.hregion.memstore.mslab.max.allocation"; final static int MAX_ALLOC_DEFAULT = 256 * 1024; // allocs bigger than this don't go through allocator final int maxAlloc; - + + private final AtomicLong wastedSpace = new AtomicLong(); + public MemStoreLAB() { this(new Configuration()); } @@ -103,22 +106,35 @@ public class MemStoreLAB { } } + public long getWastedBytes() { + Chunk cur = curChunk.get(); + long ret = wastedSpace.get(); + if (cur != null) { + ret += cur.getFreeSpace(); + } + return ret; + } + /** * Try to retire the current chunk if it is still * c. Postcondition is that curChunk.get() * != c */ private void tryRetireChunk(Chunk c) { - @SuppressWarnings("unused") boolean weRetiredIt = curChunk.compareAndSet(c, null); // If the CAS succeeds, that means that we won the race - // to retire the chunk. We could use this opportunity to - // update metrics on external fragmentation. - // + // to retire the chunk. // If the CAS fails, that means that someone else already // retired the chunk for us. + if (weRetiredIt) { + // This isn't quite right, since another thread may + // have a small allocation concurrently with our retiring + // the chunk. But it should be very close to right, + // and this is just for metrics. + wastedSpace.addAndGet(c.getFreeSpace()); + } } - + /** * Get the current chunk, or, if there is no current chunk, * allocate a new one from the JVM. @@ -239,6 +255,15 @@ public class MemStoreLAB { " allocs=" + allocCount.get() + "waste=" + (data.length - nextFreeOffset.get()); } + + private int getFreeSpace() { + int off = nextFreeOffset.get(); + if (off >= 0) { + return data.length - off; + } else { + return 0; + } + } } /** diff --git src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java index cbf6da4..6ac06be 100644 --- src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java +++ src/main/java/org/apache/hadoop/hbase/regionserver/metrics/RegionServerMetrics.java @@ -155,6 +155,9 @@ public class RegionServerMetrics implements Updater { public final MetricsIntValue totalStaticBloomSizeKB = new MetricsIntValue("totalStaticBloomSizeKB", registry); + /** Total amount of memory wasted by external fragmentation in MSLABs */ + public final MetricsIntValue totalMslabWasteKB = + new MetricsIntValue("totalMslabWasteKB", registry); /** * HDFS blocks locality index */ diff --git src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java index 26c3c26..eb69ed0 100644 --- src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java +++ src/test/java/org/apache/hadoop/hbase/regionserver/TestMemStoreLAB.java @@ -46,7 +46,9 @@ public class TestMemStoreLAB { public void testLABRandomAllocation() { Random rand = new Random(); MemStoreLAB mslab = new MemStoreLAB(); + assertEquals(0, mslab.getWastedBytes()); int expectedOff = 0; + int slabsUsed = 0; byte[] lastBuffer = null; // 100K iterations by 0-1K alloc -> 50MB expected // should be reasonable for unit test and also cover wraparound @@ -58,12 +60,21 @@ public class TestMemStoreLAB { if (alloc.getData() != lastBuffer) { expectedOff = 0; lastBuffer = alloc.getData(); + slabsUsed++; } assertEquals(expectedOff, alloc.getOffset()); assertTrue("Allocation " + alloc + " overruns buffer", alloc.getOffset() + size <= alloc.getData().length); expectedOff += size; } + + // maximum waste is 1KB per slab plus + // whatever's left in current slab + long expectedWaste = slabsUsed * 1000 + + (lastBuffer.length - expectedOff); + long waste = mslab.getWastedBytes(); + assertTrue("waste should be less than " + expectedWaste + + " but was: " + waste, waste < expectedWaste); } @Test