diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java index f741f2c..c1734cc 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/KeyValue.java @@ -44,7 +44,6 @@ import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.ClassSize; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.RawComparator; - import com.google.common.annotations.VisibleForTesting; /** @@ -2598,11 +2597,18 @@ public class KeyValue implements Cell, HeapSize, Cloneable, SettableSequenceId, int sum = 0; sum += ClassSize.OBJECT;// the KeyValue object itself sum += ClassSize.REFERENCE;// pointer to "bytes" - sum += ClassSize.align(ClassSize.ARRAY);// "bytes" - sum += ClassSize.align(length);// number of bytes of data in the "bytes" array sum += 2 * Bytes.SIZEOF_INT;// offset, length sum += Bytes.SIZEOF_LONG;// memstoreTS - return ClassSize.align(sum); + + /* + * Deep object overhead for this KV consists of two parts. The first part is the KV object + * itself, while the second part is the backing byte[]. We will only count the array overhead + * from the byte[] only if this is the first KV in there. + */ + return ClassSize.align(sum) + + (offset == 0 + ? ClassSize.sizeOf(bytes, length) // count both length and object overhead + : length); // only count the number of bytes } /** diff --git hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java index 3dce955..8a17b22 100644 --- hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java +++ hbase-common/src/main/java/org/apache/hadoop/hbase/util/ClassSize.java @@ -29,6 +29,8 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import sun.misc.Unsafe; + /** * Class for determining the "size" of a class, an attempt to calculate the * actual bytes that an object of this class will occupy in memory @@ -39,8 +41,6 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; public class ClassSize { private static final Log LOG = LogFactory.getLog(ClassSize.class); - private static int nrOfRefsPerObj = 2; - /** Array overhead */ public static final int ARRAY; @@ -127,34 +127,117 @@ public class ClassSize { } /** + * MemoryLayout abstracts details about the JVM object layout. Default implementation is used in + * case Unsafe is not available. + */ + private static class MemoryLayout { + int headerSize() { + return 2 * oopSize(); + } + + int arrayHeaderSize() { + return (int) align(3 * oopSize()); + } + + /** + * Return the size of an "ordinary object pointer". Either 4 or 8, depending on 32/64 bit, + * and CompressedOops + */ + int oopSize() { + return is32BitJVM() ? 4 : 8; + } + + /** + * Aligns a number to 8. + * @param num number to align to 8 + * @return smallest number >= input that is a multiple of 8 + */ + public long align(long num) { + //The 7 comes from that the alignSize is 8 which is the number of bytes + //stored and sent together + return ((num + 7) >> 3) << 3; + } + + long sizeOf(byte[] b, int len) { + return align(arrayHeaderSize() + len * Unsafe.ARRAY_BYTE_INDEX_SCALE); + } + } + + /** + * UnsafeLayout uses Unsafe to guesstimate the object-layout related parameters like object header + * sizes and oop sizes + * See HBASE-15950. + */ + private static class UnsafeLayout extends MemoryLayout { + @SuppressWarnings("unused") + private static final class HeaderSize { + private byte a; + } + + public UnsafeLayout() { + } + + @Override + int headerSize() { + try { + return (int) UnsafeAccess.theUnsafe.objectFieldOffset( + HeaderSize.class.getDeclaredField("a")); + } catch (NoSuchFieldException | SecurityException e) { + LOG.error(e); + } + return super.headerSize(); + } + + @Override + int arrayHeaderSize() { + return UnsafeAccess.theUnsafe.arrayBaseOffset(byte[].class); + } + + @Override + int oopSize() { + // Unsafe.addressSize() returns 8, even with CompressedOops. This is how many bytes each + // element is allocated in an Object[]. + return Unsafe.ARRAY_OBJECT_INDEX_SCALE; + } + } + + private static MemoryLayout getMemoryLayout() { + // Have a safeguard in case Unsafe estimate is wrong. This is static context, there is + // no configuration, so we look at System property. + String enabled = System.getProperty("hbase.memorylayout.use.unsafe"); + if (UnsafeAccess.theUnsafe != null && (enabled == null || Boolean.parseBoolean(enabled))) { + LOG.debug("Using Unsafe to estimate memory layout"); + return new UnsafeLayout(); + } + LOG.debug("Not using Unsafe to estimate memory layout"); + return new MemoryLayout(); + } + + private static final MemoryLayout memoryLayout = getMemoryLayout(); + + /** * Method for reading the arc settings and setting overheads according * to 32-bit or 64-bit architecture. */ static { - //Default value is set to 8, covering the case when arcModel is unknown - if (is32BitJVM()) { - REFERENCE = 4; - } else { - REFERENCE = 8; - } + REFERENCE = memoryLayout.oopSize(); - OBJECT = 2 * REFERENCE; + OBJECT = memoryLayout.headerSize(); - ARRAY = align(3 * REFERENCE); + ARRAY = memoryLayout.arrayHeaderSize(); - ARRAYLIST = align(OBJECT + align(REFERENCE) + align(ARRAY) + - (2 * Bytes.SIZEOF_INT)); + ARRAYLIST = align(OBJECT + REFERENCE + (2 * Bytes.SIZEOF_INT)) + align(ARRAY); //noinspection PointlessArithmeticExpression - BYTE_BUFFER = align(OBJECT + align(REFERENCE) + align(ARRAY) + + BYTE_BUFFER = align(OBJECT + REFERENCE + (5 * Bytes.SIZEOF_INT) + - (3 * Bytes.SIZEOF_BOOLEAN) + Bytes.SIZEOF_LONG); + (3 * Bytes.SIZEOF_BOOLEAN) + Bytes.SIZEOF_LONG) + align(ARRAY); INTEGER = align(OBJECT + Bytes.SIZEOF_INT); MAP_ENTRY = align(OBJECT + 5 * REFERENCE + Bytes.SIZEOF_BOOLEAN); - TREEMAP = align(OBJECT + (2 * Bytes.SIZEOF_INT) + align(7 * REFERENCE)); + TREEMAP = align(OBJECT + (2 * Bytes.SIZEOF_INT) + 7 * REFERENCE); // STRING is different size in jdk6 and jdk7. Just use what we estimate as size rather than // have a conditional on whether jdk7. @@ -174,9 +257,9 @@ public class ClassSize { // The size changes from jdk7 to jdk8, estimate the size rather than use a conditional CONCURRENT_SKIPLISTMAP = (int) estimateBase(ConcurrentSkipListMap.class, false); - CONCURRENT_SKIPLISTMAP_ENTRY = align( + CONCURRENT_SKIPLISTMAP_ENTRY = align(OBJECT + (3 * REFERENCE)) + /* one node per entry */ - align((OBJECT + (3 * REFERENCE))/2)); /* one index per two entries */ + align((OBJECT + (3 * REFERENCE))/2); /* one index per two entries */ REENTRANT_LOCK = align(OBJECT + (3 * REFERENCE)); @@ -218,8 +301,7 @@ public class ClassSize { private static int [] getSizeCoefficients(Class cl, boolean debug) { int primitives = 0; int arrays = 0; - //The number of references that a new object takes - int references = nrOfRefsPerObj; + int references = 0; int index = 0; for ( ; null != cl; cl = cl.getSuperclass()) { @@ -275,15 +357,14 @@ public class ClassSize { * @return the size estimate, in bytes */ private static long estimateBaseFromCoefficients(int [] coeff, boolean debug) { - long prealign_size = coeff[0] + align(coeff[1] * ARRAY) + coeff[2] * REFERENCE; + long prealign_size = OBJECT + coeff[0] + coeff[2] * REFERENCE; // Round up to a multiple of 8 - long size = align(prealign_size); - if(debug) { + long size = align(prealign_size) + align(coeff[1] * ARRAY); + if (debug) { if (LOG.isDebugEnabled()) { LOG.debug("Primitives=" + coeff[0] + ", arrays=" + coeff[1] + - ", references(includes " + nrOfRefsPerObj + - " for object overhead)=" + coeff[2] + ", refSize " + REFERENCE + + ", references=" + coeff[2] + ", refSize " + REFERENCE + ", size=" + size + ", prealign_size=" + prealign_size); } } @@ -321,9 +402,7 @@ public class ClassSize { * @return smallest number >= input that is a multiple of 8 */ public static long align(long num) { - //The 7 comes from that the alignSize is 8 which is the number of bytes - //stored and sent together - return ((num + 7) >> 3) << 3; + return memoryLayout.align(num); } /** @@ -335,5 +414,9 @@ public class ClassSize { return model != null && model.equals("32"); } + public static long sizeOf(byte[] b, int len) { + return memoryLayout.sizeOf(b, len); + } + } diff --git hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java index 6522fde..7f920e4 100644 --- hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java +++ hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java @@ -5216,8 +5216,8 @@ public class HRegion implements HeapSize, PropagatingConfigurationObserver, Regi @Override public void releaseRowLocks(List rowLocks) { if (rowLocks != null) { - for (RowLock rowLock : rowLocks) { - rowLock.release(); + for (int i = 0; i < rowLocks.size(); i++) { + rowLocks.get(i).release(); } rowLocks.clear(); } diff --git hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java index 4a4b0e9..09e2271 100644 --- hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java +++ hbase-server/src/test/java/org/apache/hadoop/hbase/io/TestHeapSize.java @@ -39,7 +39,6 @@ import org.apache.hadoop.hbase.util.ClassSize; import org.junit.BeforeClass; import org.junit.Test; import org.junit.experimental.categories.Category; - import java.io.IOException; import java.lang.management.ManagementFactory; import java.lang.management.RuntimeMXBean; @@ -57,6 +56,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; /** * Testing the sizing that HeapSize offers and compares to the size given by @@ -68,17 +68,17 @@ public class TestHeapSize { // List of classes implementing HeapSize // BatchOperation, BatchUpdate, BlockIndex, Entry, Entry, HStoreKey // KeyValue, LruBlockCache, LruHashMap, Put, WALKey - + @BeforeClass public static void beforeClass() throws Exception { // Print detail on jvm so we know what is different should below test fail. RuntimeMXBean b = ManagementFactory.getRuntimeMXBean(); - LOG.info("name=" + b.getName()); - LOG.info("specname=" + b.getSpecName()); - LOG.info("specvendor=" + b.getSpecVendor()); + LOG.info("name=" + b.getName()); + LOG.info("specname=" + b.getSpecName()); + LOG.info("specvendor=" + b.getSpecVendor()); LOG.info("vmname=" + b.getVmName()); - LOG.info("vmversion=" + b.getVmVersion()); - LOG.info("vmvendor=" + b.getVmVendor()); + LOG.info("vmversion=" + b.getVmVersion()); + LOG.info("vmvendor=" + b.getVmVendor()); Map p = b.getSystemProperties(); LOG.info("properties=" + p); } @@ -132,7 +132,7 @@ public class TestHeapSize { // Object cl = Object.class; expected = ClassSize.estimateBase(cl, false); - actual = ClassSize.OBJECT; + actual = ClassSize.align(ClassSize.OBJECT); if(expected != actual) { ClassSize.estimateBase(cl, true); assertEquals(expected, actual); @@ -390,5 +390,25 @@ public class TestHeapSize { } } + @Test + public void testReferenceSize() { + LOG.info("ClassSize.REFERENCE is " + ClassSize.REFERENCE); + // oop should be either 4 or 8 + assertTrue(ClassSize.REFERENCE == 4 || ClassSize.REFERENCE == 8); + } + + @Test + public void testObjectSize() throws IOException { + LOG.info("header:" + ClassSize.OBJECT); + LOG.info("array header:" + ClassSize.ARRAY); + + if (ClassSize.is32BitJVM()) { + assertEquals(ClassSize.OBJECT, 8); + } else { + assertTrue(ClassSize.OBJECT == 12 || ClassSize.OBJECT == 16); // depending on CompressedOops + } + assertEquals(ClassSize.OBJECT + 4, ClassSize.ARRAY); + } + }