Index: lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java =================================================================== --- lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java (revision 1300987) +++ lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java (working copy) @@ -17,8 +17,6 @@ * limitations under the License. */ -import org.apache.lucene.util.LuceneTestCase; - public class TestRamUsageEstimator extends LuceneTestCase { public void testBasic() { @@ -35,6 +33,17 @@ rue.estimateRamUsage(strings); } + public void testReferenceSize() { + if (VERBOSE) { + System.out.println("NOTE: This JVM is 64bit: " + Constants.JRE_IS_64BIT); + System.out.println("NOTE: Reference size in this JVM: " + Constants.JRE_REFERENCE_SIZE); + } + assertTrue(Constants.JRE_REFERENCE_SIZE == 4 || Constants.JRE_REFERENCE_SIZE == 8); + if (!Constants.JRE_IS_64BIT) { + assertEquals("For 32bit JVMs, reference size must always be 4", 4, Constants.JRE_REFERENCE_SIZE); + } + } + private static final class Holder { long field1 = 5000L; String name = "name"; Index: lucene/core/src/java/org/apache/lucene/util/Constants.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/Constants.java (revision 1300987) +++ lucene/core/src/java/org/apache/lucene/util/Constants.java (working copy) @@ -17,6 +17,8 @@ * limitations under the License. */ +import java.lang.reflect.Field; + import org.apache.lucene.LucenePackage; /** @@ -51,20 +53,63 @@ public static final boolean JRE_IS_64BIT; public static final boolean JRE_IS_MINIMUM_JAVA7; + public static final int JRE_REFERENCE_SIZE; static { - // NOTE: this logic may not be correct; if you know of a - // more reliable approach please raise it on java-dev! - final String x = System.getProperty("sun.arch.data.model"); - if (x != null) { - JRE_IS_64BIT = x.indexOf("64") != -1; - } else { - if (OS_ARCH != null && OS_ARCH.indexOf("64") != -1) { - JRE_IS_64BIT = true; + Object unsafe = null; + try { + final Class unsafeClass = Class.forName("sun.misc.Unsafe"); + final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe"); + unsafeField.setAccessible(true); + unsafe = unsafeField.get(null); + } catch (Exception e) { + unsafe = null; + } + + Boolean is64Bit = null; + if (unsafe != null) { + try { + final int addressSize = ((Number) unsafe.getClass().getMethod("addressSize") + .invoke(unsafe)).intValue(); + //System.out.println("Address size: " + addressSize); + switch (addressSize) { + case 4: + is64Bit = Boolean.FALSE; + break; + case 8: + is64Bit = Boolean.TRUE; + break; + } + } catch (Exception e) { + // ignore + } + } + // fallback, if Unsafe cannot be accessed: + if (is64Bit == null) { + final String x = System.getProperty("sun.arch.data.model"); + if (x != null) { + is64Bit = Boolean.valueOf(x.indexOf("64") != -1); } else { - JRE_IS_64BIT = false; + if (OS_ARCH != null && OS_ARCH.indexOf("64") != -1) { + is64Bit = Boolean.TRUE; + } else { + is64Bit = Boolean.FALSE; + } } } + assert is64Bit != null; + JRE_IS_64BIT = is64Bit.booleanValue(); + int referenceSize = JRE_IS_64BIT ? 8 : 4; + if (unsafe != null) { + try { + referenceSize = ((Number) unsafe.getClass().getMethod("arrayIndexScale", Class.class) + .invoke(unsafe, Object[].class)).intValue(); + } catch (Exception e) { + // ignore + } + } + JRE_REFERENCE_SIZE = referenceSize; + // this method only exists in Java 7: boolean v7 = true; try { Index: lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java (revision 1300987) +++ lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java (working copy) @@ -45,9 +45,49 @@ public final static int NUM_BYTES_DOUBLE = 8; public final static int NUM_BYTES_CHAR = 2; public final static int NUM_BYTES_OBJECT_HEADER = 8; - public final static int NUM_BYTES_OBJECT_REF = Constants.JRE_IS_64BIT ? 8 : 4; - public final static int NUM_BYTES_ARRAY_HEADER = NUM_BYTES_OBJECT_HEADER + NUM_BYTES_INT + NUM_BYTES_OBJECT_REF; + public final static int NUM_BYTES_OBJECT_REF = Constants.JRE_REFERENCE_SIZE; + public final static int NUM_BYTES_ARRAY_HEADER = NUM_BYTES_OBJECT_HEADER + NUM_BYTES_INT /* array length */; + + /** Aligns an object size to be the next multiple of 8. */ + public static long alignObjectSize(long size) { + return (size + 7) & 0x7FFFFFF8; + } + + /** Returns the size in bytes of the byte[] object. */ + public static long sizeOf(byte[] arr) { + return alignObjectSize(NUM_BYTES_ARRAY_HEADER + arr.length); + } + + /** Returns the size in bytes of the char[] object. */ + public static long sizeOf(char[] arr) { + return alignObjectSize(NUM_BYTES_ARRAY_HEADER + NUM_BYTES_CHAR * arr.length); + } + /** Returns the size in bytes of the short[] object. */ + public static long sizeOf(short[] arr) { + return alignObjectSize(NUM_BYTES_ARRAY_HEADER + NUM_BYTES_SHORT * arr.length); + } + + /** Returns the size in bytes of the int[] object. */ + public static long sizeOf(int[] arr) { + return alignObjectSize(NUM_BYTES_ARRAY_HEADER + NUM_BYTES_INT * arr.length); + } + + /** Returns the size in bytes of the float[] object. */ + public static long sizeOf(float[] arr) { + return alignObjectSize(NUM_BYTES_ARRAY_HEADER + NUM_BYTES_FLOAT * arr.length); + } + + /** Returns the size in bytes of the long[] object. */ + public static long sizeOf(long[] arr) { + return alignObjectSize(NUM_BYTES_ARRAY_HEADER + NUM_BYTES_LONG * arr.length); + } + + /** Returns the size in bytes of the double[] object. */ + public static long sizeOf(double[] arr) { + return alignObjectSize(NUM_BYTES_ARRAY_HEADER + NUM_BYTES_DOUBLE * arr.length); + } + private MemoryModel memoryModel; private final Map seen; @@ -161,25 +201,24 @@ clazz = clazz.getSuperclass(); } size += classSize; - return size; + return alignObjectSize(size); } private long sizeOfArray(Object obj) { + long size = arraySize; int len = Array.getLength(obj); - if (len == 0) { - return 0; - } - long size = arraySize; - Class arrayElementClazz = obj.getClass().getComponentType(); - if (arrayElementClazz.isPrimitive()) { - size += len * memoryModel.getPrimitiveSize(arrayElementClazz); - } else { - for (int i = 0; i < len; i++) { - size += refSize + size(Array.get(obj, i)); + if (len > 0) { + Class arrayElementClazz = obj.getClass().getComponentType(); + if (arrayElementClazz.isPrimitive()) { + size += len * memoryModel.getPrimitiveSize(arrayElementClazz); + } else { + for (int i = 0; i < len; i++) { + size += refSize + size(Array.get(obj, i)); + } } } - return size; + return alignObjectSize(size); } private static final long ONE_KB = 1024; Index: lucene/core/src/java/org/apache/lucene/util/AverageGuessMemoryModel.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/AverageGuessMemoryModel.java (revision 1300987) +++ lucene/core/src/java/org/apache/lucene/util/AverageGuessMemoryModel.java (working copy) @@ -20,59 +20,41 @@ import java.util.IdentityHashMap; import java.util.Map; -/** - * An average, best guess, MemoryModel that should work okay on most systems. - * - */ +/** An average, best guess, MemoryModel that should work okay on most systems. */ public class AverageGuessMemoryModel extends MemoryModel { + // best guess primitive sizes private final Map,Integer> sizes = new IdentityHashMap,Integer>() { { put(boolean.class, Integer.valueOf(1)); put(byte.class, Integer.valueOf(1)); - put(char.class, Integer.valueOf(2)); - put(short.class, Integer.valueOf(2)); - put(int.class, Integer.valueOf(4)); - put(float.class, Integer.valueOf(4)); - put(double.class, Integer.valueOf(8)); - put(long.class, Integer.valueOf(8)); + put(char.class, Integer.valueOf(RamUsageEstimator.NUM_BYTES_CHAR)); + put(short.class, Integer.valueOf(RamUsageEstimator.NUM_BYTES_SHORT)); + put(int.class, Integer.valueOf(RamUsageEstimator.NUM_BYTES_INT)); + put(float.class, Integer.valueOf(RamUsageEstimator.NUM_BYTES_FLOAT)); + put(double.class, Integer.valueOf(RamUsageEstimator.NUM_BYTES_DOUBLE)); + put(long.class, Integer.valueOf(RamUsageEstimator.NUM_BYTES_LONG)); } }; - /* - * (non-Javadoc) - * - * @see org.apache.lucene.util.MemoryModel#getArraySize() - */ @Override public int getArraySize() { - return 16; + return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER; } - /* - * (non-Javadoc) - * - * @see org.apache.lucene.util.MemoryModel#getClassSize() - */ @Override public int getClassSize() { - return 8; + return RamUsageEstimator.NUM_BYTES_OBJECT_HEADER; } - /* (non-Javadoc) - * @see org.apache.lucene.util.MemoryModel#getPrimitiveSize(java.lang.Class) - */ @Override public int getPrimitiveSize(Class clazz) { return sizes.get(clazz).intValue(); } - /* (non-Javadoc) - * @see org.apache.lucene.util.MemoryModel#getReferenceSize() - */ @Override public int getReferenceSize() { - return 4; + return RamUsageEstimator.NUM_BYTES_OBJECT_REF; } }