Index: lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java =================================================================== --- lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 1301918) +++ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy) @@ -59,6 +59,7 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Constants; // for javadocs +import org.apache.lucene.util.RamUsageEstimator; /** * High-performance single-document main memory Apache Lucene fulltext search index. @@ -473,38 +474,11 @@ /** * Returns a reasonable approximation of the main memory [bytes] consumed by - * this instance. Useful for smart memory sensititive caches/pools. Assumes - * fieldNames are interned, whereas tokenized terms are memory-overlaid. - * + * this instance. Useful for smart memory sensititive caches/pools. * @return the main memory consumption */ - public int getMemorySize() { - // for example usage in a smart cache see nux.xom.pool.Pool - int PTR = VM.PTR; - int INT = VM.INT; - int size = 0; - size += VM.sizeOfObject(2*PTR + INT); // memory index - if (sortedFields != null) size += VM.sizeOfObjectArray(sortedFields.length); - - size += VM.sizeOfHashMap(fields.size()); - for (Map.Entry entry : fields.entrySet()) { // for each Field Info - Info info = entry.getValue(); - size += VM.sizeOfObject(2*INT + 3*PTR); // Info instance vars - if (info.sortedTerms != null) size += VM.sizeOfObjectArray(info.sortedTerms.length); - - int len = info.terms.size(); - size += VM.sizeOfHashMap(len); - Iterator> iter2 = info.terms.entrySet().iterator(); - while (--len >= 0) { // for each term - Map.Entry e = iter2.next(); - // FIXME: this calculation is probably not correct since we use bytes now. - size += VM.sizeOfObject(PTR + 3*INT); // assumes substring() memory overlay -// size += STR + 2 * ((String) e.getKey()).length(); - ArrayIntList positions = e.getValue(); - size += VM.sizeOfArrayIntList(positions.size()); - } - } - return size; + public long getMemorySize() { + return new RamUsageEstimator().estimateRamUsage(this); } private int numPositions(ArrayIntList positions) { @@ -1126,61 +1100,4 @@ return norms; } } - - - /////////////////////////////////////////////////////////////////////////////// - // Nested classes: - /////////////////////////////////////////////////////////////////////////////// - private static final class VM { - - public static final int PTR = Constants.JRE_IS_64BIT ? 8 : 4; - - public static final int INT = 4; - private static final int LOG_PTR = (int) Math.round(log2(PTR)); - - /** - * Object header of any heap allocated Java object. - * ptr to class, info for monitor, gc, hash, etc. - */ - private static final int OBJECT_HEADER = 2*PTR; - - private VM() {} // not instantiable - - // assumes n > 0 - // 64 bit VM: - // 0 --> 0*PTR - // 1..8 --> 1*PTR - // 9..16 --> 2*PTR - private static int sizeOf(int n) { - return (((n-1) >> LOG_PTR) + 1) << LOG_PTR; - } - - public static int sizeOfObject(int n) { - return sizeOf(OBJECT_HEADER + n); - } - - public static int sizeOfObjectArray(int len) { - return sizeOfObject(INT + PTR*len); - } - - public static int sizeOfIntArray(int len) { - return sizeOfObject(INT + INT*len); - } - - public static int sizeOfHashMap(int len) { - return sizeOfObject(4*PTR + 4*INT) + sizeOfObjectArray(len) - + len * sizeOfObject(3*PTR + INT); // entries - } - - public static int sizeOfArrayIntList(int len) { - return sizeOfObject(PTR + INT) + sizeOfIntArray(len); - } - - /** logarithm to the base 2. Example: log2(4) == 2, log2(8) == 3 */ - private static double log2(double value) { - return Math.log(value) / Math.log(2); - } - - } - } Index: lucene/core/src/java/org/apache/lucene/util/AverageGuessMemoryModel.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/AverageGuessMemoryModel.java (revision 1301918) +++ lucene/core/src/java/org/apache/lucene/util/AverageGuessMemoryModel.java (working copy) @@ -1,78 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.IdentityHashMap; -import java.util.Map; - -/** - * An average, best guess, MemoryModel that should work okay on most systems. - * - */ -public class AverageGuessMemoryModel extends MemoryModel { - // best guess primitive sizes - private final Map,Integer> sizes = new IdentityHashMap,Integer>() { - { - put(boolean.class, Integer.valueOf(1)); - put(byte.class, Integer.valueOf(1)); - put(char.class, Integer.valueOf(2)); - put(short.class, Integer.valueOf(2)); - put(int.class, Integer.valueOf(4)); - put(float.class, Integer.valueOf(4)); - put(double.class, Integer.valueOf(8)); - put(long.class, Integer.valueOf(8)); - } - }; - - /* - * (non-Javadoc) - * - * @see org.apache.lucene.util.MemoryModel#getArraySize() - */ - @Override - public int getArraySize() { - return 16; - } - - /* - * (non-Javadoc) - * - * @see org.apache.lucene.util.MemoryModel#getClassSize() - */ - @Override - public int getClassSize() { - return 8; - } - - /* (non-Javadoc) - * @see org.apache.lucene.util.MemoryModel#getPrimitiveSize(java.lang.Class) - */ - @Override - public int getPrimitiveSize(Class clazz) { - return sizes.get(clazz).intValue(); - } - - /* (non-Javadoc) - * @see org.apache.lucene.util.MemoryModel#getReferenceSize() - */ - @Override - public int getReferenceSize() { - return 4; - } - -} Index: lucene/core/src/java/org/apache/lucene/util/Constants.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/Constants.java (revision 1301918) +++ lucene/core/src/java/org/apache/lucene/util/Constants.java (working copy) @@ -17,6 +17,7 @@ * limitations under the License. */ +import java.lang.reflect.Field; import org.apache.lucene.LucenePackage; /** @@ -48,22 +49,36 @@ @Deprecated public static final boolean JRE_IS_MINIMUM_JAVA6 = new Boolean(true).booleanValue(); // prevent inlining in foreign class files - - public static final boolean JRE_IS_64BIT; + public static final boolean JRE_IS_MINIMUM_JAVA7; + + /** True iff running on a 64bit JVM */ + public static final boolean JRE_IS_64BIT; + static { - // NOTE: this logic may not be correct; if you know of a - // more reliable approach please raise it on java-dev! - final String x = System.getProperty("sun.arch.data.model"); - if (x != null) { - JRE_IS_64BIT = x.indexOf("64") != -1; - } else { - if (OS_ARCH != null && OS_ARCH.indexOf("64") != -1) { - JRE_IS_64BIT = true; + boolean is64Bit = false; + try { + final Class unsafeClass = Class.forName("sun.misc.Unsafe"); + final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe"); + unsafeField.setAccessible(true); + final Object unsafe = unsafeField.get(null); + final int addressSize = ((Number) unsafeClass.getMethod("addressSize") + .invoke(unsafe)).intValue(); + //System.out.println("Address size: " + addressSize); + is64Bit = addressSize >= 8; + } catch (Exception e) { + final String x = System.getProperty("sun.arch.data.model"); + if (x != null) { + is64Bit = x.indexOf("64") != -1; } else { - JRE_IS_64BIT = false; + if (OS_ARCH != null && OS_ARCH.indexOf("64") != -1) { + is64Bit = true; + } else { + is64Bit = false; + } } } + JRE_IS_64BIT = is64Bit; // this method only exists in Java 7: boolean v7 = true; Index: lucene/core/src/java/org/apache/lucene/util/MemoryModel.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/MemoryModel.java (revision 1301918) +++ lucene/core/src/java/org/apache/lucene/util/MemoryModel.java (working copy) @@ -1,48 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with this - * work for additional information regarding copyright ownership. The ASF - * licenses this file to You under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -/** - * Returns primitive memory sizes for estimating RAM usage. - * - */ -public abstract class MemoryModel { - - /** - * @return size of array beyond contents - */ - public abstract int getArraySize(); - - /** - * @return Class size overhead - */ - public abstract int getClassSize(); - - /** - * @param clazz a primitive Class - bool, byte, char, short, long, float, - * short, double, int - * @return the size in bytes of given primitive Class - */ - public abstract int getPrimitiveSize(Class clazz); - - /** - * @return size of reference - */ - public abstract int getReferenceSize(); - -} Index: lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java (revision 1301918) +++ lucene/core/src/java/org/apache/lucene/util/packed/Direct16.java (working copy) @@ -78,8 +78,7 @@ } public long ramBytesUsed() { - return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + - values.length * RamUsageEstimator.NUM_BYTES_SHORT; + return RamUsageEstimator.sizeOf(values); } public void clear() { Index: lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java (revision 1301918) +++ lucene/core/src/java/org/apache/lucene/util/packed/Direct32.java (working copy) @@ -74,8 +74,7 @@ } public long ramBytesUsed() { - return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + - values.length * RamUsageEstimator.NUM_BYTES_INT; + return RamUsageEstimator.sizeOf(values); } public void clear() { Index: lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java (revision 1301918) +++ lucene/core/src/java/org/apache/lucene/util/packed/Direct64.java (working copy) @@ -70,8 +70,7 @@ } public long ramBytesUsed() { - return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + - values.length * RamUsageEstimator.NUM_BYTES_LONG; + return RamUsageEstimator.sizeOf(values); } public void clear() { Index: lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java (revision 1301918) +++ lucene/core/src/java/org/apache/lucene/util/packed/Direct8.java (working copy) @@ -79,7 +79,7 @@ } public long ramBytesUsed() { - return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER + values.length; + return RamUsageEstimator.sizeOf(values); } public void clear() { Index: lucene/core/src/java/org/apache/lucene/util/packed/Packed32.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Packed32.java (revision 1301918) +++ lucene/core/src/java/org/apache/lucene/util/packed/Packed32.java (working copy) @@ -222,7 +222,6 @@ } public long ramBytesUsed() { - return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER - + blocks.length * RamUsageEstimator.NUM_BYTES_INT; + return RamUsageEstimator.sizeOf(blocks); } } Index: lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java (revision 1301918) +++ lucene/core/src/java/org/apache/lucene/util/packed/Packed64.java (working copy) @@ -207,8 +207,7 @@ } public long ramBytesUsed() { - return RamUsageEstimator.NUM_BYTES_ARRAY_HEADER - + blocks.length * RamUsageEstimator.NUM_BYTES_LONG; + return RamUsageEstimator.sizeOf(blocks); } public void clear() { Index: lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java (revision 1301918) +++ lucene/core/src/java/org/apache/lucene/util/RamUsageEstimator.java (working copy) @@ -17,18 +17,20 @@ * limitations under the License. */ +import java.lang.management.ManagementFactory; import java.lang.reflect.*; import java.text.DecimalFormat; import java.util.*; /** - * Estimates the size of a given Object using a given MemoryModel for primitive - * size information. + * Estimates the size of a given Object using a simple memory model + * for primitive size information. * * Resource Usage: * - * Internally uses a Map to temporally hold a reference to every - * object seen. + * Internally uses a Set to temporally hold a reference to every + * object seen. + * This class is not thread safe, use one instance for each thread! * * If checkInterned, all Strings checked will be interned, but those * that were not already interned will be released for GC when the @@ -38,32 +40,194 @@ */ public final class RamUsageEstimator { + public final static int NUM_BYTES_BOOLEAN = 1; + public final static int NUM_BYTES_BYTE = 1; + public final static int NUM_BYTES_CHAR = 2; public final static int NUM_BYTES_SHORT = 2; public final static int NUM_BYTES_INT = 4; + public final static int NUM_BYTES_FLOAT = 4; public final static int NUM_BYTES_LONG = 8; - public final static int NUM_BYTES_FLOAT = 4; public final static int NUM_BYTES_DOUBLE = 8; - public final static int NUM_BYTES_CHAR = 2; - public final static int NUM_BYTES_OBJECT_HEADER = 8; - public final static int NUM_BYTES_OBJECT_REF = Constants.JRE_IS_64BIT ? 8 : 4; - public final static int NUM_BYTES_ARRAY_HEADER = NUM_BYTES_OBJECT_HEADER + NUM_BYTES_INT + NUM_BYTES_OBJECT_REF; - private MemoryModel memoryModel; + public final static int NUM_BYTES_OBJECT_REF; + + public final static int NUM_BYTES_OBJECT_HEADER; + public final static int NUM_BYTES_ARRAY_HEADER; + + public final static int NUM_BYTES_OBJECT_ALIGNMENT; + + private static final Map,Integer> primitiveSizes = + new IdentityHashMap,Integer>(); + + private final static Object theUnsafe; + private final static Method objectFieldOffsetMethod; + private final static boolean useUnsafe; + static { + primitiveSizes.put(boolean.class, Integer.valueOf(NUM_BYTES_BOOLEAN)); + primitiveSizes.put(byte.class, Integer.valueOf(NUM_BYTES_BYTE)); + primitiveSizes.put(char.class, Integer.valueOf(NUM_BYTES_CHAR)); + primitiveSizes.put(short.class, Integer.valueOf(NUM_BYTES_SHORT)); + primitiveSizes.put(int.class, Integer.valueOf(NUM_BYTES_INT)); + primitiveSizes.put(float.class, Integer.valueOf(NUM_BYTES_FLOAT)); + primitiveSizes.put(double.class, Integer.valueOf(NUM_BYTES_DOUBLE)); + primitiveSizes.put(long.class, Integer.valueOf(NUM_BYTES_LONG)); - private final Map seen; + int referenceSize = Constants.JRE_IS_64BIT ? 8 : 4; + int objectHeader = Constants.JRE_IS_64BIT ? 16 : 8; + // the following is objectHeader + NUM_BYTES_INT, but aligned to platform address size: + int arrayHeader = Constants.JRE_IS_64BIT ? 24 : 12; + Object unsafe = null; + Method objectFieldOffsetM = null; + try { + final Class unsafeClass = Class.forName("sun.misc.Unsafe"); + final Field unsafeField = unsafeClass.getDeclaredField("theUnsafe"); + unsafeField.setAccessible(true); + unsafe = unsafeField.get(null); + + // get object reference size by getting scale factor of Object[] arrays: + try { + final Method arrayIndexScaleM = unsafeClass.getMethod("arrayIndexScale", Class.class); + referenceSize = ((Number) arrayIndexScaleM.invoke(unsafe, Object[].class)).intValue(); + } catch (Exception e) { + // ignore + } + + // updated best guess based on reference size: + objectHeader = Constants.JRE_IS_64BIT ? (8 + referenceSize) : 8; + arrayHeader = Constants.JRE_IS_64BIT ? (8 + 2 * referenceSize) : 12; + + // get the object header size: + // - first try out if the field offsets are not scaled (see warning in Unsafe docs) + // - get the object header size by getting the field offset of the first field of a dummy object + // If the scaling is byte-wise and unsafe is available, enable dynamic size measurement for + // estimateRamUsage(). + try { + objectFieldOffsetM = unsafeClass.getMethod("objectFieldOffset", Field.class); + final Field dummy1Field = DummyTwoLongObject.class.getDeclaredField("dummy1"); + final int ofs1 = ((Number) objectFieldOffsetM.invoke(unsafe, dummy1Field)).intValue(); + final Field dummy2Field = DummyTwoLongObject.class.getDeclaredField("dummy2"); + final int ofs2 = ((Number) objectFieldOffsetM.invoke(unsafe, dummy2Field)).intValue(); + if (Math.abs(ofs2 - ofs1) == NUM_BYTES_LONG) { + final Field baseField = DummyOneFieldObject.class.getDeclaredField("base"); + objectHeader = ((Number) objectFieldOffsetM.invoke(unsafe, baseField)).intValue(); + } else { + // it is not safe to use Unsafe.objectFieldOffset(), + // as it may be scaled (see "cookie" comment in Unsafe), better use defaults + // and conventional size estimation: + objectFieldOffsetM = null; + } + } catch (Exception e) { + // on exception ensure useUnsafe will be set to false later: + objectFieldOffsetM = null; + } + + // get the array header size by retrieving the array base offset + // (offset of the first element of an array): + try { + final Method arrayBaseOffsetM = unsafeClass.getMethod("arrayBaseOffset", Class.class); + // we calculate that only for byte[] arrays, it's actually the same for all types: + arrayHeader = ((Number) arrayBaseOffsetM.invoke(unsafe, byte[].class)).intValue(); + } catch (Exception e) { + // ignore + } + } catch (Exception e) { + // ignore + } + NUM_BYTES_OBJECT_REF = referenceSize; + NUM_BYTES_OBJECT_HEADER = objectHeader; + NUM_BYTES_ARRAY_HEADER = arrayHeader; + useUnsafe = (unsafe != null && objectFieldOffsetM != null); + if (useUnsafe) { + theUnsafe = unsafe; + objectFieldOffsetMethod = objectFieldOffsetM; + } else { + theUnsafe = objectFieldOffsetMethod = null; + } + + // try to get the object alignment (default is 8): + int objectAlignment = 8; + try { + final Class beanClazz = Class.forName("com.sun.management.HotSpotDiagnosticMXBean"); + final Object hotSpotBean = ManagementFactory.newPlatformMXBeanProxy( + ManagementFactory.getPlatformMBeanServer(), + "com.sun.management:type=HotSpotDiagnostic", + beanClazz + ); + final Method getVMOptionMethod = beanClazz.getMethod("getVMOption", String.class); + final Object vmOption = getVMOptionMethod.invoke(hotSpotBean, "ObjectAlignmentInBytes"); + objectAlignment = Integer.parseInt( + vmOption.getClass().getMethod("getValue") + .invoke(vmOption).toString() + ); + } catch (Exception e) { + // ignore + } + NUM_BYTES_OBJECT_ALIGNMENT = objectAlignment; + }; - private int refSize; - private int arraySize; - private int classSize; + // Object with just one field to determine the object header size by getting the offset of the dummy field: + @SuppressWarnings("unused") + private static final class DummyOneFieldObject { + public byte base; + } - private boolean checkInterned; + // Another test object for checking, if the difference in offsets of dummy1 and dummy2 is 8 bytes. + // Only then we can be sure that those are real, unscaled offsets: + @SuppressWarnings("unused") + private static final class DummyTwoLongObject { + public long dummy1, dummy2; + } + /** Aligns an object size to be the next multiple of {@link #NUM_BYTES_OBJECT_ALIGNMENT}. */ + public static long alignObjectSize(long size) { + size += (long) NUM_BYTES_OBJECT_ALIGNMENT - 1L; + return size - (size % NUM_BYTES_OBJECT_ALIGNMENT); + } + + /** Returns the size in bytes of the byte[] object. */ + public static long sizeOf(byte[] arr) { + return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + arr.length); + } + + /** Returns the size in bytes of the char[] object. */ + public static long sizeOf(char[] arr) { + return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_CHAR * arr.length); + } + + /** Returns the size in bytes of the short[] object. */ + public static long sizeOf(short[] arr) { + return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_SHORT * arr.length); + } + + /** Returns the size in bytes of the int[] object. */ + public static long sizeOf(int[] arr) { + return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_INT * arr.length); + } + + /** Returns the size in bytes of the float[] object. */ + public static long sizeOf(float[] arr) { + return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_FLOAT * arr.length); + } + + /** Returns the size in bytes of the long[] object. */ + public static long sizeOf(long[] arr) { + return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_LONG * arr.length); + } + + /** Returns the size in bytes of the double[] object. */ + public static long sizeOf(double[] arr) { + return alignObjectSize((long) NUM_BYTES_ARRAY_HEADER + (long) NUM_BYTES_DOUBLE * arr.length); + } + + private final Set seen = Collections.newSetFromMap(new IdentityHashMap(64)); + private final boolean checkInterned; + /** * Constructs this object with an AverageGuessMemoryModel and * checkInterned = true. */ public RamUsageEstimator() { - this(new AverageGuessMemoryModel()); + this(true); } /** @@ -73,113 +237,111 @@ * intern checking. */ public RamUsageEstimator(boolean checkInterned) { - this(new AverageGuessMemoryModel(), checkInterned); + this.checkInterned = checkInterned; } - /** - * @param memoryModel MemoryModel to use for primitive object sizes. + /** Estimates the RAM usage by the given object. It will + * walk the object tree and sum up all referenced objects. */ - public RamUsageEstimator(MemoryModel memoryModel) { - this(memoryModel, true); + public long estimateRamUsage(Object obj) { + try { + return size(obj, true); + } finally { + seen.clear(); + } } - /** - * @param memoryModel MemoryModel to use for primitive object sizes. - * @param checkInterned check if Strings are interned and don't add to size - * if they are. Defaults to true but if you know the objects you are checking - * won't likely contain many interned Strings, it will be faster to turn off - * intern checking. + /** Estimates the RAM usage by the given object. It will + * not count the RAM usage of referenced objects. */ - public RamUsageEstimator(MemoryModel memoryModel, boolean checkInterned) { - this.memoryModel = memoryModel; - this.checkInterned = checkInterned; - // Use Map rather than Set so that we can use an IdentityHashMap - not - // seeing an IdentityHashSet - seen = new IdentityHashMap(64); - this.refSize = memoryModel.getReferenceSize(); - this.arraySize = memoryModel.getArraySize(); - this.classSize = memoryModel.getClassSize(); + public long estimateShallowRamUsage(Object obj) { + try { + return size(obj, false); + } finally { + seen.clear(); + } } - public long estimateRamUsage(Object obj) { - long size = size(obj); - seen.clear(); - return size; - } - - private long size(Object obj) { + private long size(Object obj, boolean deep) { if (obj == null) { return 0; } + // interned not part of this object if (checkInterned && obj instanceof String - && obj == ((String) obj).intern()) { // interned string will be eligible - // for GC on - // estimateRamUsage(Object) return + && obj == ((String) obj).intern()) { + // interned string will be eligible + // for GC on estimateRamUsage(Object) return return 0; } // skip if we have seen before - if (seen.containsKey(obj)) { + if (seen.contains(obj)) { return 0; } // add to seen - seen.put(obj, null); + seen.add(obj); Class clazz = obj.getClass(); if (clazz.isArray()) { - return sizeOfArray(obj); + return sizeOfArray(obj, deep); } - long size = 0; - + long size = (long) NUM_BYTES_OBJECT_HEADER, innerSize = 0L; + // walk type hierarchy while (clazz != null) { Field[] fields = clazz.getDeclaredFields(); - for (int i = 0; i < fields.length; i++) { - if (Modifier.isStatic(fields[i].getModifiers())) { + for (final Field f : fields) { + if (Modifier.isStatic(f.getModifiers())) { continue; } - if (fields[i].getType().isPrimitive()) { - size += memoryModel.getPrimitiveSize(fields[i].getType()); - } else { - size += refSize; - fields[i].setAccessible(true); - try { - Object value = fields[i].get(obj); - if (value != null) { - size += size(value); - } - } catch (IllegalAccessException ex) { - // ignore for now? + try { + final Class type = f.getType(); + final int fsize = type.isPrimitive() ? primitiveSizes.get(type) : NUM_BYTES_OBJECT_REF; + if (useUnsafe) { + final long offsetPlusSize = + ((Number) objectFieldOffsetMethod.invoke(theUnsafe, f)).longValue() + fsize; + size = Math.max(size, offsetPlusSize); + } else { + size += fsize; } + if (deep && !type.isPrimitive()) { + f.setAccessible(true); + innerSize += size(f.get(obj), deep); + } + } catch (IllegalAccessException ex) { + // ignore for now? + } catch (InvocationTargetException ite) { + // ignore for now? } - } clazz = clazz.getSuperclass(); } - size += classSize; - return size; + return alignObjectSize(size) + innerSize; } - private long sizeOfArray(Object obj) { - int len = Array.getLength(obj); - if (len == 0) { - return 0; - } - long size = arraySize; - Class arrayElementClazz = obj.getClass().getComponentType(); - if (arrayElementClazz.isPrimitive()) { - size += len * memoryModel.getPrimitiveSize(arrayElementClazz); - } else { - for (int i = 0; i < len; i++) { - size += refSize + size(Array.get(obj, i)); + private long sizeOfArray(Object obj, boolean deep) { + long size = NUM_BYTES_ARRAY_HEADER; + final int len = Array.getLength(obj); + if (len > 0) { + Class arrayElementClazz = obj.getClass().getComponentType(); + if (arrayElementClazz.isPrimitive()) { + size += (long) len * primitiveSizes.get(arrayElementClazz); + } else { + if (deep) { + for (int i = 0; i < len; i++) { + size += (long) NUM_BYTES_OBJECT_REF + size(Array.get(obj, i), deep); + } + } else { + size += (long) len * NUM_BYTES_OBJECT_REF; + } } } - return size; + return alignObjectSize(size); } private static final long ONE_KB = 1024; Index: lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java =================================================================== --- lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java (revision 1301918) +++ lucene/core/src/test/org/apache/lucene/util/TestRamUsageEstimator.java (working copy) @@ -17,24 +17,43 @@ * limitations under the License. */ -import org.apache.lucene.util.LuceneTestCase; - public class TestRamUsageEstimator extends LuceneTestCase { public void testBasic() { - RamUsageEstimator rue = new RamUsageEstimator(); - rue.estimateRamUsage("test str"); + final RamUsageEstimator rue = new RamUsageEstimator(); + // interned strings + assertEquals(0, rue.estimateRamUsage("test str")); + assertEquals(0, rue.estimateShallowRamUsage("test str")); - rue.estimateRamUsage("test strin"); + // not interned strings + assertTrue(rue.estimateRamUsage(new String("test strin")) > + rue.estimateShallowRamUsage(new String("test strin"))); Holder holder = new Holder(); holder.holder = new Holder("string2", 5000L); - rue.estimateRamUsage(holder); + assertTrue(rue.estimateRamUsage(holder) > rue.estimateShallowRamUsage(holder)); + assertEquals(rue.estimateShallowRamUsage(holder), rue.estimateShallowRamUsage(holder.holder)); + assertTrue(rue.estimateRamUsage(holder) > rue.estimateRamUsage(holder.holder)); String[] strings = new String[]{new String("test strin"), new String("hollow"), new String("catchmaster")}; - rue.estimateRamUsage(strings); + assertTrue(rue.estimateRamUsage(strings) > rue.estimateShallowRamUsage(strings)); } + public void testReferenceSize() { + if (VERBOSE) { + System.out.println("This JVM is 64bit: " + Constants.JRE_IS_64BIT); + System.out.println("Reference size in this JVM: " + RamUsageEstimator.NUM_BYTES_OBJECT_REF); + System.out.println("Object header size in this JVM: " + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER); + System.out.println("Array header size in this JVM: " + RamUsageEstimator.NUM_BYTES_ARRAY_HEADER); + System.out.println("Object alignment in this JVM: " + RamUsageEstimator.NUM_BYTES_OBJECT_ALIGNMENT); + } + assertTrue(RamUsageEstimator.NUM_BYTES_OBJECT_REF == 4 || RamUsageEstimator.NUM_BYTES_OBJECT_REF == 8); + if (!Constants.JRE_IS_64BIT) { + assertEquals("For 32bit JVMs, reference size must always be 4", 4, RamUsageEstimator.NUM_BYTES_OBJECT_REF); + } + } + + @SuppressWarnings("unused") private static final class Holder { long field1 = 5000L; String name = "name";