Index: src/test/org/apache/lucene/search/TestSort.java =================================================================== --- src/test/org/apache/lucene/search/TestSort.java (revision 798925) +++ src/test/org/apache/lucene/search/TestSort.java (working copy) @@ -320,16 +320,26 @@ } - // test sorts where the type of field is specified and a custom field parser is used, that - // uses a simple char encoding. The sorted string contains a character beginning from 'A' that - // is mapped to a numeric value using some "funny" algorithm to be different for each data type. + /** + * test sorts where the type of field is specified and a custom field parser + * is used, that uses a simple char encoding. The sorted string contains a + * character beginning from 'A' that is mapped to a numeric value using some + * "funny" algorithm to be different for each data type. + */ public void testCustomFieldParserSort() throws Exception { + // since tests explicilty uses different parsers on the same fieldname + // we explicitly check/purge the FieldCache between each assertMatch + FieldCache fc = FieldCache.DEFAULT; + + sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.IntParser(){ public final int parseInt(final String val) { return (val.charAt(0)-'A') * 123456; } }), SortField.FIELD_DOC }); assertMatches (full, queryA, sort, "JIHGFEDCBA"); + assertSaneFieldCaches(getName() + " IntParser"); + fc.purgeAllCaches(); sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.FloatParser(){ public final float parseFloat(final String val) { @@ -337,6 +347,8 @@ } }), SortField.FIELD_DOC }); assertMatches (full, queryA, sort, "JIHGFEDCBA"); + assertSaneFieldCaches(getName() + " FloatParser"); + fc.purgeAllCaches(); sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.LongParser(){ public final long parseLong(final String val) { @@ -344,6 +356,8 @@ } }), SortField.FIELD_DOC }); assertMatches (full, queryA, sort, "JIHGFEDCBA"); + assertSaneFieldCaches(getName() + " LongParser"); + fc.purgeAllCaches(); sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.DoubleParser(){ public final double parseDouble(final String val) { @@ -351,6 +365,8 @@ } }), SortField.FIELD_DOC }); assertMatches (full, queryA, sort, "JIHGFEDCBA"); + assertSaneFieldCaches(getName() + " DoubleParser"); + fc.purgeAllCaches(); sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.ByteParser(){ public final byte parseByte(final String val) { @@ -358,6 +374,8 @@ } }), SortField.FIELD_DOC }); assertMatches (full, queryA, sort, "JIHGFEDCBA"); + assertSaneFieldCaches(getName() + " ByteParser"); + fc.purgeAllCaches(); sort.setSort (new SortField[] { new SortField ("parser", new FieldCache.ShortParser(){ public final short parseShort(final String val) { @@ -365,6 +383,8 @@ } }), SortField.FIELD_DOC }); assertMatches (full, queryA, sort, "JIHGFEDCBA"); + assertSaneFieldCaches(getName() + " ShortParser"); + fc.purgeAllCaches(); } // test sorts when there's nothing in the index @@ -934,12 +954,6 @@ sort.setSort("string", true); assertMatches(multi, queryA, sort, "CBEFGHIAJD"); - sort.setSort(new SortField[] { new SortField ("string", Locale.US) }); - assertMatches(multi, queryA, sort, "DJAIHGFEBC"); - - sort.setSort(new SortField[] { new SortField ("string", Locale.US, true) }); - assertMatches(multi, queryA, sort, "CBEFGHIAJD"); - sort.setSort(new String[] {"int","float"}); assertMatches(multi, queryA, sort, "IDHFGJEABC"); @@ -960,6 +974,22 @@ sort.setSort("string", true); assertMatches(multi, queryF, sort, "IJZ"); + + // up to this point, all of the searches should have "sane" + // FieldCache behavior, and should have reused hte cache in several cases + assertSaneFieldCaches(getName() + " "); + // next we'll check an alternate Locale for string, so purge first + FieldCache.DEFAULT.purgeAllCaches(); + + sort.setSort(new SortField[] { new SortField ("string", Locale.US) }); + assertMatches(multi, queryA, sort, "DJAIHGFEBC"); + + sort.setSort(new SortField[] { new SortField ("string", Locale.US, true) }); + assertMatches(multi, queryA, sort, "CBEFGHIAJD"); + + assertSaneFieldCaches(getName() + " "); + FieldCache.DEFAULT.purgeAllCaches(); + } // make sure the documents returned by the search match the expected list Index: src/test/org/apache/lucene/search/TestStressSort.java =================================================================== --- src/test/org/apache/lucene/search/TestStressSort.java (revision 798925) +++ src/test/org/apache/lucene/search/TestStressSort.java (working copy) @@ -107,9 +107,13 @@ doc.add(doubleField); doc2.add(doubleField); + // we use two diff string fields so our FieldCache usage + // is less suspicious to cache inspection final Field stringField = new Field("string", "", Field.Store.NO, Field.Index.NOT_ANALYZED); doc.add(stringField); - // doc2 doesn't have stringField, so we get nulls + final Field stringFieldIdx = new Field("stringIdx", "", Field.Store.NO, Field.Index.NOT_ANALYZED); + doc.add(stringFieldIdx); + // doc2 doesn't have stringField or stringFieldIdx, so we get nulls for(int i=0;itearDown() in your unit test, make sure you * call super.setUp() and * super.tearDown(). + * */ - public abstract class LuceneTestCase extends TestCase { public LuceneTestCase() { @@ -46,14 +64,92 @@ ConcurrentMergeScheduler.setTestMode(); } + /** + * Forcible purges all cache entries from the FieldCache. + *

+ * This method will be called by tearDown to clean up FieldCache.DEFAULT. + * If a (poorly written) test has some expectation that the FieldCache + * will persist across test methods (ie: a static IndexReader) this + * method can be overridden to do nothing. + *

+ * @see FieldCache#purgeAllCaches() + */ + protected void purgeFieldCache(final FieldCache fc) { + fc.purgeAllCaches(); + } + protected void tearDown() throws Exception { - if (ConcurrentMergeScheduler.anyUnhandledExceptions()) { - // Clear the failure so that we don't just keep - // failing subsequent test cases - ConcurrentMergeScheduler.clearUnhandledExceptions(); - fail("ConcurrentMergeScheduler hit unhandled exceptions"); + try { + // this isn't as useful as calling directly from the scope where the + // index readers are used, because they could be gc'ed just before + // tearDown is called. + // But it's better then nothing. + assertSaneFieldCaches(getClass().getName() + "." + getName()); + + // :TODO: should we also add explicit calls to assertSaneFieldCaches in every test method where we expect FieldCache usage (in the scope of the IndexReader?) + + if (ConcurrentMergeScheduler.anyUnhandledExceptions()) { + // Clear the failure so that we don't just keep + // failing subsequent test cases + ConcurrentMergeScheduler.clearUnhandledExceptions(); + fail("ConcurrentMergeScheduler hit unhandled exceptions"); + } + } finally { + purgeFieldCache(FieldCache.DEFAULT); } } + + /** :TODO: add good javadocs */ + protected void assertSaneFieldCaches(final String msg) { + final Set entries = FieldCache.DEFAULT.getCacheEntries(); + Collection losers = entries; + try { + // after each check, set losers to entries so any Exceptions + // thrown will trigger a dump of the whole FieldCache. + + losers = FieldCacheSanityChecker.checkFieldCacheTypeSanity(entries); + assertTrue(msg + ": multiple FieldCaches for same " + + "reader/fieldname with diff types", + losers.isEmpty()); + losers = entries; + + losers = FieldCacheSanityChecker.checkFieldCacheSubReaderSanity(entries); + assertTrue(msg + ": multiple FieldCaches for same " + + "fieldname with parent reader", + losers.isEmpty()); + losers = entries; + + // :TODO: add more tests here + + losers = null; // made it through fine + } finally { + + // report this in the event of any exception/failure + // if no failure, then losers will be empty anyway. + if (null != losers && !losers.isEmpty()) { + dumpIterator(msg + ": FieldCache" + + (losers == entries ? "" : " Losers"), + losers.iterator(), System.err); + +// // :HACK: BEGIN +// } else { +// if (!entries.isEmpty()) +// dumpIterator(getName() + " FIELD CACHE ------------------- CLEAN CLEAN CLEAN CLEAN", +// entries.iterator(), System.err); +// // :HACK: END + } + + } + } + + public static void dumpIterator(String label, Iterator iter, + PrintStream stream) { + stream.println("*** BEGIN "+label+" ***"); + while (iter.hasNext()) { + stream.println(iter.next().toString()); + } + stream.println("*** END "+label+" ***"); + } /** * Returns a {@link Random} instance for generating random numbers during the test. Index: src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java =================================================================== --- src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java (revision 0) +++ src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java (revision 0) @@ -0,0 +1,154 @@ +package org.apache.lucene.util; + +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; + +import java.io.IOException; + +public class TestFieldCacheSanityChecker extends LuceneTestCase { + + protected IndexReader readerA; + protected IndexReader readerB; + protected IndexReader readerX; + + private static final int NUM_DOCS = 1000; + + protected void setUp() throws Exception { + super.setUp(); + + RAMDirectory dirA = new RAMDirectory(); + RAMDirectory dirB = new RAMDirectory(); + + IndexWriter wA = new IndexWriter(dirA, new WhitespaceAnalyzer(), true, + IndexWriter.MaxFieldLength.LIMITED); + IndexWriter wB = new IndexWriter(dirB, new WhitespaceAnalyzer(), true, + IndexWriter.MaxFieldLength.LIMITED); + + long theLong = Long.MAX_VALUE; + double theDouble = Double.MAX_VALUE; + byte theByte = Byte.MAX_VALUE; + short theShort = Short.MAX_VALUE; + int theInt = Integer.MAX_VALUE; + float theFloat = Float.MAX_VALUE; + for (int i = 0; i < NUM_DOCS; i++){ + Document doc = new Document(); + doc.add(new Field("theLong", String.valueOf(theLong--), Field.Store.NO, Field.Index.NOT_ANALYZED)); + doc.add(new Field("theDouble", String.valueOf(theDouble--), Field.Store.NO, Field.Index.NOT_ANALYZED)); + doc.add(new Field("theByte", String.valueOf(theByte--), Field.Store.NO, Field.Index.NOT_ANALYZED)); + doc.add(new Field("theShort", String.valueOf(theShort--), Field.Store.NO, Field.Index.NOT_ANALYZED)); + doc.add(new Field("theInt", String.valueOf(theInt--), Field.Store.NO, Field.Index.NOT_ANALYZED)); + doc.add(new Field("theFloat", String.valueOf(theFloat--), Field.Store.NO, Field.Index.NOT_ANALYZED)); + if (0 == i % 3) { + wA.addDocument(doc); + } else { + wB.addDocument(doc); + } + } + wA.close(); + wB.close(); + readerA = IndexReader.open(dirA); + readerB = IndexReader.open(dirB); + readerX = new MultiReader(new IndexReader[] { readerA, readerB }); + } + + public void tearDown() throws Exception { + super.tearDown(); + readerA.close(); + readerB.close(); + readerX.close(); + } + + public void testSanity() throws IOException { + FieldCache cache = FieldCache.DEFAULT; + cache.purgeAllCaches(); + + double [] doubles; + int [] ints; + + doubles = cache.getDoubles(readerA, "theDouble"); + doubles = cache.getDoubles(readerA, "theDouble", + FieldCache.DEFAULT_DOUBLE_PARSER); + doubles = cache.getDoubles(readerB, "theDouble", + FieldCache.DEFAULT_DOUBLE_PARSER); + + ints = cache.getInts(readerX, "theInt"); + ints = cache.getInts(readerX, "theInt", + FieldCache.DEFAULT_INT_PARSER); + + assertEquals("shouldn't be any subreader cache errors", + 0, FieldCacheSanityChecker.checkFieldCacheSubReaderSanity + (cache.getCacheEntries()).size()); + assertEquals("shouldn't be any type cache errors", + 0, FieldCacheSanityChecker.checkFieldCacheTypeSanity + (cache.getCacheEntries()).size()); + + } + + public void testInsanity1() throws IOException { + FieldCache cache = FieldCache.DEFAULT; + cache.purgeAllCaches(); + + int [] ints; + String [] strings; + byte [] bytes; + + ints = cache.getInts(readerX, "theInt", FieldCache.DEFAULT_INT_PARSER); + strings = cache.getStrings(readerX, "theInt"); + + // this one is ok + bytes = cache.getBytes(readerX, "theByte"); + + assertEquals("wrong number of type cache errors", + 2, FieldCacheSanityChecker.checkFieldCacheTypeSanity + (cache.getCacheEntries()).size()); + + // we expect bad things, don't let teraDown complain about them + cache.purgeAllCaches(); + } + + public void testInsanity2() throws IOException { + FieldCache cache = FieldCache.DEFAULT; + cache.purgeAllCaches(); + + String [] strings; + byte [] bytes; + + strings = cache.getStrings(readerA, "theString"); + strings = cache.getStrings(readerB, "theString"); + strings = cache.getStrings(readerX, "theString"); + + // this one is ok + bytes = cache.getBytes(readerX, "theByte"); + + assertEquals("wrong number of subreader cache errors", + 3, FieldCacheSanityChecker.checkFieldCacheSubReaderSanity + (cache.getCacheEntries()).size()); + + // we expect bad things, don't let teraDown complain about them + cache.purgeAllCaches(); + } + +} Property changes on: src/test/org/apache/lucene/util/TestFieldCacheSanityChecker.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: src/test/org/apache/lucene/util/TestRamCalculator.java =================================================================== --- src/test/org/apache/lucene/util/TestRamCalculator.java (revision 0) +++ src/test/org/apache/lucene/util/TestRamCalculator.java (revision 0) @@ -0,0 +1,40 @@ +package org.apache.lucene.util; + +import junit.framework.TestCase; + +public class TestRamCalculator extends TestCase { + + public void testBasic() { + String string = new String("test str"); + RamUsageEstimator rue = new RamUsageEstimator(); + long size = rue.estimateRamUsage(string); + System.out.println("size:" + size); + + string = new String("test strin"); + size = rue.estimateRamUsage(string); + System.out.println("size:" + size); + + Holder holder = new Holder(); + holder.holder = new Holder("string2", 5000L); + size = rue.estimateRamUsage(holder); + System.out.println("size:" + size); + + String[] strings = new String[]{new String("test strin"), new String("hollow"), new String("catchmaster")}; + size = rue.estimateRamUsage(strings); + System.out.println("size:" + size); + } + + private static final class Holder { + long field1 = 5000L; + String name = "name"; + Holder holder; + + Holder() { + } + + Holder(String name, long field1) { + this.name = name; + this.field1 = field1; + } + } +} Property changes on: src/test/org/apache/lucene/util/TestRamCalculator.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: src/java/org/apache/lucene/search/FieldCache.java =================================================================== --- src/java/org/apache/lucene/search/FieldCache.java (revision 798925) +++ src/java/org/apache/lucene/search/FieldCache.java (working copy) @@ -19,12 +19,17 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.util.NumericUtils; +import org.apache.lucene.util.RamUsageEstimator; import org.apache.lucene.document.NumericField; // for javadocs import org.apache.lucene.analysis.NumericTokenStream; // for javadocs import java.io.IOException; import java.io.Serializable; +import java.text.DecimalFormat; +import java.util.Locale; +import java.util.Set; + /** * Expert: Maintains caches of term values. * @@ -35,6 +40,10 @@ */ public interface FieldCache { + public static final class CreationPlaceholder { + Object value; + } + /** Indicator for StringIndex values in the cache. */ // NOTE: the value assigned to this constant must not be // the same as any of those in SortField!! @@ -146,6 +155,9 @@ protected Object readResolve() { return DEFAULT_BYTE_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_BYTE_PARSER"; + } }; /** The default parser for short values, which are encoded by {@link Short#toString(short)} */ @@ -156,6 +168,9 @@ protected Object readResolve() { return DEFAULT_SHORT_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_SHORT_PARSER"; + } }; /** The default parser for int values, which are encoded by {@link Integer#toString(int)} */ @@ -166,6 +181,9 @@ protected Object readResolve() { return DEFAULT_INT_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_INT_PARSER"; + } }; /** The default parser for float values, which are encoded by {@link Float#toString(float)} */ @@ -176,6 +194,9 @@ protected Object readResolve() { return DEFAULT_FLOAT_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_FLOAT_PARSER"; + } }; /** The default parser for long values, which are encoded by {@link Long#toString(long)} */ @@ -186,6 +207,9 @@ protected Object readResolve() { return DEFAULT_LONG_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_LONG_PARSER"; + } }; /** The default parser for double values, which are encoded by {@link Double#toString(double)} */ @@ -196,6 +220,9 @@ protected Object readResolve() { return DEFAULT_DOUBLE_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_DOUBLE_PARSER"; + } }; /** @@ -212,6 +239,9 @@ protected Object readResolve() { return NUMERIC_UTILS_INT_PARSER; } + public String toString() { + return FieldCache.class.getName()+".NUMERIC_UTILS_INT_PARSER"; + } }; /** @@ -228,6 +258,9 @@ protected Object readResolve() { return NUMERIC_UTILS_FLOAT_PARSER; } + public String toString() { + return FieldCache.class.getName()+".NUMERIC_UTILS_FLOAT_PARSER"; + } }; /** @@ -244,6 +277,9 @@ protected Object readResolve() { return NUMERIC_UTILS_LONG_PARSER; } + public String toString() { + return FieldCache.class.getName()+".NUMERIC_UTILS_LONG_PARSER"; + } }; /** @@ -260,6 +296,9 @@ protected Object readResolve() { return NUMERIC_UTILS_DOUBLE_PARSER; } + public String toString() { + return FieldCache.class.getName()+".NUMERIC_UTILS_DOUBLE_PARSER"; + } }; /** Checks the internal cache for an appropriate entry, and if none is @@ -477,5 +516,47 @@ */ public Comparable[] getCustom (IndexReader reader, String field, SortComparator comparator) throws IOException; + + /** + * A unique Identifier for each itme in the FieldCache + */ + public static abstract class CacheEntry { + public abstract Object getReaderKey(); + public abstract String getFieldName(); + public abstract Class getCacheType(); + /** :TODO: is this still needed? nothing seems to use it? */ + public abstract int getSortFieldType(); + public abstract Object getCustom(); + public abstract Locale getLocale(); + public abstract Object getValue(); + public String guessSize() { + RamUsageEstimator ramCalc = new RamUsageEstimator(128); + long size = ramCalc.estimateRamUsage(getValue()); + return RamUsageEstimator.humanReadableUnits(size, new DecimalFormat("0")); + } + + public String toString() { + StringBuffer b = new StringBuffer(); + b.append("'").append(getReaderKey()).append("'=>"); + b.append("'").append(getFieldName()).append("',"); + b.append(getCacheType()).append(",").append(getSortFieldType()); + b.append(",").append(getCustom()).append(",").append(getLocale()); + b.append("=>").append(getValue().getClass().getName()).append("#"); + b.append(System.identityHashCode(getValue())); + + // :TODO: remove from toString() + b.append(" size guess:").append(guessSize()); + + return b.toString(); + } + } + + /** :TODO: javadocs, expert */ + public abstract Set getCacheEntries(); + + /** :TODO: javadocs, expert */ + public abstract void purgeAllCaches(); + + } Index: src/java/org/apache/lucene/search/FieldSortedHitQueue.java =================================================================== --- src/java/org/apache/lucene/search/FieldSortedHitQueue.java (revision 798925) +++ src/java/org/apache/lucene/search/FieldSortedHitQueue.java (working copy) @@ -180,7 +180,7 @@ * caches comparators instead of term values. */ static final FieldCacheImpl.Cache Comparators = new FieldCacheImpl.Cache() { - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, FieldCacheImpl.Entry entryKey) throws IOException { FieldCacheImpl.Entry entry = (FieldCacheImpl.Entry) entryKey; String fieldname = entry.field; Index: src/java/org/apache/lucene/search/FieldCacheImpl.java =================================================================== --- src/java/org/apache/lucene/search/FieldCacheImpl.java (revision 798925) +++ src/java/org/apache/lucene/search/FieldCacheImpl.java (working copy) @@ -23,9 +23,12 @@ import org.apache.lucene.index.TermEnum; import java.io.IOException; +import java.util.Iterator; +import java.util.HashSet; import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.WeakHashMap; /** @@ -40,6 +43,85 @@ // TODO: change interface to FieldCache in 3.0 when removed class FieldCacheImpl implements ExtendedFieldCache { + private Map caches; + FieldCacheImpl() { + init(); + } + private synchronized void init() { + caches = new HashMap(7); + caches.put(Byte.TYPE, new ByteCache(this)); + caches.put(Short.TYPE, new ShortCache(this)); + caches.put(Integer.TYPE, new IntCache(this)); + caches.put(Float.TYPE, new FloatCache(this)); + caches.put(Long.TYPE, new LongCache(this)); + caches.put(Double.TYPE, new DoubleCache(this)); + caches.put(String.class, new StringCache(this)); + caches.put(StringIndex.class, new StringIndexCache(this)); + caches.put(Comparable.class, new CustomCache(this)); + caches.put(Object.class, new AutoCache(this)); + } + + public void purgeAllCaches() { + init(); + } + + public Set getCacheEntries() { + Set result = new HashSet(17); + Iterator outerKeys = caches.keySet().iterator(); + while (outerKeys.hasNext()) { + Class cacheType = (Class)outerKeys.next(); + Cache cache = (Cache)caches.get(cacheType); + Iterator innerKeys = cache.readerCache.keySet().iterator(); + while (innerKeys.hasNext()) { + // we've now materialized a hard ref + Object readerKey = innerKeys.next(); + // innerKeys was backed by WeakHashMap, sanity check + // that it wasn't GCed before we made hard ref + if (null != readerKey && cache.readerCache.containsKey(readerKey)) { + Map innerCache = ((Map)cache.readerCache.get(readerKey)); + Iterator keys = innerCache.keySet().iterator(); + while (keys.hasNext()) { + Entry entry = (Entry) keys.next(); + result.add(new CacheEntryImpl(readerKey, entry.field, + cacheType, entry.type, + entry.custom, entry.locale, + innerCache.get(entry))); + } + } + } + } + return result; + } + + private static final class CacheEntryImpl extends CacheEntry { + private final Object readerKey; + private final String fieldName; + private final Class cacheType; + /** :TODO: is this still needed? nothing seems to use it? */ + private final int sortFieldType; + private final Object custom; + private final Locale locale; + private final Object value; + CacheEntryImpl(Object readerKey, String fieldName, + Class cacheType, int sortFieldType, + Object custom, Locale locale, Object value) { + this.readerKey = readerKey; + this.fieldName = fieldName; + this.cacheType = cacheType; + this.sortFieldType = sortFieldType; + this.custom = custom; + this.locale = locale; + this.value = value; + } + public Object getReaderKey() { return readerKey; } + public String getFieldName() { return fieldName; } + public Class getCacheType() { return cacheType; } + public int getSortFieldType() { return sortFieldType; } + public Object getCustom() { return custom; } + public Locale getLocale() { return locale; } + public Object getValue() { return value; } + } + /** * Hack: When thrown from a Parser (NUMERIC_UTILS_* ones), this stops * processing terms and returns the current FieldCache @@ -50,12 +132,22 @@ /** Expert: Internal cache. */ abstract static class Cache { - private final Map readerCache = new WeakHashMap(); + Cache() { + this.wrapper = null; + } + + Cache(FieldCache wrapper) { + this.wrapper = wrapper; + } + + final FieldCache wrapper; + + final Map readerCache = new WeakHashMap(); - protected abstract Object createValue(IndexReader reader, Object key) + protected abstract Object createValue(IndexReader reader, Entry key) throws IOException; - public Object get(IndexReader reader, Object key) throws IOException { + public Object get(IndexReader reader, Entry key) throws IOException { Map innerCache; Object value; final Object readerKey = reader.getFieldCacheKey(); @@ -90,13 +182,10 @@ } } - static final class CreationPlaceholder { - Object value; - } - /** Expert: Every composite-key in the internal cache is of this type. */ static class Entry { final String field; // which Fieldable + /** :TODO: is this still needed? nothing seems to use it? */ final int type; // which SortField type final Object custom; // which custom comparator or parser final Locale locale; // the locale we're sorting (if string) @@ -156,18 +245,20 @@ // inherit javadocs public byte[] getBytes(IndexReader reader, String field, ByteParser parser) throws IOException { - return (byte[]) bytesCache.get(reader, new Entry(field, parser)); + return (byte[]) ((Cache)caches.get(Byte.TYPE)).get(reader, new Entry(field, parser)); } - Cache bytesCache = new Cache() { - - protected Object createValue(IndexReader reader, Object entryKey) + static final class ByteCache extends Cache { + ByteCache(FieldCache wrapper) { + super(wrapper); + } + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; ByteParser parser = (ByteParser) entry.custom; if (parser == null) { - return getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER); + return wrapper.getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER); } final byte[] retArray = new byte[reader.maxDoc()]; TermDocs termDocs = reader.termDocs(); @@ -199,18 +290,21 @@ // inherit javadocs public short[] getShorts(IndexReader reader, String field, ShortParser parser) throws IOException { - return (short[]) shortsCache.get(reader, new Entry(field, parser)); + return (short[]) ((Cache)caches.get(Short.TYPE)).get(reader, new Entry(field, parser)); } - Cache shortsCache = new Cache() { + static final class ShortCache extends Cache { + ShortCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; ShortParser parser = (ShortParser) entry.custom; if (parser == null) { - return getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER); + return wrapper.getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER); } final short[] retArray = new short[reader.maxDoc()]; TermDocs termDocs = reader.termDocs(); @@ -242,21 +336,24 @@ // inherit javadocs public int[] getInts(IndexReader reader, String field, IntParser parser) throws IOException { - return (int[]) intsCache.get(reader, new Entry(field, parser)); + return (int[]) ((Cache)caches.get(Integer.TYPE)).get(reader, new Entry(field, parser)); } - Cache intsCache = new Cache() { + static final class IntCache extends Cache { + IntCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; IntParser parser = (IntParser) entry.custom; if (parser == null) { try { - return getInts(reader, field, DEFAULT_INT_PARSER); + return wrapper.getInts(reader, field, DEFAULT_INT_PARSER); } catch (NumberFormatException ne) { - return getInts(reader, field, NUMERIC_UTILS_INT_PARSER); + return wrapper.getInts(reader, field, NUMERIC_UTILS_INT_PARSER); } } int[] retArray = null; @@ -294,24 +391,28 @@ // inherit javadocs public float[] getFloats(IndexReader reader, String field, FloatParser parser) - throws IOException { - return (float[]) floatsCache.get(reader, new Entry(field, parser)); + throws IOException { + + return (float[]) ((Cache)caches.get(Float.TYPE)).get(reader, new Entry(field, parser)); } - Cache floatsCache = new Cache() { + static final class FloatCache extends Cache { + FloatCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; FloatParser parser = (FloatParser) entry.custom; if (parser == null) { try { - return getFloats(reader, field, DEFAULT_FLOAT_PARSER); + return wrapper.getFloats(reader, field, DEFAULT_FLOAT_PARSER); } catch (NumberFormatException ne) { - return getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER); + return wrapper.getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER); } - } + } float[] retArray = null; TermDocs termDocs = reader.termDocs(); TermEnum termEnum = reader.terms (new Term (field)); @@ -346,27 +447,30 @@ // inherit javadocs public long[] getLongs(IndexReader reader, String field, FieldCache.LongParser parser) throws IOException { - return (long[]) longsCache.get(reader, new Entry(field, parser)); + return (long[]) ((Cache)caches.get(Long.TYPE)).get(reader, new Entry(field, parser)); } /** @deprecated Will be removed in 3.0, this is for binary compatibility only */ public long[] getLongs(IndexReader reader, String field, ExtendedFieldCache.LongParser parser) throws IOException { - return (long[]) longsCache.get(reader, new Entry(field, parser)); + return (long[]) ((Cache)caches.get(Long.TYPE)).get(reader, new Entry(field, parser)); } - Cache longsCache = new Cache() { + static final class LongCache extends Cache { + LongCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; FieldCache.LongParser parser = (FieldCache.LongParser) entry.custom; if (parser == null) { try { - return getLongs(reader, field, DEFAULT_LONG_PARSER); + return wrapper.getLongs(reader, field, DEFAULT_LONG_PARSER); } catch (NumberFormatException ne) { - return getLongs(reader, field, NUMERIC_UTILS_LONG_PARSER); + return wrapper.getLongs(reader, field, NUMERIC_UTILS_LONG_PARSER); } } long[] retArray = null; @@ -404,27 +508,30 @@ // inherit javadocs public double[] getDoubles(IndexReader reader, String field, FieldCache.DoubleParser parser) throws IOException { - return (double[]) doublesCache.get(reader, new Entry(field, parser)); + return (double[]) ((Cache)caches.get(Double.TYPE)).get(reader, new Entry(field, parser)); } /** @deprecated Will be removed in 3.0, this is for binary compatibility only */ public double[] getDoubles(IndexReader reader, String field, ExtendedFieldCache.DoubleParser parser) throws IOException { - return (double[]) doublesCache.get(reader, new Entry(field, parser)); + return (double[]) ((Cache)caches.get(Double.TYPE)).get(reader, new Entry(field, parser)); } - Cache doublesCache = new Cache() { + static final class DoubleCache extends Cache { + DoubleCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; FieldCache.DoubleParser parser = (FieldCache.DoubleParser) entry.custom; if (parser == null) { try { - return getDoubles(reader, field, DEFAULT_DOUBLE_PARSER); + return wrapper.getDoubles(reader, field, DEFAULT_DOUBLE_PARSER); } catch (NumberFormatException ne) { - return getDoubles(reader, field, NUMERIC_UTILS_DOUBLE_PARSER); + return wrapper.getDoubles(reader, field, NUMERIC_UTILS_DOUBLE_PARSER); } } double[] retArray = null; @@ -456,14 +563,17 @@ // inherit javadocs public String[] getStrings(IndexReader reader, String field) throws IOException { - return (String[]) stringsCache.get(reader, field); + return (String[]) ((Cache)caches.get(String.class)).get(reader, new Entry(field, SortField.STRING_VAL, (Parser)null)); } - Cache stringsCache = new Cache() { + static final class StringCache extends Cache { + StringCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object fieldKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { - String field = ((String) fieldKey).intern(); + String field = entryKey.field; final String[] retArray = new String[reader.maxDoc()]; TermDocs termDocs = reader.termDocs(); TermEnum termEnum = reader.terms (new Term (field)); @@ -488,14 +598,17 @@ // inherit javadocs public StringIndex getStringIndex(IndexReader reader, String field) throws IOException { - return (StringIndex) stringsIndexCache.get(reader, field); + return (StringIndex) ((Cache)caches.get(StringIndex.class)).get(reader, new Entry(field, SortField.STRING, (Parser)null)); } - Cache stringsIndexCache = new Cache() { + static final class StringIndexCache extends Cache { + StringIndexCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object fieldKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { - String field = ((String) fieldKey).intern(); + String field = entryKey.field; final int[] retArray = new int[reader.maxDoc()]; String[] mterms = new String[reader.maxDoc()+1]; TermDocs termDocs = reader.termDocs(); @@ -562,7 +675,7 @@ // inherit javadocs public Object getAuto(IndexReader reader, String field) throws IOException { - return autoCache.get(reader, field); + return ((Cache)caches.get(Object.class)).get(reader, new Entry(field, SortField.AUTO, (Parser)null)); } /** @@ -570,11 +683,14 @@ * Especially, guessing does not work with the new * {@link NumericField} type. */ - Cache autoCache = new Cache() { + static final class AutoCache extends Cache { + AutoCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object fieldKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { - String field = ((String)fieldKey).intern(); + String field = entryKey.field; TermEnum enumerator = reader.terms (new Term (field)); try { Term term = enumerator.term(); @@ -587,17 +703,17 @@ try { Integer.parseInt (termtext); - ret = getInts (reader, field); + ret = wrapper.getInts (reader, field); } catch (NumberFormatException nfe1) { try { Long.parseLong(termtext); - ret = getLongs (reader, field); + ret = wrapper.getLongs (reader, field); } catch (NumberFormatException nfe2) { try { Float.parseFloat (termtext); - ret = getFloats (reader, field); + ret = wrapper.getFloats (reader, field); } catch (NumberFormatException nfe3) { - ret = getStringIndex (reader, field); + ret = wrapper.getStringIndex (reader, field); } } } @@ -614,13 +730,16 @@ /** @deprecated */ public Comparable[] getCustom(IndexReader reader, String field, SortComparator comparator) throws IOException { - return (Comparable[]) customCache.get(reader, new Entry(field, comparator)); + return (Comparable[]) ((Cache)caches.get(Comparable.class)).get(reader, new Entry(field, comparator)); } /** @deprecated */ - Cache customCache = new Cache() { + static final class CustomCache extends Cache { + CustomCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; Index: src/java/org/apache/lucene/util/JavaImpl.java =================================================================== --- src/java/org/apache/lucene/util/JavaImpl.java (revision 0) +++ src/java/org/apache/lucene/util/JavaImpl.java (revision 0) @@ -0,0 +1,13 @@ +package org.apache.lucene.util; + +public abstract class JavaImpl { + + public abstract int getArraySize(); + + public abstract int getClassSize(); + + public abstract int getReferenceSize(); + + public abstract int getSize(Class clazz); + +} Property changes on: src/java/org/apache/lucene/util/JavaImpl.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: src/java/org/apache/lucene/util/AverageGuessJavaImpl.java =================================================================== --- src/java/org/apache/lucene/util/AverageGuessJavaImpl.java (revision 0) +++ src/java/org/apache/lucene/util/AverageGuessJavaImpl.java (revision 0) @@ -0,0 +1,37 @@ +package org.apache.lucene.util; + +import java.util.IdentityHashMap; +import java.util.Map; + +public class AverageGuessJavaImpl extends JavaImpl { + // best guess primitive sizes + private final Map sizes = new IdentityHashMap() { + { + put(boolean.class, new Integer(1)); + put(byte.class, new Integer(1)); + put(char.class, new Integer(2)); + put(short.class, new Integer(2)); + put(int.class, new Integer(4)); + put(float.class, new Integer(4)); + put(double.class, new Integer(8)); + put(long.class, new Integer(8)); + } + }; + + public int getArraySize() { + return 16; + } + + public int getClassSize() { + return 8; + } + + public int getReferenceSize() { + return 4; + } + + public int getSize(Class clazz) { + return ((Integer) sizes.get(clazz)).intValue(); + } + +} Property changes on: src/java/org/apache/lucene/util/AverageGuessJavaImpl.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: src/java/org/apache/lucene/util/MapOfSets.java =================================================================== --- src/java/org/apache/lucene/util/MapOfSets.java (revision 0) +++ src/java/org/apache/lucene/util/MapOfSets.java (revision 0) @@ -0,0 +1,63 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.util.Set; +import java.util.HashSet; +import java.util.Map; + +/** + * Helper class for keeping Listss of Objects associated with keys. + */ +public class MapOfSets { + + private final Map theMap; + + /** + * @param m the backing store for this object + */ + public MapOfSets(Map m) { + theMap = m; + } + + /** + * @return direct access to the map backing this object. + */ + public Map getMap() { + return theMap; + } + + /** + * Adds val to the Set associated with key in the Map. If key is not + * already in the map, a new Set will first be created. + * @return the size of the Set associated with key once val is added to it. + */ + public int put(Object key, Object val) { + final Set theSet; + if (theMap.containsKey(key)) { + theSet = (Set)theMap.get(key); + } else { + theSet = new HashSet(23); + theMap.put(key, theSet); + } + theSet.add(val); + return theSet.size(); + } + +} Property changes on: src/java/org/apache/lucene/util/MapOfSets.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: src/java/org/apache/lucene/util/FieldCacheSanityChecker.java =================================================================== --- src/java/org/apache/lucene/util/FieldCacheSanityChecker.java (revision 0) +++ src/java/org/apache/lucene/util/FieldCacheSanityChecker.java (revision 0) @@ -0,0 +1,183 @@ +package org.apache.lucene.util; +/** + * Copyright 2009 The Apache Software Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.ConcurrentMergeScheduler; + +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.FieldCache.CacheEntry; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.ArrayList; +import java.util.Map; +import java.util.HashMap; +import java.util.Set; +import java.util.HashSet; +import java.util.TreeSet; +import java.util.Iterator; + +/** + * Provides methods for sanity checking that entries in the FieldCache + * are not wasteful or inconsistent. + * + * :TODO: better javadocs, experimental warning. + * + */ +public final class FieldCacheSanityChecker { + + private FieldCacheSanityChecker() { /* :NOOP: */ } + + + /** + *

+ * Tests that no two entries have the same reader/fieldname but + * differnet cached values. This can happen if different datatypes, + * parsers, or Locales are used -- and while it's not neccessarily a bug + * it's typically an indication of a possible problem. + *

+ *

+ * Two important Things to note... + *

+ * + * + *

+ * :TODO: move to a utility class out of test + *

+ * + * :TODO: a lot of redundency between checkFieldCacheTypeSanity & checkFieldCacheSubReaderSanity .. refactor + * * + * @return the subset of cacheEntries that violate the check + */ + public static Collection checkFieldCacheTypeSanity(Set cacheEntries) { + final MapOfSets mapper = new MapOfSets(new HashMap(17)); + final Iterator iter = cacheEntries.iterator(); + final Set loserKeys = new HashSet(); + final MapOfSets valItems = new MapOfSets(new HashMap(17)); + while (iter.hasNext()) { + final CacheEntry item = (CacheEntry)iter.next(); + final Object val = item.getValue(); + + if (val instanceof FieldCache.CreationPlaceholder) + continue; + + // :HACK: using string when we should be using sane composite key + final Integer readerId = new Integer(System.identityHashCode + (item.getReaderKey())); + final String k = readerId + " " + item.getFieldName(); + + final Integer valId = new Integer(System.identityHashCode(val)); + + // indirect mapping, so the MapOfSet will dedup identical valIds for us + valItems.put(valId, item); + if (1 < mapper.put(k, valId)) { + loserKeys.add(k); + } + } + + if (loserKeys.isEmpty()) return new HashSet(); + + final Map keyMap = mapper.getMap(); + final Map valMap = valItems.getMap(); + final List losers = new ArrayList(loserKeys.size() * 3); + final Iterator loserIter = loserKeys.iterator(); + while (loserIter.hasNext()) { + final Iterator valIter = ((Set)keyMap.get(loserIter.next())).iterator(); + while (valIter.hasNext()) { + losers.addAll((Set)valMap.get(valIter.next())); + } + } + return losers; + } + + + /** + * Tests that no reader/field combo matches in a sub/super reader + * + * :TODO: move to a utility class out of test + * + * :TODO: a lot of redundency between checkFieldCacheTypeSanity & checkFieldCacheSubReaderSanity .. refactor + * + * @return the subset of cacheEntries that violate the check + */ + public static Collection checkFieldCacheSubReaderSanity(Set cacheEntries) { + final MapOfSets mapper = new MapOfSets(new HashMap(17)); + final Iterator iter = cacheEntries.iterator(); + final Set loserKeys = new HashSet(); + final MapOfSets valItems = new MapOfSets(new HashMap(17)); + while (iter.hasNext()) { + final CacheEntry item = (CacheEntry)iter.next(); + final Object val = item.getValue(); + + if (val instanceof FieldCache.CreationPlaceholder) + continue; + + List readerKeys = new ArrayList(); // grows as we walk the subreaders + readerKeys.add(item.getReaderKey()); + + for (int i = 0; i < readerKeys.size(); i++) { + Object rKey = readerKeys.get(i); + if (rKey instanceof IndexReader) { + IndexReader[] subs = ((IndexReader)rKey).getSequentialSubReaders(); + for (int j = 0; (null != subs) && (j < subs.length); j++) { + readerKeys.add(subs[j].getFieldCacheKey()); + } + } + + // :HACK: using string when we should be using sane composite key + final Integer readerId = new Integer(System.identityHashCode(rKey)); + final String k = readerId + " " + item.getFieldName(); + + final Integer valId = new Integer(System.identityHashCode(val)); + + // indirect mapping, so the MapOfSet will dedup identical valIds for us + valItems.put(valId, item); + if (1 < mapper.put(k, valId)) { + loserKeys.add(k); + } + } + } + + if (loserKeys.isEmpty()) return new HashSet(); + + final Map keyMap = mapper.getMap(); + final Map valMap = valItems.getMap(); + final HashSet losers = new HashSet(17); + final Iterator loserIter = loserKeys.iterator(); + while (loserIter.hasNext()) { + final Iterator valIter = ((Set)keyMap.get(loserIter.next())).iterator(); + while (valIter.hasNext()) { + losers.addAll((Set)valMap.get(valIter.next())); + } + } + return losers; + } + +} Property changes on: src/java/org/apache/lucene/util/FieldCacheSanityChecker.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: src/java/org/apache/lucene/util/RamUsageEstimator.java =================================================================== --- src/java/org/apache/lucene/util/RamUsageEstimator.java (revision 0) +++ src/java/org/apache/lucene/util/RamUsageEstimator.java (revision 0) @@ -0,0 +1,151 @@ +package org.apache.lucene.util; + +import java.lang.reflect.*; +import java.text.DecimalFormat; +import java.util.*; + +/** + * + */ +public final class RamUsageEstimator { + private JavaImpl javaImpl; + + private final Map seen; + + private final List objects = new LinkedList(); + + public RamUsageEstimator() { + this(64); + } + + public RamUsageEstimator(JavaImpl arcitecture) { + this(arcitecture, 64); + } + + public RamUsageEstimator(JavaImpl arcitecture, int approxObjects) { + this.javaImpl = arcitecture; + // Use Map rather than Set so that we can use an IdentityHashMap - not + // seeing an IdentityHashSet + seen = new IdentityHashMap(64); + } + + public RamUsageEstimator(int approxObjects) { + this(new AverageGuessJavaImpl(), approxObjects); + } + + public long estimateRamUsage(Object obj) { + long size = walk(obj); + seen.clear(); + return size; + } + + private long walk(Object obj) { + long size = size(obj); + Iterator it = objects.iterator(); + while (it.hasNext()) { + size += size(it.next()); + } + return size; + } + + private long size(Object obj) { + if (obj == null) { + return 0; + } + // interned could be counted once, but its in perm space, not heap, so skip + if (obj instanceof String && obj == ((String) obj).intern()) { + return 0; + } + + // skip if we have seen before + if (seen.containsKey(obj)) { + return 0; + } + + // add to seen + seen.put(obj, null); + + Class clazz = obj.getClass(); + if (clazz.isArray()) { + return sizeOfArray(obj); + } + + long size = 0; + + // walk type hierarchy + while (clazz != null) { + Field[] fields = clazz.getDeclaredFields(); + for (int i = 0; i < fields.length; i++) { + if (Modifier.isStatic(fields[i].getModifiers())) { + // TODO: handle this differently? + } + + if (fields[i].getType().isPrimitive()) { + size += javaImpl.getSize(fields[i].getType()); + } else { + size += javaImpl.getReferenceSize(); + fields[i].setAccessible(true); + try { + Object value = fields[i].get(obj); + if (value != null) { + size += walk(value); + } + } catch (IllegalAccessException ex) { + // ignore for now? + } + } + + } + clazz = clazz.getSuperclass(); + } + size += javaImpl.getClassSize(); + return size; + } + + private long sizeOfArray(Object obj) { + int len = Array.getLength(obj); + if (len == 0) { + return 0; + } + long size = javaImpl.getArraySize(); + Class arrayElementClazz = obj.getClass().getComponentType(); + if (arrayElementClazz.isPrimitive()) { + size += len * javaImpl.getSize(arrayElementClazz); + } else { + for (int i = 0; i < len; i++) { + size += javaImpl.getReferenceSize() + size(Array.get(obj, i)); + //System.out.println("array ob:" + javaImpl.getReferenceSize() + size(Array.get(obj, i))); + } + } + + return size; + } + + private static final long ONE_KB = 1024; + + private static final long ONE_MB = ONE_KB * ONE_KB; + + private static final long ONE_GB = ONE_KB * ONE_MB; + + /** + * Return good default units based on byte size. + */ + public static String humanReadableUnits(long bytes, DecimalFormat df) { + String newSizeAndUnits; + + if (bytes / ONE_GB > 0) { + newSizeAndUnits = String.valueOf(df.format((float) bytes / ONE_GB)) + + " GB"; + } else if (bytes / ONE_MB > 0) { + newSizeAndUnits = String.valueOf(df.format((float) bytes / ONE_MB)) + + " MB"; + } else if (bytes / ONE_KB > 0) { + newSizeAndUnits = String.valueOf(df.format((float) bytes / ONE_KB)) + + " KB"; + } else { + newSizeAndUnits = String.valueOf(bytes) + " bytes"; + } + + return newSizeAndUnits; + } +} Property changes on: src/java/org/apache/lucene/util/RamUsageEstimator.java ___________________________________________________________________ Name: svn:keywords + Date Author Id Revision HeadURL Name: svn:eol-style + native Index: contrib/remote/src/test/org/apache/lucene/search/TestRemoteSort.java =================================================================== --- contrib/remote/src/test/org/apache/lucene/search/TestRemoteSort.java (revision 798925) +++ contrib/remote/src/test/org/apache/lucene/search/TestRemoteSort.java (working copy) @@ -248,11 +248,18 @@ assertMatches (multi, queryX, sort, "CAIEG"); sort.setSort (new SortField ("custom", SampleComparable.getComparatorSource(), true)); assertMatches (multi, queryY, sort, "HJDBF"); + + assertSaneFieldCaches(getName() + " ComparatorSource"); + FieldCache.DEFAULT.purgeAllCaches(); + SortComparator custom = SampleComparable.getComparator(); sort.setSort (new SortField ("custom", custom)); assertMatches (multi, queryX, sort, "CAIEG"); sort.setSort (new SortField ("custom", custom, true)); assertMatches (multi, queryY, sort, "HJDBF"); + + assertSaneFieldCaches(getName() + " Comparator"); + FieldCache.DEFAULT.purgeAllCaches(); } // test that the relevancy scores are the same even if @@ -347,12 +354,6 @@ sort.setSort("string", true); assertMatches(multi, queryA, sort, "CBEFGHIAJD"); - sort.setSort(new SortField[] { new SortField ("string", Locale.US) }); - assertMatches(multi, queryA, sort, "DJAIHGFEBC"); - - sort.setSort(new SortField[] { new SortField ("string", Locale.US, true) }); - assertMatches(multi, queryA, sort, "CBEFGHIAJD"); - sort.setSort(new String[] {"int","float"}); assertMatches(multi, queryA, sort, "IDHFGJEABC"); @@ -373,6 +374,21 @@ sort.setSort("string", true); assertMatches(multi, queryF, sort, "IJZ"); + + // up to this point, all of the searches should have "sane" + // FieldCache behavior, and should have reused hte cache in several cases + assertSaneFieldCaches(getName() + " Basics"); + // next we'll check an alternate Locale for string, so purge first + FieldCache.DEFAULT.purgeAllCaches(); + + sort.setSort(new SortField[] { new SortField ("string", Locale.US) }); + assertMatches(multi, queryA, sort, "DJAIHGFEBC"); + + sort.setSort(new SortField[] { new SortField ("string", Locale.US, true)}); + assertMatches(multi, queryA, sort, "CBEFGHIAJD"); + + assertSaneFieldCaches(getName() + " Locale.US"); + FieldCache.DEFAULT.purgeAllCaches(); } // make sure the documents returned by the search match the expected list