Index: src/test/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- src/test/org/apache/lucene/util/LuceneTestCase.java (revision 794282) +++ src/test/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -19,7 +19,15 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.ConcurrentMergeScheduler; + +import org.apache.lucene.search.FieldCache; + import junit.framework.TestCase; + +import java.io.PrintStream; + +import java.util.Set; +import java.util.Iterator; import java.util.Random; /** Base class for all Lucene unit tests. Currently the @@ -48,6 +56,12 @@ } protected void tearDown() throws Exception { + // this isn't as useful as calling directly from the scope where the + // index readers are used, because they could be gc'ed just before + // tearDown is called. + assertSaneFieldCaches(); + + if (ConcurrentMergeScheduler.anyUnhandledExceptions()) { // Clear the failure so that we don't just keep // failing subsequent test cases @@ -55,6 +69,29 @@ fail("ConcurrentMergeScheduler hit unhandled exceptions"); } } + + protected void assertSaneFieldCaches() { + Set enteries = FieldCache.DEFAULT.getCacheEntries(); + + // :TODO: walk enteries, assert... + // :TODO: - no reader/field combo has multiple parsers + // :TODO: - no reader/field combo matches in a sub/super reader + + + // :TODO: only dump if asserts fails + if (!enteries.isEmpty()) + dumpIterator(getName()+" FIELD CACHE", enteries.iterator(), System.err); + } + + + public static void dumpIterator(String label, Iterator iter, + PrintStream stream) { + stream.println("*** BEGIN "+label+" ***"); + while (iter.hasNext()) { + stream.println(iter.next().toString()); + } + stream.println("*** END "+label+" ***"); + } /** * Returns a {@link Random} instance for generating random numbers during the test. Index: src/java/org/apache/lucene/search/FieldCache.java =================================================================== --- src/java/org/apache/lucene/search/FieldCache.java (revision 794282) +++ src/java/org/apache/lucene/search/FieldCache.java (working copy) @@ -25,6 +25,9 @@ import java.io.IOException; import java.io.Serializable; +import java.util.Locale; +import java.util.Set; + /** * Expert: Maintains caches of term values. * @@ -146,6 +149,9 @@ protected Object readResolve() { return DEFAULT_BYTE_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_BYTE_PARSER"; + } }; /** The default parser for short values, which are encoded by {@link Short#toString(short)} */ @@ -156,6 +162,9 @@ protected Object readResolve() { return DEFAULT_SHORT_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_SHORT_PARSER"; + } }; /** The default parser for int values, which are encoded by {@link Integer#toString(int)} */ @@ -166,6 +175,9 @@ protected Object readResolve() { return DEFAULT_INT_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_INT_PARSER"; + } }; /** The default parser for float values, which are encoded by {@link Float#toString(float)} */ @@ -176,6 +188,9 @@ protected Object readResolve() { return DEFAULT_FLOAT_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_FLOAT_PARSER"; + } }; /** The default parser for long values, which are encoded by {@link Long#toString(long)} */ @@ -186,6 +201,9 @@ protected Object readResolve() { return DEFAULT_LONG_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_LONG_PARSER"; + } }; /** The default parser for double values, which are encoded by {@link Double#toString(double)} */ @@ -196,6 +214,9 @@ protected Object readResolve() { return DEFAULT_DOUBLE_PARSER; } + public String toString() { + return FieldCache.class.getName()+".DEFAULT_DOUBLE_PARSER"; + } }; /** @@ -212,6 +233,9 @@ protected Object readResolve() { return NUMERIC_UTILS_INT_PARSER; } + public String toString() { + return FieldCache.class.getName()+".NUMERIC_UTILS_INT_PARSER"; + } }; /** @@ -228,6 +252,9 @@ protected Object readResolve() { return NUMERIC_UTILS_FLOAT_PARSER; } + public String toString() { + return FieldCache.class.getName()+".NUMERIC_UTILS_FLOAT_PARSER"; + } }; /** @@ -244,6 +271,9 @@ protected Object readResolve() { return NUMERIC_UTILS_LONG_PARSER; } + public String toString() { + return FieldCache.class.getName()+".NUMERIC_UTILS_LONG_PARSER"; + } }; /** @@ -260,6 +290,9 @@ protected Object readResolve() { return NUMERIC_UTILS_DOUBLE_PARSER; } + public String toString() { + return FieldCache.class.getName()+".NUMERIC_UTILS_DOUBLE_PARSER"; + } }; /** Checks the internal cache for an appropriate entry, and if none is @@ -477,5 +510,31 @@ */ public Comparable[] getCustom (IndexReader reader, String field, SortComparator comparator) throws IOException; + + /** + * A unique Identifier for each itme in the FieldCache + */ + public static abstract class CacheEntry { + public abstract Object getReaderKey(); + public abstract String getFieldName(); + public abstract Class getCacheType(); + public abstract int getSortFieldType(); + public abstract Object getCustom(); + public abstract Locale getLocale(); + public abstract Object getValue(); + public String toString() { + StringBuffer b = new StringBuffer(); + b.append("'").append(getReaderKey()).append("'=>"); + b.append("'").append(getFieldName()).append("',"); + b.append(getCacheType()).append(",").append(getSortFieldType()); + b.append(",").append(getCustom()).append(",").append(getLocale()); + b.append("=>").append(getValue().getClass().getName()).append("#"); + b.append(System.identityHashCode(getValue())); + // :TODO: it would be nice to know how much ram getValue() takes up + return b.toString(); + } + } + public abstract Set getCacheEntries(); + } Index: src/java/org/apache/lucene/search/FieldSortedHitQueue.java =================================================================== --- src/java/org/apache/lucene/search/FieldSortedHitQueue.java (revision 794282) +++ src/java/org/apache/lucene/search/FieldSortedHitQueue.java (working copy) @@ -180,7 +180,7 @@ * caches comparators instead of term values. */ static final FieldCacheImpl.Cache Comparators = new FieldCacheImpl.Cache() { - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, FieldCacheImpl.Entry entryKey) throws IOException { FieldCacheImpl.Entry entry = (FieldCacheImpl.Entry) entryKey; String fieldname = entry.field; Index: src/java/org/apache/lucene/search/FieldCacheImpl.java =================================================================== --- src/java/org/apache/lucene/search/FieldCacheImpl.java (revision 794282) +++ src/java/org/apache/lucene/search/FieldCacheImpl.java (working copy) @@ -23,9 +23,12 @@ import org.apache.lucene.index.TermEnum; import java.io.IOException; +import java.util.Iterator; +import java.util.HashSet; import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.WeakHashMap; /** @@ -34,12 +37,100 @@ * *

Created: May 19, 2004 4:40:36 PM * + * + * + * + * + * + * :TODO: is the "int" sort type still needed? ... doesn't seem to be used anywhere, code just tests "custom" for SortComparator vs Parser. + * + * + * + * + * + * + * + * + * + * * @since lucene 1.4 * @version $Id$ */ // TODO: change interface to FieldCache in 3.0 when removed class FieldCacheImpl implements ExtendedFieldCache { + /** Map from cache type (Class) to cache (Cache) */ + private Map caches = new HashMap(7); + FieldCacheImpl() { + caches.put(Byte.TYPE, new ByteCache(this)); + caches.put(Short.TYPE, new ShortCache(this)); + caches.put(Integer.TYPE, new IntCache(this)); + caches.put(Float.TYPE, new FloatCache(this)); + caches.put(Long.TYPE, new LongCache(this)); + caches.put(Double.TYPE, new DoubleCache(this)); + caches.put(String.class, new StringCache(this)); + caches.put(StringIndex.class, new StringIndexCache(this)); + caches.put(Comparable.class, new CustomCache(this)); + caches.put(Object.class, new AutoCache(this)); + } + + public Set getCacheEntries() { + Set result = new HashSet(17); + Iterator outerKeys = caches.keySet().iterator(); + while (outerKeys.hasNext()) { + Class cacheType = (Class)outerKeys.next(); + Cache cache = (Cache)caches.get(cacheType); + Iterator innerKeys = cache.readerCache.keySet().iterator(); + while (innerKeys.hasNext()) { + // we've now materialized a hard ref + Object readerKey = innerKeys.next(); + // innerKeys was backed by WeakHashMap, sanity check + // that it wasn't GCed before we made hard ref + if (null != readerKey && cache.readerCache.containsKey(readerKey)) { + Map innerCache = ((Map)cache.readerCache.get(readerKey)); + Iterator keys = innerCache.keySet().iterator(); + while (keys.hasNext()) { + Entry entry = (Entry) keys.next(); + result.add(new CacheEntryImpl(readerKey, entry.field, + cacheType, entry.type, + entry.custom, entry.locale, + innerCache.get(entry))); + } + } + } + } + return result; + } + + private static final class CacheEntryImpl extends CacheEntry { + private final Object readerKey; + private final String fieldName; + private final Class cacheType; + private final int sortFieldType; + private final Object custom; + private final Locale locale; + private final Object value; + CacheEntryImpl(Object readerKey, String fieldName, + Class cacheType, int sortFieldType, + Object custom, Locale locale, Object value) { + this.readerKey = readerKey; + this.fieldName = fieldName; + this.cacheType = cacheType; + this.sortFieldType = sortFieldType; + this.custom = custom; + this.locale = locale; + this.value = value; + } + public Object getReaderKey() { return readerKey; } + public String getFieldName() { return fieldName; } + public Class getCacheType() { return cacheType; } + public int getSortFieldType() { return sortFieldType; } + public Object getCustom() { return custom; } + public Locale getLocale() { return locale; } + public Object getValue() { return value; } + } + + /** * Hack: When thrown from a Parser (NUMERIC_UTILS_* ones), this stops * processing terms and returns the current FieldCache @@ -50,12 +141,19 @@ /** Expert: Internal cache. */ abstract static class Cache { - private final Map readerCache = new WeakHashMap(); + Cache() { + this.wrapper = null; + } + Cache(FieldCache wrapper) { + this.wrapper = wrapper; + } + final FieldCache wrapper; + final Map readerCache = new WeakHashMap(); - protected abstract Object createValue(IndexReader reader, Object key) + protected abstract Object createValue(IndexReader reader, Entry key) throws IOException; - public Object get(IndexReader reader, Object key) throws IOException { + public Object get(IndexReader reader, Entry key) throws IOException { Map innerCache; Object value; final Object readerKey = reader.getFieldCacheKey(); @@ -156,18 +254,20 @@ // inherit javadocs public byte[] getBytes(IndexReader reader, String field, ByteParser parser) throws IOException { - return (byte[]) bytesCache.get(reader, new Entry(field, parser)); + return (byte[]) ((Cache)caches.get(Byte.TYPE)).get(reader, new Entry(field, parser)); } - Cache bytesCache = new Cache() { - - protected Object createValue(IndexReader reader, Object entryKey) + static final class ByteCache extends Cache { + ByteCache(FieldCache wrapper) { + super(wrapper); + } + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; ByteParser parser = (ByteParser) entry.custom; if (parser == null) { - return getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER); + return wrapper.getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER); } final byte[] retArray = new byte[reader.maxDoc()]; TermDocs termDocs = reader.termDocs(); @@ -199,18 +299,21 @@ // inherit javadocs public short[] getShorts(IndexReader reader, String field, ShortParser parser) throws IOException { - return (short[]) shortsCache.get(reader, new Entry(field, parser)); + return (short[]) ((Cache)caches.get(Short.TYPE)).get(reader, new Entry(field, parser)); } - Cache shortsCache = new Cache() { + static final class ShortCache extends Cache { + ShortCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; ShortParser parser = (ShortParser) entry.custom; if (parser == null) { - return getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER); + return wrapper.getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER); } final short[] retArray = new short[reader.maxDoc()]; TermDocs termDocs = reader.termDocs(); @@ -242,21 +345,24 @@ // inherit javadocs public int[] getInts(IndexReader reader, String field, IntParser parser) throws IOException { - return (int[]) intsCache.get(reader, new Entry(field, parser)); + return (int[]) ((Cache)caches.get(Integer.TYPE)).get(reader, new Entry(field, parser)); } - Cache intsCache = new Cache() { + static final class IntCache extends Cache { + IntCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; IntParser parser = (IntParser) entry.custom; if (parser == null) { try { - return getInts(reader, field, DEFAULT_INT_PARSER); + return wrapper.getInts(reader, field, DEFAULT_INT_PARSER); } catch (NumberFormatException ne) { - return getInts(reader, field, NUMERIC_UTILS_INT_PARSER); + return wrapper.getInts(reader, field, NUMERIC_UTILS_INT_PARSER); } } int[] retArray = null; @@ -294,24 +400,28 @@ // inherit javadocs public float[] getFloats(IndexReader reader, String field, FloatParser parser) - throws IOException { - return (float[]) floatsCache.get(reader, new Entry(field, parser)); + throws IOException { + + return (float[]) ((Cache)caches.get(Float.TYPE)).get(reader, new Entry(field, parser)); } - Cache floatsCache = new Cache() { + static final class FloatCache extends Cache { + FloatCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; FloatParser parser = (FloatParser) entry.custom; if (parser == null) { try { - return getFloats(reader, field, DEFAULT_FLOAT_PARSER); + return wrapper.getFloats(reader, field, DEFAULT_FLOAT_PARSER); } catch (NumberFormatException ne) { - return getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER); + return wrapper.getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER); } - } + } float[] retArray = null; TermDocs termDocs = reader.termDocs(); TermEnum termEnum = reader.terms (new Term (field)); @@ -346,27 +456,30 @@ // inherit javadocs public long[] getLongs(IndexReader reader, String field, FieldCache.LongParser parser) throws IOException { - return (long[]) longsCache.get(reader, new Entry(field, parser)); + return (long[]) ((Cache)caches.get(Long.TYPE)).get(reader, new Entry(field, parser)); } /** @deprecated Will be removed in 3.0, this is for binary compatibility only */ public long[] getLongs(IndexReader reader, String field, ExtendedFieldCache.LongParser parser) throws IOException { - return (long[]) longsCache.get(reader, new Entry(field, parser)); + return (long[]) ((Cache)caches.get(Long.TYPE)).get(reader, new Entry(field, parser)); } - Cache longsCache = new Cache() { + static final class LongCache extends Cache { + LongCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; FieldCache.LongParser parser = (FieldCache.LongParser) entry.custom; if (parser == null) { try { - return getLongs(reader, field, DEFAULT_LONG_PARSER); + return wrapper.getLongs(reader, field, DEFAULT_LONG_PARSER); } catch (NumberFormatException ne) { - return getLongs(reader, field, NUMERIC_UTILS_LONG_PARSER); + return wrapper.getLongs(reader, field, NUMERIC_UTILS_LONG_PARSER); } } long[] retArray = null; @@ -404,27 +517,30 @@ // inherit javadocs public double[] getDoubles(IndexReader reader, String field, FieldCache.DoubleParser parser) throws IOException { - return (double[]) doublesCache.get(reader, new Entry(field, parser)); + return (double[]) ((Cache)caches.get(Double.TYPE)).get(reader, new Entry(field, parser)); } /** @deprecated Will be removed in 3.0, this is for binary compatibility only */ public double[] getDoubles(IndexReader reader, String field, ExtendedFieldCache.DoubleParser parser) throws IOException { - return (double[]) doublesCache.get(reader, new Entry(field, parser)); + return (double[]) ((Cache)caches.get(Double.TYPE)).get(reader, new Entry(field, parser)); } - Cache doublesCache = new Cache() { + static final class DoubleCache extends Cache { + DoubleCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field; FieldCache.DoubleParser parser = (FieldCache.DoubleParser) entry.custom; if (parser == null) { try { - return getDoubles(reader, field, DEFAULT_DOUBLE_PARSER); + return wrapper.getDoubles(reader, field, DEFAULT_DOUBLE_PARSER); } catch (NumberFormatException ne) { - return getDoubles(reader, field, NUMERIC_UTILS_DOUBLE_PARSER); + return wrapper.getDoubles(reader, field, NUMERIC_UTILS_DOUBLE_PARSER); } } double[] retArray = null; @@ -456,14 +572,17 @@ // inherit javadocs public String[] getStrings(IndexReader reader, String field) throws IOException { - return (String[]) stringsCache.get(reader, field); + return (String[]) ((Cache)caches.get(String.class)).get(reader, new Entry(field, SortField.STRING_VAL, (Parser)null)); } - Cache stringsCache = new Cache() { + static final class StringCache extends Cache { + StringCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object fieldKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { - String field = ((String) fieldKey).intern(); + String field = entryKey.field; final String[] retArray = new String[reader.maxDoc()]; TermDocs termDocs = reader.termDocs(); TermEnum termEnum = reader.terms (new Term (field)); @@ -488,14 +607,17 @@ // inherit javadocs public StringIndex getStringIndex(IndexReader reader, String field) throws IOException { - return (StringIndex) stringsIndexCache.get(reader, field); + return (StringIndex) ((Cache)caches.get(StringIndex.class)).get(reader, new Entry(field, SortField.STRING, (Parser)null)); } - Cache stringsIndexCache = new Cache() { + static final class StringIndexCache extends Cache { + StringIndexCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object fieldKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { - String field = ((String) fieldKey).intern(); + String field = entryKey.field; final int[] retArray = new int[reader.maxDoc()]; String[] mterms = new String[reader.maxDoc()+1]; TermDocs termDocs = reader.termDocs(); @@ -562,7 +684,7 @@ // inherit javadocs public Object getAuto(IndexReader reader, String field) throws IOException { - return autoCache.get(reader, field); + return ((Cache)caches.get(Object.class)).get(reader, new Entry(field, SortField.AUTO, (Parser)null)); } /** @@ -570,11 +692,14 @@ * Especially, guessing does not work with the new * {@link NumericField} type. */ - Cache autoCache = new Cache() { + static final class AutoCache extends Cache { + AutoCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object fieldKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { - String field = ((String)fieldKey).intern(); + String field = entryKey.field; TermEnum enumerator = reader.terms (new Term (field)); try { Term term = enumerator.term(); @@ -587,17 +712,17 @@ try { Integer.parseInt (termtext); - ret = getInts (reader, field); + ret = wrapper.getInts (reader, field); } catch (NumberFormatException nfe1) { try { Long.parseLong(termtext); - ret = getLongs (reader, field); + ret = wrapper.getLongs (reader, field); } catch (NumberFormatException nfe2) { try { Float.parseFloat (termtext); - ret = getFloats (reader, field); + ret = wrapper.getFloats (reader, field); } catch (NumberFormatException nfe3) { - ret = getStringIndex (reader, field); + ret = wrapper.getStringIndex (reader, field); } } } @@ -614,13 +739,16 @@ /** @deprecated */ public Comparable[] getCustom(IndexReader reader, String field, SortComparator comparator) throws IOException { - return (Comparable[]) customCache.get(reader, new Entry(field, comparator)); + return (Comparable[]) ((Cache)caches.get(Comparable.class)).get(reader, new Entry(field, comparator)); } /** @deprecated */ - Cache customCache = new Cache() { + static final class CustomCache extends Cache { + CustomCache(FieldCache wrapper) { + super(wrapper); + } - protected Object createValue(IndexReader reader, Object entryKey) + protected Object createValue(IndexReader reader, Entry entryKey) throws IOException { Entry entry = (Entry) entryKey; String field = entry.field;