Index: src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldSortComparator.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldSortComparator.java (revision 732759) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldSortComparator.java (working copy) @@ -105,9 +105,8 @@ IndexReader r = (IndexReader) readers.get(i); starts[i] = maxDoc; maxDoc += r.maxDoc(); - indexes[i] = SharedFieldCache.INSTANCE.getStringIndex(r, field, - FieldNames.createNamedValue(propertyName, ""), - SharedFieldSortComparator.this, createComparatorValues); + indexes[i] = SharedFieldCache.INSTANCE.getStringIndex(r, field, FieldNames.createNamedValue(propertyName, + ""), SharedFieldSortComparator.this, createComparatorValues); } starts[readers.size()] = maxDoc; @@ -117,8 +116,8 @@ int idx1 = readerIndex(i.doc); int idx2 = readerIndex(j.doc); - String iTerm = indexes[idx1].terms[i.doc - starts[idx1]]; - String jTerm = indexes[idx2].terms[j.doc - starts[idx2]]; + String iTerm = indexes[idx1].getTerm(i.doc - starts[idx1]); + String jTerm = indexes[idx2].getTerm(j.doc - starts[idx2]); if (iTerm == jTerm) { return 0; @@ -141,8 +140,8 @@ */ public Comparable sortValue(final ScoreDoc i) { if (createComparatorValues) { - StringIndex index = indexes[readerIndex(i.doc)]; - return index.terms[i.doc]; + int idx = readerIndex(i.doc); + return indexes[idx].getTerm(i.doc - starts[idx]); } else { // return dummy value return ""; Index: src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldCache.java =================================================================== --- src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldCache.java (revision 732759) +++ src/main/java/org/apache/jackrabbit/core/query/lucene/SharedFieldCache.java (working copy) @@ -43,6 +43,13 @@ public static class StringIndex { /** + * Some heuristic factor that determines whether the array is sparse. Note that if less then + * 1% is set, we already count the array as sparse. This is because it will become memory consuming + * quickly by keeping the (sparse) arrays + */ + private static final int SPARSE_FACTOR = 100; + + /** * All the term values, in natural order. */ public final String[] lookup; @@ -50,15 +57,65 @@ /** * Terms indexed by document id. */ - public final String[] terms; + private final String[] terms; + + /** + * Terms map indexed by document id. + */ + public final Map termsMap; + + /** + * Boolean indicating whether the hashMap impl has to be used + */ + + public boolean sparse = false; /** * Creates one of these objects */ - public StringIndex(String[] terms, String[] lookup) { - this.terms = terms; + public StringIndex(String[] terms, String[] lookup, int setValues) { + if (isSparse(terms, setValues)) { + this.sparse = true; + this.terms = null; + if (setValues == 0) { + this.termsMap = null; + } else { + this.termsMap = getTermsMap(terms, setValues); + } + } else { + this.terms = terms; + this.termsMap = null; + } this.lookup = lookup; } + + public String getTerm(int i) { + if (sparse) { + return termsMap == null ? null : (String) termsMap.get(new Integer(i)); + } else { + return terms[i]; + } + } + + private Map getTermsMap(String[] terms, int setValues) { + Map map = new HashMap(setValues); + for (int i = 0; i < terms.length && setValues > 0; i++) { + if (terms[i] != null) { + map.put(new Integer(i), terms[i]); + setValues--; + } + } + return map; + } + + private boolean isSparse(String[] terms, int setValues) { + // some really simple test to test whether the array is sparse. Currently, when less then 10% is set, the array is already sparse + // for this typical cache to avoid memory issues + if (setValues * SPARSE_FACTOR < terms.length) { + return true; + } + return false; + } } /** @@ -95,12 +152,8 @@ * information. * @throws IOException if an error occurs while reading from the index. */ - public SharedFieldCache.StringIndex getStringIndex(IndexReader reader, - String field, - String prefix, - SortComparator comparator, - boolean includeLookup) - throws IOException { + public SharedFieldCache.StringIndex getStringIndex(IndexReader reader, String field, String prefix, + SortComparator comparator, boolean includeLookup) throws IOException { if (reader instanceof ReadOnlyIndexReader) { reader = ((ReadOnlyIndexReader) reader).getBase(); @@ -114,6 +167,7 @@ if (includeLookup) { mterms = new ArrayList(); } + int setValues = 0; if (retArray.length > 0) { TermDocs termDocs = reader.termDocs(); TermEnum termEnum = reader.terms(new Term(field, prefix)); @@ -141,6 +195,7 @@ termDocs.seek(termEnum); while (termDocs.next()) { + setValues++; retArray[termDocs.doc()] = term.text().substring(prefix.length()); } } while (termEnum.next()); @@ -153,7 +208,7 @@ if (includeLookup) { lookup = (String[]) mterms.toArray(new String[mterms.size()]); } - SharedFieldCache.StringIndex value = new SharedFieldCache.StringIndex(retArray, lookup); + SharedFieldCache.StringIndex value = new SharedFieldCache.StringIndex(retArray, lookup, setValues); store(reader, field, prefix, comparator, value); return value; } @@ -163,8 +218,7 @@ /** * See if a StringIndex object is in the cache. */ - SharedFieldCache.StringIndex lookup(IndexReader reader, String field, - String prefix, SortComparator comparer) { + SharedFieldCache.StringIndex lookup(IndexReader reader, String field, String prefix, SortComparator comparer) { Key key = new Key(field, prefix, comparer); synchronized (this) { HashMap readerCache = (HashMap) cache.get(reader); @@ -178,8 +232,8 @@ /** * Put a StringIndex value to cache. */ - Object store(IndexReader reader, String field, String prefix, - SortComparator comparer, SharedFieldCache.StringIndex value) { + Object store(IndexReader reader, String field, String prefix, SortComparator comparer, + SharedFieldCache.StringIndex value) { Key key = new Key(field, prefix, comparer); synchronized (this) { HashMap readerCache = (HashMap) cache.get(reader); @@ -217,9 +271,7 @@ public boolean equals(Object o) { if (o instanceof Key) { Key other = (Key) o; - return other.field == field - && other.prefix == prefix - && other.comparator.equals(comparator); + return other.field == field && other.prefix == prefix && other.comparator.equals(comparator); } return false; }