Index: src/test/org/apache/lucene/search/TestMemoryCachedRangeFilterPerformance.java =================================================================== --- src/test/org/apache/lucene/search/TestMemoryCachedRangeFilterPerformance.java (revision 0) +++ src/test/org/apache/lucene/search/TestMemoryCachedRangeFilterPerformance.java (revision 0) @@ -0,0 +1,113 @@ +package org.apache.lucene.search; + +import java.io.File; +import java.io.IOException; +import java.util.Date; +import java.util.Random; + +import junit.framework.TestCase; + +import org.apache.lucene.analysis.KeywordAnalyzer; +import org.apache.lucene.document.DateTools; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.DateTools.Resolution; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.RangeFilter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.RAMDirectory; + +/** + * Compares performance between MemoryCachedRangeFilter and standard RangeFilter + * + */ +public class TestMemoryCachedRangeFilterPerformance extends TestCase { + + public static final long INTERVAL = 5 * 365 * 24 * 60 * 60 * 1000L; + + Random r = new Random(1); + + public void testPerformance() throws IOException { + RAMDirectory ramDir = new RAMDirectory(); +// Directory ramDir = FSDirectory.getDirectory(new File("/tmp/dateindex")); + IndexWriter writer = new IndexWriter(ramDir, new KeywordAnalyzer(), true); + + Date endInterval = new Date(); + + Date startInterval = new Date(endInterval.getTime() - (INTERVAL)); + + System.out.println("Start interval: " + startInterval.toString()); + System.out.println("End interval: " + endInterval.toString()); + + System.out.println("Creating RAMDirectory index..."); + for (int i=0; i<100000; i++) { + long newInterval = Math.round(r.nextDouble() * INTERVAL); + + Date curDate = new Date(startInterval.getTime() + newInterval); + String dateStr = DateTools.dateToString(curDate, Resolution.MINUTE); + + Document document = new Document(); + + document.add(new Field("id", String.valueOf(i), Store.YES, Index.NO)); + document.add(new Field("date", dateStr, Store.YES, Index.UN_TOKENIZED)); + + writer.addDocument(document); + } + + writer.optimize(); + writer.close(); + + IndexReader reader = IndexReader.open(ramDir); + + System.out.println("Reader opened with " + reader.maxDoc() + " documents. Creating RangeFilters..."); + + long s = System.currentTimeMillis(); + for (int i=0; i<1000; i++) { + // Generate random date interval + Date date1 = new Date(startInterval.getTime() + Math.round(r.nextDouble() * INTERVAL)); + Date date2 = new Date(startInterval.getTime() + Math.round(r.nextDouble() * INTERVAL)); + + String start, end; + if (date1.after(date2)) { + start = DateTools.dateToString(date2, Resolution.MINUTE); + end = DateTools.dateToString(date1, Resolution.MINUTE); + } else { + start = DateTools.dateToString(date1, Resolution.MINUTE); + end = DateTools.dateToString(date2, Resolution.MINUTE); + } + + RangeFilter filter = new RangeFilter("date", start, end, true, true); + filter.bits(reader); + } + long e = System.currentTimeMillis() - s; + System.out.println("Standard RangeFilter finished in " + e + "ms"); + + new MemoryCachedRangeFilter("date", 0, 1, true, true).warmup(reader); + s = System.currentTimeMillis(); + for (int i=0; i<1000; i++) { + // Generate random date interval + Date date1 = new Date(startInterval.getTime() + Math.round(r.nextDouble() * INTERVAL)); + Date date2 = new Date(startInterval.getTime() + Math.round(r.nextDouble() * INTERVAL)); + + String start, end; + if (date1.after(date2)) { + start = DateTools.dateToString(date2, Resolution.MINUTE); + end = DateTools.dateToString(date1, Resolution.MINUTE); + } else { + start = DateTools.dateToString(date1, Resolution.MINUTE); + end = DateTools.dateToString(date2, Resolution.MINUTE); + } + + MemoryCachedRangeFilter filter = new MemoryCachedRangeFilter("date", Long.parseLong(start), Long.parseLong(end), true, true); + filter.bits(reader); + } + + e = System.currentTimeMillis() - s; + System.out.println("MemoryCachedRangeFilter inished in " + e + "ms"); + } + +} Index: src/test/org/apache/lucene/search/TestMemoryCachedRangeFilter.java =================================================================== --- src/test/org/apache/lucene/search/TestMemoryCachedRangeFilter.java (revision 0) +++ src/test/org/apache/lucene/search/TestMemoryCachedRangeFilter.java (revision 0) @@ -0,0 +1,108 @@ +package org.apache.lucene.search; + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BaseTestRangeFilter; +import org.apache.lucene.search.Hits; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.RangeFilter; +import org.apache.lucene.search.TermQuery; + + +/** + * Bulk of the code lifted from TestRangeFilter + */ +public class TestMemoryCachedRangeFilter extends BaseTestRangeFilter { + public TestMemoryCachedRangeFilter(String name) { + super(name); + } + + public TestMemoryCachedRangeFilter() { + super(); + } + + public void testRangeFilterId() throws IOException { + IndexReader reader = IndexReader.open(index); + IndexSearcher search = new IndexSearcher(reader); + + int medId = ((maxId - minId) / 2); + + long minIP = minId; + long maxIP = maxId; + long medIP = medId; + + int numDocs = reader.numDocs(); + + assertEquals("num of docs", numDocs, 1 + maxId - minId); + + Hits result; + Query q = new TermQuery(new Term("body", "body")); + + // test id, bounded on both ends + + result = search.search(q, new MemoryCachedRangeFilter("id", minIP, maxIP, T, T)); + assertEquals("find all", numDocs, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", minIP, maxIP, T, F)); + assertEquals("all but last", numDocs - 1, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", minIP, maxIP, F, T)); + assertEquals("all but first", numDocs - 1, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", minIP, maxIP, F, F)); + assertEquals("all but ends", numDocs - 2, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", medIP, maxIP, T, T)); + assertEquals("med and up", 1 + maxId - medId, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", minIP, medIP, T, T)); + assertEquals("up to med", 1 + medId - minId, result.length()); + + // unbounded id + + result = search.search(q, new MemoryCachedRangeFilter("id", minIP, Long.MAX_VALUE, T, F)); + assertEquals("min and up", numDocs, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", Long.MIN_VALUE, maxIP, F, T)); + assertEquals("max and down", numDocs, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", minIP, Long.MAX_VALUE, F, F)); + assertEquals("not min, but up", numDocs - 1, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", Long.MIN_VALUE, maxIP, F, F)); + assertEquals("not max, but down", numDocs - 1, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", medIP, maxIP, T, F)); + assertEquals("med and up, not max", maxId - medId, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", minIP, medIP, F, T)); + assertEquals("not min, up to med", medId - minId, result.length()); + + // very small sets + + result = search.search(q, new MemoryCachedRangeFilter("id", minIP, minIP, F, F)); + assertEquals("min,min,F,F", 0, result.length()); + result = search.search(q, new MemoryCachedRangeFilter("id", medIP, medIP, F, F)); + assertEquals("med,med,F,F", 0, result.length()); + result = search.search(q, new MemoryCachedRangeFilter("id", maxIP, maxIP, F, F)); + assertEquals("max,max,F,F", 0, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", minIP, minIP, T, T)); + assertEquals("min,min,T,T", 1, result.length()); + result = search.search(q, new MemoryCachedRangeFilter("id", Long.MIN_VALUE, minIP, F, T)); + assertEquals("nul,min,F,T", 1, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", maxIP, maxIP, T, T)); + assertEquals("max,max,T,T", 1, result.length()); + result = search.search(q, new MemoryCachedRangeFilter("id", maxIP, Long.MAX_VALUE, T, F)); + assertEquals("max,nul,T,T", 1, result.length()); + + result = search.search(q, new MemoryCachedRangeFilter("id", medIP, medIP, T, T)); + assertEquals("med,med,T,T", 1, result.length()); + + } + +} Index: src/java/org/apache/lucene/search/MemoryCachedRangeFilter.java =================================================================== --- src/java/org/apache/lucene/search/MemoryCachedRangeFilter.java (revision 0) +++ src/java/org/apache/lucene/search/MemoryCachedRangeFilter.java (revision 0) @@ -0,0 +1,273 @@ +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.WeakHashMap; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.Filter; + +/** + * Caches numeric field values in memory to improve range filter performance. During a warmup phase, which happens + * once for every field this Filter is applied to, numeric values are read from the index, sorted, and stored in + * a SortedFieldCache. When the filter is used, binary searches are used to find where the upper and lower + * bounds values are indexed within the SortedFieldCache. + * + * MemoryCachedRangeFilter can be used where range filter performance is critical and memory consumption is not + * an issue. Memory requirements: (sizeof(int) + sizeof(long)) * numDocs. Warmup requires creation of FieldCache + * as well. + * + * @author Andy Liu + */ +public class MemoryCachedRangeFilter extends Filter { +// private static WeakHashMap> cache = new WeakHashMap>(); + private static WeakHashMap cache = new WeakHashMap(); + + private String field; + private long lower; + private long upper; + private boolean includeLower; + private boolean includeUpper; + + private ValueParser parser = new ValueParser() { + public long parse(String str) { + return Long.parseLong(str); + } + }; + + /** + * Constructs a MemoryCachedRangeFilter + * + * @param field The field this range applies to + * @param lower The lower bound on this range + * @param upper The upper bound on this range + * @param includeLower Is the lower bound value inclusive? + * @param includeUpper Is the upper bound value inclusive? + */ + public MemoryCachedRangeFilter(String field, long lower, long upper, boolean includeLower, boolean includeUpper) { + super(); + this.field = field; + this.lower = lower; + this.upper = upper; + this.includeLower = includeLower; + this.includeUpper = includeUpper; + } + + /** + * Reads all values of a given field and creates a SortedFieldCache. Stores SortedFieldCache + * in memory. Public method, so that warmup can be performed during application intialization. + * This can take a while for large document sets. + * + * @param reader + * @throws IOException + */ + public synchronized void warmup(IndexReader reader) throws IOException { + // Check to see if warmup was already performed. If so, then return. + if (getFromCache(reader) != null) + return; + + String[] fieldCache = FieldCache.DEFAULT.getStrings(reader, field); + List tuples = new ArrayList(); + for (int docId=0; docId innerCache = cache.get(reader); + Map innerCache = (Map) cache.get(reader); + if (innerCache == null) { + innerCache = new HashMap(); + cache.put(reader, innerCache); + } + + innerCache.put(field, sfCache); + } + + private SortedFieldCache getFromCache(IndexReader reader) { + Map innerCache = (Map) cache.get(reader); + + if (innerCache == null) { + return null; + } + + return (SortedFieldCache) innerCache.get(field); + } + + public BitSet bits(IndexReader reader) throws IOException { + SortedFieldCache sfCache = null; + + synchronized (this) { + sfCache = getFromCache(reader); + + if (sfCache == null) { + warmup(reader); + sfCache = getFromCache(reader); + } + } + + int lowerIndex = 0; + + // If we're asking for a value lower than the minimum value, set index to 0 + if (lower < sfCache.values[0]) { + lowerIndex = 0; + } else { + // Otherwise, use binary search to look for index of lower + lowerIndex = Arrays.binarySearch(sfCache.values, lower); + + // Binary search doesn't take in account series of repeated values. So the index + // returned can be in the middle of a series. Depending on whether or not this + // bound is inclusive, we will have to adjust to point to the first of the series + // or the last of the series. + if ( (includeLower) && (lowerIndex >= 0) ) { + lowerIndex = rewind(sfCache.values, lowerIndex); + } else if ( (!includeLower) && (lowerIndex >= 0) ) { + lowerIndex = forward(sfCache.values, lowerIndex); + lowerIndex++; + } else { + lowerIndex = Math.abs(lowerIndex); + } + } + + int upperIndex = 0; + + // If we're asking for a value higher than the maximum value, set index to length-1 + if (upper > sfCache.values[sfCache.values.length-1]) { + upperIndex = sfCache.values.length-1; + } else { + upperIndex = Arrays.binarySearch(sfCache.values, upper); + if ( (includeUpper) && (upperIndex >= 0) ) { + upperIndex = forward(sfCache.values, upperIndex); + } else if ( (!includeUpper) && (upperIndex >= 0) ) { + upperIndex = rewind(sfCache.values, upperIndex); + upperIndex--; + } else { + upperIndex = Math.abs(upperIndex); + + // binarySearch returns where the value *should* be. It will be pointing to a + // value in the array greater than the value that we're looking for. Rewind. + upperIndex--; + } + } + + // Special case handling may cause pointer to run off the array. Adjust. + if (upperIndex >= sfCache.values.length) + upperIndex = sfCache.values.length-1; + if (lowerIndex < 0) + lowerIndex = 0; + + // Construct bitset. The complete range of documents is all docId values + // between lowerIndex and upperIndex. + BitSet bitset = new BitSet(); + for (int i=lowerIndex; i<=upperIndex; i++) { + bitset.set(sfCache.docId[i]); + } + + return bitset; + } + + /** + * Rewinds pointer to the first of a series of repeated values + * + * @param values + * @param beginIndex + * @return + */ + private int rewind(long[] values, int beginIndex) { + long beginValue = values[beginIndex]; + while ( (beginIndex > 0) && (beginValue == values[beginIndex-1]) ) { + beginIndex--; + } + return beginIndex; + } + + /** + * Moves forward pointer to the last of a series of repeated values + * + * @param values + * @param beginIndex + * @return + */ + private int forward(long[] values, int beginIndex) { + long beginValue = values[beginIndex]; + while ( (beginIndex < values.length-1) && (beginValue == values[beginIndex+1]) ) { + beginIndex++; + } + return beginIndex; + } + + public String toString() { + StringBuffer buffer = new StringBuffer(); + buffer.append(field); + buffer.append(":"); + buffer.append(includeLower ? "[" : "{"); + buffer.append(lower); + buffer.append("-"); + buffer.append(upper); + buffer.append(includeUpper ? "]" : "}"); + return buffer.toString(); + } + + public ValueParser getParser() { + return parser; + } + + public void setParser(ValueParser parser) { + this.parser = parser; + } + + /** + * Parallel arrays to stare [value, docId] pairs. Sorted by value + */ + static class SortedFieldCache { + long[] values; + int[] docId; + + public SortedFieldCache(List tuples) { + values = new long[tuples.size()]; + docId = new int[tuples.size()]; + + Collections.sort(tuples); + for (int i=0; i