Index: /Users/matt/code/lucene/src/java/org/apache/lucene/search/FilterManager.java =================================================================== --- /Users/matt/code/lucene/src/java/org/apache/lucene/search/FilterManager.java (revision 0) +++ /Users/matt/code/lucene/src/java/org/apache/lucene/search/FilterManager.java (revision 0) @@ -0,0 +1,194 @@ +package org.apache.lucene.search; + +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; +import java.util.TreeSet; + +/** + * Singleton Cache that will save filters + * Can be used by the RemoteCachingWrapperFilter or just used to save filters for reuse + * It will allow users to use caching when using RMI + * because it keeps the cache on the seaercher side of the RMI connection + * + * Also could be used as a persistent storage for any filter + * As long as the filter provides a proper hasCode() + * Since I use hashCode() as the key + * @author Matt Ericson + * + */ + +public class FilterManager { + + protected static FilterManager manager; + + protected static final String CLEAN_SIZE = "filterManagerCacheSize"; + + protected Map cache; + protected int cacheCleanSize; //If the cache is bigger then this we need to clean it + protected long cleanSleepTime; + protected FilterCleaner filterCleaner; + + public static FilterManager getInstance() { + if (manager == null) { + manager = new FilterManager(); + } + return manager; + } + + /** + * This will set up the FilterManager + * This needs to be a singleton + * + * The user can provide a system property filterManagerCacheSize=100 + * To change the default cache size + * + */ + protected FilterManager() { + cache = new HashMap(); + Properties p = System.getProperties(); + cacheCleanSize = getInt(p,CLEAN_SIZE, 100); + cleanSleepTime = 1000 * 60 * 10; // 10 minutes between Cleanings + + filterCleaner = new FilterCleaner(); + Thread fcThread = new Thread(filterCleaner); + //I set this to be a Daemon so I dont have to stop it + fcThread.setDaemon(true); + fcThread.start(); + } + + /** + * Will return a cached version of the filter. It will allow the Client + * To pass up a small filter but this will keep a persistent version around + * And allow the caching filter to do its job + * + * @param filter The input filter + * @return The cached version of the filter + */ + public Filter getFilter(Filter filter) { + synchronized(cache) { + FilterItem f = null; + f = (FilterItem)cache.get(new Integer(filter.hashCode())); + if (f != null) { + f.timestamp = new Date().getTime(); + return f.filter; + } + cache.put(new Integer(filter.hashCode()), new FilterItem(filter)); + return filter; + } + } + + /** + * Helper method that will allow someone to override the size of the cache + * @param p + * @param key + * @param defaultValue + * @return + */ + private int getInt(Properties p, String key, int defaultValue) { + String value = p.getProperty(key); + if (value != null) { + try { + return Integer.parseInt(value); + } catch(NumberFormatException ex) { + //NO OP + } + + } + return defaultValue; + } + + /** + * Will hold the filter and the time the filter was used + * So that I can make a thread to LRU out filters that are not used + * When I can use java 1.5 this will go away + * + * @author Matt Ericson + */ + protected class FilterItem { + public Filter filter; + public long timestamp; + + public FilterItem (Filter filter) { + this.filter = filter; + this.timestamp = new Date().getTime(); + } + } + + + + /** + * This class will keep the cache from getting too big + * If we were using Java 1.5 I could use LinkedHashMap and we would not need this thread + * to clean out the cache + * + * @author Matt Ericson + * + */ + protected class FilterCleaner implements Runnable { + + private boolean running = true; + private TreeSet set; + + public FilterCleaner() { + set = new TreeSet(new Comparator() { + public int compare(Object a, Object b) { + if( a instanceof Map.Entry && b instanceof Map.Entry) { + FilterItem fia = (FilterItem) ((Map.Entry)a).getValue(); + FilterItem fib = (FilterItem) ((Map.Entry)b).getValue(); + if ( fia.timestamp == fib.timestamp ) { + return 0; + } + //We want the smaller timestamp to be first + if ( fia.timestamp < fib.timestamp ) { + return -1; + } + //We want the larger timestamp to be last + //if ( fia.timestamp > fib.timestamp ) + //No need to do the if + return 1; + + } else { + throw new ClassCastException("Objects are not Map.Entry"); + } + } + }); + } + + public void run () { + while (running) { + + //The set will sort items from Oldest to newest + //We delete the oldest filters + if (cache.size() > cacheCleanSize) { + //The first thing I do is empty the set + set.clear(); + synchronized (cache) { + set.addAll(cache.entrySet()); + Iterator it = set.iterator(); + int numToDelete = (int) ((cache.size() - cacheCleanSize) * 1.5); + int counter = 0; + //Now I loop over the set and delete all of the cache entries + //that have not been used in a while + while (it.hasNext() && counter++ < numToDelete) { + Map.Entry entry = (Map.Entry)it.next(); + cache.remove(entry.getKey()); + } + } + //Now that we are done lets empty the set So we dont tie up the memory + set.clear(); + } + //We are done of the cache is not too big so we sleep + try { + Thread.sleep(cleanSleepTime); + } catch (InterruptedException e) { + //Dont do anything we just keep going + } + } + } + + } +} Property changes on: /Users/matt/code/lucene/src/java/org/apache/lucene/search/FilterManager.java ___________________________________________________________________ Name: svn:eol-style + native Index: /Users/matt/code/lucene/src/java/org/apache/lucene/search/RemoteCachingWrapperFilter.java =================================================================== --- /Users/matt/code/lucene/src/java/org/apache/lucene/search/RemoteCachingWrapperFilter.java (revision 0) +++ /Users/matt/code/lucene/src/java/org/apache/lucene/search/RemoteCachingWrapperFilter.java (revision 0) @@ -0,0 +1,39 @@ +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.BitSet; + +import org.apache.lucene.index.IndexReader; + +/** + * Will provide a filter cache. It will cache any filter used + * Since all of the filter caching is done in hash maps we need to reuse filters to get + * any caching to work. This will cache a filter based on is .hashCode() and + * if it sees the same filter twice will reuse the original version + * + * This will work with {@link CachingWrapperFilter} to + * keep the cache on the remote searcher + * + * To cache a result you must do something like + * RemoteCachingWrapperFilter f = new RemoteCachingWrapperFilter(new achingWrapperFilter(myFilter)); + * + * @author Matt Ericson + * + */ +public class RemoteCachingWrapperFilter extends Filter { + protected Filter filter; + + public RemoteCachingWrapperFilter(Filter filter) { + this.filter = filter; + } + + /** + * Will use the RemoteFilterManager to keep the cache for a filter on the + * searcher side of a remote connection + */ + public BitSet bits(IndexReader reader) throws IOException { + //We use the Singleton To get the real Filter Manager + Filter cachedFilter = FilterManager.getInstance().getFilter(filter); + return cachedFilter.bits(reader); + } +} Property changes on: /Users/matt/code/lucene/src/java/org/apache/lucene/search/RemoteCachingWrapperFilter.java ___________________________________________________________________ Name: svn:eol-style + native Index: /Users/matt/code/lucene/src/test/org/apache/lucene/search/DebugRemoteCachingWrapperFilter.java =================================================================== --- /Users/matt/code/lucene/src/test/org/apache/lucene/search/DebugRemoteCachingWrapperFilter.java (revision 0) +++ /Users/matt/code/lucene/src/test/org/apache/lucene/search/DebugRemoteCachingWrapperFilter.java (revision 0) @@ -0,0 +1,39 @@ +package org.apache.lucene.search; + +import java.io.IOException; +import java.util.BitSet; + +import junit.framework.TestCase; + +import org.apache.lucene.index.IndexReader; + +/** + * This is written to test the RemoteCachingWrapperFilter It will assert that it is working correctly + * And will make sure that the caching will work on server side + * @author Matt Ericson + * + */ +public class DebugRemoteCachingWrapperFilter extends RemoteCachingWrapperFilter { + + boolean isFirstTime; + + public DebugRemoteCachingWrapperFilter(Filter filter, boolean isFirstTime) { + super(filter); + this.isFirstTime = isFirstTime; + // TODO Auto-generated constructor stub + } + + public BitSet bits(IndexReader reader) throws IOException { + //We use the Singleton To get the real Filter Manager + Filter cachedFilter = FilterManager.getInstance().getFilter(this.filter); + + TestCase.assertNotNull("Filter should not be null" , cachedFilter); + if (isFirstTime) { + TestCase.assertSame("First time filter should be the same ", filter ,cachedFilter); + } else { + TestCase.assertNotSame("We should have a cached version of the filter",filter ,cachedFilter); + } + return cachedFilter.bits(reader); + } + +} Property changes on: /Users/matt/code/lucene/src/test/org/apache/lucene/search/DebugRemoteCachingWrapperFilter.java ___________________________________________________________________ Name: svn:eol-style + native Index: /Users/matt/code/lucene/src/test/org/apache/lucene/search/TestRemoteCachingWrapperFilter.java =================================================================== --- /Users/matt/code/lucene/src/test/org/apache/lucene/search/TestRemoteCachingWrapperFilter.java (revision 0) +++ /Users/matt/code/lucene/src/test/org/apache/lucene/search/TestRemoteCachingWrapperFilter.java (revision 0) @@ -0,0 +1,104 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import junit.framework.TestCase; +import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.document.*; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.RAMDirectory; + +import java.io.IOException; +import java.rmi.Naming; +import java.rmi.registry.LocateRegistry; +import java.util.BitSet; +import java.util.Collections; +import java.util.Set; +import java.util.HashSet; + +/** + * @version $Id: $ + * NOTE This is copied from Test RemoteSearchable since it already had a remote index set up + * Then will test that the index is cached on the searcher side of things + */ +public class TestRemoteCachingWrapperFilter extends TestCase { + public TestRemoteCachingWrapperFilter(String name) { + super(name); + } + + private static Searchable getRemote() throws Exception { + try { + return lookupRemote(); + } catch (Throwable e) { + startServer(); + return lookupRemote(); + } + } + + private static Searchable lookupRemote() throws Exception { + return (Searchable)Naming.lookup("//localhost/Searchable"); + } + + private static void startServer() throws Exception { + // construct an index + RAMDirectory indexStore = new RAMDirectory(); + IndexWriter writer = new IndexWriter(indexStore,new SimpleAnalyzer(),true); + Document doc = new Document(); + doc.add(new Field("test", "test text", Field.Store.YES, Field.Index.TOKENIZED)); + doc.add(new Field("type", "A", Field.Store.YES, Field.Index.TOKENIZED)); + doc.add(new Field("other", "other test text", Field.Store.YES, Field.Index.TOKENIZED)); + writer.addDocument(doc); + //Need a second document to search for + doc = new Document(); + doc.add(new Field("test", "test text", Field.Store.YES, Field.Index.TOKENIZED)); + doc.add(new Field("type", "B", Field.Store.YES, Field.Index.TOKENIZED)); + doc.add(new Field("other", "other test text", Field.Store.YES, Field.Index.TOKENIZED)); + writer.addDocument(doc); + writer.optimize(); + writer.close(); + + // publish it + LocateRegistry.createRegistry(1099); + Searchable local = new IndexSearcher(indexStore); + RemoteSearchable impl = new RemoteSearchable(local); + Naming.rebind("//localhost/Searchable", impl); + } + + private static void search(Query query, Filter filter) throws Exception { + Searchable[] searchables = { getRemote() }; + Searcher searcher = new MultiSearcher(searchables); + Hits result = searcher.search(query,filter); + assertEquals(1, result.length()); + Document document = result.doc(0); + assertTrue("document is null and it shouldn't be", document != null); + assertEquals("A", document.get("type")); + assertTrue("document.getFields() Size: " + document.getFields().size() + " is not: " + 3, document.getFields().size() == 3); + } + + + public void testTermRemoteFilter() throws Exception { + Filter filter = new DebugRemoteCachingWrapperFilter(new QueryFilter(new TermQuery(new Term("type", "a"))), true); + search(new TermQuery(new Term("test", "test")) , filter ); + filter = new DebugRemoteCachingWrapperFilter(new QueryFilter(new TermQuery(new Term("type", "a"))),false); + search(new TermQuery(new Term("test", "test")) , filter ); + } + + +} Property changes on: /Users/matt/code/lucene/src/test/org/apache/lucene/search/TestRemoteCachingWrapperFilter.java ___________________________________________________________________ Name: svn:eol-style + native