Index: src/test/org/apache/lucene/search/CachingWrapperFilterHelper.java =================================================================== --- src/test/org/apache/lucene/search/CachingWrapperFilterHelper.java (revision 0) +++ src/test/org/apache/lucene/search/CachingWrapperFilterHelper.java (revision 0) @@ -0,0 +1,96 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.IndexReader; +import java.util.BitSet; +import java.util.WeakHashMap; +import java.util.Map; +import java.io.IOException; + +import junit.framework.TestCase; + +/** + * This version is only here to test that the cache is REALLY being cached + * + * Wraps another filter's result and caches it. The caching + * behavior is like {@link QueryFilter}. The purpose is to allow + * filters to simply filter, and then wrap with this class to add + * caching, keeping the two concerns decoupled yet composable. + */ +public class CachingWrapperFilterHelper extends Filter { + private Filter filter; + + /** + * @todo What about serialization in RemoteSearchable? Caching won't work. + * Should transient be removed? + */ + private transient Map cache; + + private boolean shouldHaveCache = false; + + /** + * @param filter Filter to cache results of + */ + public CachingWrapperFilterHelper(Filter filter) { + this.filter = filter; + } + + public void setShouldHaveCache(boolean shouldHaveCache) { + this.shouldHaveCache = shouldHaveCache; + } + + public BitSet bits(IndexReader reader) throws IOException { + if (cache == null) { + cache = new WeakHashMap(); + } + + synchronized (cache) { // check cache + BitSet cached = (BitSet) cache.get(reader); + if (shouldHaveCache) { + TestCase.assertNotNull("Cache should have data ", cached); + } else { + TestCase.assertNull("Cache should be null " + cached , cached); + } + if (cached != null) { + return cached; + } + } + + final BitSet bits = filter.bits(reader); + + synchronized (cache) { // update cache + cache.put(reader, bits); + } + + return bits; + } + + public String toString() { + return "CachingWrapperFilterHelper("+filter+")"; + } + + public boolean equals(Object o) { + if (!(o instanceof CachingWrapperFilterHelper)) return false; + return this.filter.equals(((CachingWrapperFilterHelper)o)); + } + + public int hashCode() { + return filter.hashCode() ^ 0x1117BF25; + } +} Property changes on: src/test/org/apache/lucene/search/CachingWrapperFilterHelper.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/test/org/apache/lucene/search/RemoteCachingWrapperFilterHelper.java =================================================================== --- src/test/org/apache/lucene/search/RemoteCachingWrapperFilterHelper.java (revision 0) +++ src/test/org/apache/lucene/search/RemoteCachingWrapperFilterHelper.java (revision 0) @@ -0,0 +1,64 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.BitSet; + +import junit.framework.TestCase; + +import org.apache.lucene.index.IndexReader; + +/** + * This is written to test the RemoteCachingWrapperFilter It will assert that it is working correctly + * And will make sure that the caching will work on server side + * @author Matt Ericson + * + */ +public class RemoteCachingWrapperFilterHelper extends RemoteCachingWrapperFilter { + + boolean shouldHaveCache; + + public RemoteCachingWrapperFilterHelper(Filter filter, boolean shouldHaveCache) { + super(filter); + this.shouldHaveCache = shouldHaveCache; + } + + public void shouldHaveCache(boolean shouldHaveCache) { + this.shouldHaveCache = shouldHaveCache; + } + + public BitSet bits(IndexReader reader) throws IOException { + //We use the Singleton To get the real Filter Manager + Filter cachedFilter = FilterManager.getInstance().getFilter(this.filter); + + + TestCase.assertNotNull("Filter should not be null" , cachedFilter); + if (!shouldHaveCache) { + TestCase.assertSame("First time filter should be the same ", filter ,cachedFilter); + } else { + TestCase.assertNotSame("We should have a cached version of the filter",filter ,cachedFilter); + } + + if (filter instanceof CachingWrapperFilterHelper) { + ((CachingWrapperFilterHelper)cachedFilter).setShouldHaveCache(shouldHaveCache); + } + return cachedFilter.bits(reader); + } + +} Property changes on: src/test/org/apache/lucene/search/RemoteCachingWrapperFilterHelper.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/test/org/apache/lucene/search/TestRemoteCachingWrapperFilter.java =================================================================== --- src/test/org/apache/lucene/search/TestRemoteCachingWrapperFilter.java (revision 0) +++ src/test/org/apache/lucene/search/TestRemoteCachingWrapperFilter.java (revision 0) @@ -0,0 +1,117 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import junit.framework.TestCase; +import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.document.*; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.RAMDirectory; + +import java.io.IOException; +import java.rmi.Naming; +import java.rmi.registry.LocateRegistry; +import java.util.BitSet; +import java.util.Collections; +import java.util.Set; +import java.util.HashSet; + +/** + * @version $Id: $ + * NOTE This is copied from Test RemoteSearchable since it already had a remote index set up + * Then will test that the index is cached on the searcher side of things + */ +public class TestRemoteCachingWrapperFilter extends TestCase { + public TestRemoteCachingWrapperFilter(String name) { + super(name); + } + + private static Searchable getRemote() throws Exception { + try { + return lookupRemote(); + } catch (Throwable e) { + startServer(); + return lookupRemote(); + } + } + + private static Searchable lookupRemote() throws Exception { + return (Searchable)Naming.lookup("//localhost/Searchable"); + } + + private static void startServer() throws Exception { + // construct an index + RAMDirectory indexStore = new RAMDirectory(); + IndexWriter writer = new IndexWriter(indexStore,new SimpleAnalyzer(),true); + Document doc = new Document(); + doc.add(new Field("test", "test text", Field.Store.YES, Field.Index.TOKENIZED)); + doc.add(new Field("type", "A", Field.Store.YES, Field.Index.TOKENIZED)); + doc.add(new Field("other", "other test text", Field.Store.YES, Field.Index.TOKENIZED)); + writer.addDocument(doc); + //Need a second document to search for + doc = new Document(); + doc.add(new Field("test", "test text", Field.Store.YES, Field.Index.TOKENIZED)); + doc.add(new Field("type", "B", Field.Store.YES, Field.Index.TOKENIZED)); + doc.add(new Field("other", "other test text", Field.Store.YES, Field.Index.TOKENIZED)); + writer.addDocument(doc); + writer.optimize(); + writer.close(); + + // publish it + LocateRegistry.createRegistry(1099); + Searchable local = new IndexSearcher(indexStore); + RemoteSearchable impl = new RemoteSearchable(local); + Naming.rebind("//localhost/Searchable", impl); + } + + private static void search(Query query, Filter filter) throws Exception { + Searchable[] searchables = { getRemote() }; + Searcher searcher = new MultiSearcher(searchables); + Hits result = searcher.search(query,filter); + assertEquals(1, result.length()); + Document document = result.doc(0); + assertTrue("document is null and it shouldn't be", document != null); + assertEquals("A", document.get("type")); + assertTrue("document.getFields() Size: " + document.getFields().size() + " is not: " + 3, document.getFields().size() == 3); + } + + + public void testTermRemoteFilter() throws Exception { + CachingWrapperFilterHelper f1 = new CachingWrapperFilterHelper(new QueryFilter(new TermQuery(new Term("type", "a")))); + + //NOTE this is what we are fixing if the users just uses a CachingWrapperFilterHelper it will never + //Have any cache + f1.setShouldHaveCache(false); + search(new TermQuery(new Term("test", "test")) , f1 ); + f1.setShouldHaveCache(false); + search(new TermQuery(new Term("test", "test")) , f1 ); + + //Bu Adding the RemoteCachingWrapperFilterHelper it will cache in the searcher side + + RemoteCachingWrapperFilterHelper filter = new RemoteCachingWrapperFilterHelper(f1,false); + search(new TermQuery(new Term("test", "test")) , filter ); + //filter = new RemoteCachingWrapperFilterHelper(new QueryFilter(new TermQuery(new Term("type", "a"))),false); + filter.shouldHaveCache(true); + search(new TermQuery(new Term("test", "test")) , filter ); + + } + + +} Property changes on: src/test/org/apache/lucene/search/TestRemoteCachingWrapperFilter.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/search/RemoteCachingWrapperFilter.java =================================================================== --- src/java/org/apache/lucene/search/RemoteCachingWrapperFilter.java (revision 0) +++ src/java/org/apache/lucene/search/RemoteCachingWrapperFilter.java (revision 0) @@ -0,0 +1,56 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.BitSet; + +import org.apache.lucene.index.IndexReader; + +/** + * Will provide a filter cache. It will cache any filter used + * Since all of the filter caching is done in hash maps we need to reuse filters to get + * any caching to work. This will cache a filter based on is .hashCode() and + * if it sees the same filter twice will reuse the original version + * + * This will work with {@link CachingWrapperFilter} to + * keep the cache on the remote searcher + * + * To cache a result you must do something like + * RemoteCachingWrapperFilter f = new RemoteCachingWrapperFilter(new CachingWrapperFilter(myFilter)); + * + * @author Matt Ericson + * + */ +public class RemoteCachingWrapperFilter extends Filter { + protected Filter filter; + + public RemoteCachingWrapperFilter(Filter filter) { + this.filter = filter; + } + + /** + * Will use the RemoteFilterManager to keep the cache for a filter on the + * searcher side of a remote connection + */ + public BitSet bits(IndexReader reader) throws IOException { + //We use the Singleton To get the real Filter Manager + Filter cachedFilter = FilterManager.getInstance().getFilter(filter); + return cachedFilter.bits(reader); + } +} Property changes on: src/java/org/apache/lucene/search/RemoteCachingWrapperFilter.java ___________________________________________________________________ Name: svn:eol-style + native Index: src/java/org/apache/lucene/search/FilterManager.java =================================================================== --- src/java/org/apache/lucene/search/FilterManager.java (revision 0) +++ src/java/org/apache/lucene/search/FilterManager.java (revision 0) @@ -0,0 +1,211 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Comparator; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; +import java.util.TreeSet; + +/** + * Singleton Cache that will save filters + * Can be used by the RemoteCachingWrapperFilter or just used to save filters for reuse + * It will allow users to use caching when using RMI + * because it keeps the cache on the seaercher side of the RMI connection + * + * Also could be used as a persistent storage for any filter + * As long as the filter provides a proper hasCode() + * Since I use hashCode() as the key + * @author Matt Ericson + * + */ + +public class FilterManager { + + protected static FilterManager manager; + + protected static final int DEFAULT_CACHE_CLEAN_SIZE = 100; // + protected static final long DEFAULT_CACHE_SLEEO_TIME = 1000 * 60 * 10; // 10 minutes between Cleanings + + + protected Map cache; + protected int cacheCleanSize; //If the cache is bigger then this we need to clean it + protected long cleanSleepTime; + protected FilterCleaner filterCleaner; + + public synchronized static FilterManager getInstance() { + if (manager == null) { + manager = new FilterManager(); + } + return manager; + } + + /** + * Sets up the FilterManager + * This needs to be a singleton + * + * The user can change the cache size with setCacheSize(int); + * To change the default cache size + * + */ + protected FilterManager() { + cache = new HashMap(); + cacheCleanSize = DEFAULT_CACHE_CLEAN_SIZE; //Only let the cache get to 100 items + cleanSleepTime = DEFAULT_CACHE_SLEEO_TIME; // 10 minutes between Cleanings + + filterCleaner = new FilterCleaner(); + Thread fcThread = new Thread(filterCleaner); + //I set this to be a Daemon so I dont have to stop it + fcThread.setDaemon(true); + fcThread.start(); + } + + /** + * Sets the max size that cache should reach before it is cleaned up + * @param cacheCleanSize + */ + public void setCacheSize(int cacheCleanSize) { + this.cacheCleanSize = cacheCleanSize; + } + + public void setCleanThreadSleepTime (long cleanSleepTime) { + this.cleanSleepTime = cleanSleepTime; + } + + /** + * Returns a cached version of the filter. It will allow the Client + * To pass up a small filter but this will keep a persistent version around + * And allow the caching filter to do its job + * + * @param filter The input filter + * @return The cached version of the filter + */ + public Filter getFilter(Filter filter) { + synchronized(cache) { + FilterItem f = null; + f = (FilterItem)cache.get(new Integer(filter.hashCode())); + if (f != null) { + f.timestamp = new Date().getTime(); + return f.filter; + } + cache.put(new Integer(filter.hashCode()), new FilterItem(filter)); + return filter; + } + } + + /** + * Holds the filter and the time the filter was used + * So that I can make a thread to LRU out filters that are not used + * When I can use java 1.5 this will go away + * + * @author Matt Ericson + */ + protected class FilterItem { + public Filter filter; + public long timestamp; + + public FilterItem (Filter filter) { + this.filter = filter; + this.timestamp = new Date().getTime(); + } + } + + + /** + * Keeps the cache from getting too big + * If we were using Java 1.5 I could use LinkedHashMap and we would not need this thread + * to clean out the cache + * + * The SortedSet sortedFilterItems is only used to sort the items from the cache + * so when its time to clean up we have the TreeSet sort the FilterItems + * + * Removes 1.5 * the numbers of items to make the cache smaller + * If cache clean size is at 10 and they cache is at 15 + * We would remove (15 -10) * 1.5 = 7.5 round up to 8 + * This way we clean the cache out a bit more but the cache cleaner does not run as often + * + * @author Matt Ericson + * + */ + protected class FilterCleaner implements Runnable { + + private boolean running = true; + private TreeSet sortedFilterItems; + + public FilterCleaner() { + sortedFilterItems = new TreeSet(new Comparator() { + public int compare(Object a, Object b) { + if( a instanceof Map.Entry && b instanceof Map.Entry) { + FilterItem fia = (FilterItem) ((Map.Entry)a).getValue(); + FilterItem fib = (FilterItem) ((Map.Entry)b).getValue(); + if ( fia.timestamp == fib.timestamp ) { + return 0; + } + //We want the smaller timestamp to be first + if ( fia.timestamp < fib.timestamp ) { + return -1; + } + //We want the larger timestamp to be last + //if ( fia.timestamp > fib.timestamp ) + //No need to do the if + return 1; + + } else { + throw new ClassCastException("Objects are not Map.Entry"); + } + } + }); + } + + public void run () { + while (running) { + + //The set will sort items from Oldest to newest + //We delete the oldest filters + if (cache.size() > cacheCleanSize) { + //The first thing I do is empty the set + sortedFilterItems.clear(); + synchronized (cache) { + sortedFilterItems.addAll(cache.entrySet()); + Iterator it = sortedFilterItems.iterator(); + int numToDelete = (int) ((cache.size() - cacheCleanSize) * 1.5); + int counter = 0; + //Now I loop over the set and delete all of the cache entries + //that have not been used in a while + while (it.hasNext() && counter++ < numToDelete) { + Map.Entry entry = (Map.Entry)it.next(); + cache.remove(entry.getKey()); + } + } + //Now that we are done lets empty the set So we dont tie up the memory + sortedFilterItems.clear(); + } + //We are done of the cache is not too big so we sleep + try { + Thread.sleep(cleanSleepTime); + } catch (InterruptedException e) { + //Dont do anything we just keep going + } + } + } + + } +} Property changes on: src/java/org/apache/lucene/search/FilterManager.java ___________________________________________________________________ Name: svn:eol-style + native