diff -urN src0/org/meresco/lucene/CachingKeyCollector.java src/org/meresco/lucene/CachingKeyCollector.java --- src0/org/meresco/lucene/CachingKeyCollector.java 1970-01-01 01:00:00.000000000 +0100 +++ src/org/meresco/lucene/CachingKeyCollector.java 2013-10-18 09:10:37.247091057 +0200 @@ -0,0 +1,189 @@ +/* begin license * + * + * "Meresco Lucene" is a set of components and tools to integrate Lucene (based on PyLucene) into Meresco + * + * Copyright (C) 2013 Seecr (Seek You Too B.V.) http://seecr.nl + * Copyright (C) 2013 Stichting Bibliotheek.nl (BNL) http://www.bibliotheek.nl + * + * This file is part of "Meresco Lucene" + * + * "Meresco Lucene" is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * "Meresco Lucene" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "Meresco Lucene"; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * end license */ + +package org.meresco.lucene; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; +import java.util.Map; +import java.util.WeakHashMap; + +import org.apache.lucene.facet.collections.LRUHashMap; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.search.CollectionTerminatedException; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.OpenBitSet; + +/** + * A KeyCollector for implementing joins between two or more Lucene indexs. This + * version caches the keys and the filters. + * + * First use this collector to collect keys from a field containing integers + * keys in a NumericDocValues field. Then get a Filter with getFilter() and use + * this to filter on keys in another index. + * + * @author erik@seecr.nl + * */ +public class CachingKeyCollector extends KeyCollector { + + /** + * Create a collector that collects keys from a field. It caches the keys on + * segment level. The collector itself is also cached, as well as its + * Filters. + * + * @param query + * The Query this collector will be used with. + * @param keyName + * The name of the field that contains the keys. This field must + * refer to a NumericDocValues field containing integer keys. + */ + public static CachingKeyCollector create(Query query, String keyName) { + LRUHashMap collectorCache = CachingKeyCollector.queryCache.get(query); + if (collectorCache == null) { + collectorCache = new LRUHashMap(5); + CachingKeyCollector.queryCache.put(query, collectorCache); + } + + CachingKeyCollector keyCollector = collectorCache.get(keyName); + if (keyCollector == null) { + keyCollector = new CachingKeyCollector(keyName); + collectorCache.put(keyName, keyCollector); + } + keyCollector.reset(); + return keyCollector; + } + + /** + * Get a BitSet containing all the collected keys. This returns a new + * BitSet. Use {@link getFilter} with + * {@link org.apache.lucene.queries.BooleanFilter} or + * {@link org.apache.lucene.queries.ChainedFilter} for combining + * results from different collectors to do cross-filtering. + */ + @Override + public OpenBitSet getCollectedKeys() { + updateCache(); + OpenBitSet keySet = new OpenBitSet(); + for (OpenBitSet b : this.seen) + keySet.or(b); + return keySet; + } + + /** + * Create a Lucene Filter for filtering docs based on a key in this + * KeyCollector. It can be used for any index, as long as the keyName is + * correct. + * + * @param keyName + * The name of the field containing the keys to match. This field + * must refer to a NumericDocValues field containing integer + * keys. + * @return A Lucene Filter object. It is cached by this KeyCollector and it + * caches the docIds of the index is is applied to. + */ + public KeyFilter getFilter(String keyName) { + KeyFilter filter = this.keyFilterCache.get(keyName); + if (filter == null) { + filter = new KeyFilter(this, keyName); + this.keyFilterCache.put(keyName, filter); + } + filter.reset(); + return filter; + } + + @Override + public void setNextReader(AtomicReaderContext context) throws IOException { + updateCache(); + Object readerKey = context.reader().getCombinedCoreAndDeletesKey(); + OpenBitSet bitSet = this.keySetCache.get(readerKey); + if (bitSet != null) { + this.seen.add(bitSet); + throw new CollectionTerminatedException(); // already have this one + } + super.setNextReader(context); + this.currentReaderKey = readerKey; + this.keySet = new OpenBitSet(); + } + + public void printKeySetCacheSize() { + int size = 0; + for (OpenBitSet b : this.keySetCache.values()) + size += b.size(); + System.out.print("cache: " + this.keySetCache.size() + " entries, " + (size / 8 / 1024 / 1024) + " MB"); + } + + /** + * Caches KeyCollectors for (Query,keyName) pairs. KeyCollectors can become + * large, tens of MB. + */ + private static LRUHashMap> queryCache = new LRUHashMap>(10); + + /** + * Caches bitsets (containing keys) for specific readers within one index. + * Entries disappear when readers are closed and garbage collected. Note + * that keys are quasi-randomly distributed, so all bitsets are of the same + * size, and the whole cache may become large. + */ + private Map keySetCache = new WeakHashMap(); + + /** + * Caches Lucene Filters for for different keys. The filter can be used on + * any index as long as the keyName refers to te proper field containing the + * keys, so this cache may contain readers from different indexes. + */ + private Map keyFilterCache = new LRUHashMap(5); + + /** + * Records which BitSets belong to the result of the most recent collect + * cycle (The cache might contain more; for readers not yet GC'd. + */ + private List seen = new ArrayList(); + + /** + * This indicates the reader for which collection is going on. At the end, + * this key is used to populate the cache. + */ + private Object currentReaderKey = null; + + private CachingKeyCollector(String keyName) { + super(keyName); + } + + private void reset() { + this.currentReaderKey = null; + this.seen.clear(); + } + + private void updateCache() { + if (this.currentReaderKey != null) { + this.keySet.trimTrailingZeros(); + this.keySetCache.put(this.currentReaderKey, this.keySet); + this.seen.add(this.keySet); + this.currentReaderKey = null; + } + } +} diff -urN src0/org/meresco/lucene/KeyCollector.java src/org/meresco/lucene/KeyCollector.java --- src0/org/meresco/lucene/KeyCollector.java 1970-01-01 01:00:00.000000000 +0100 +++ src/org/meresco/lucene/KeyCollector.java 2013-10-18 09:10:37.247091057 +0200 @@ -0,0 +1,68 @@ +/* begin license * + * + * "Meresco Lucene" is a set of components and tools to integrate Lucene (based on PyLucene) into Meresco + * + * Copyright (C) 2013 Seecr (Seek You Too B.V.) http://seecr.nl + * Copyright (C) 2013 Stichting Bibliotheek.nl (BNL) http://www.bibliotheek.nl + * + * This file is part of "Meresco Lucene" + * + * "Meresco Lucene" is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * "Meresco Lucene" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "Meresco Lucene"; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * end license */ + +package org.meresco.lucene; + +import java.io.IOException; + +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.util.OpenBitSet; + +public class KeyCollector extends Collector { + + private String keyName; + private NumericDocValues keyValues; + protected OpenBitSet keySet = new OpenBitSet(); + + public KeyCollector(String keyName) { + this.keyName = keyName; + } + + @Override + public void collect(int docId) throws IOException { + this.keySet.set((int)this.keyValues.get(docId)); + } + + @Override + public void setNextReader(AtomicReaderContext context) throws IOException { + this.keyValues = context.reader().getNumericDocValues(this.keyName); + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + + @Override + public void setScorer(Scorer scorer) throws IOException { + } + + public OpenBitSet getCollectedKeys() { + return this.keySet; + } +} diff -urN src0/org/meresco/lucene/KeyFilter.java src/org/meresco/lucene/KeyFilter.java --- src0/org/meresco/lucene/KeyFilter.java 1970-01-01 01:00:00.000000000 +0100 +++ src/org/meresco/lucene/KeyFilter.java 2013-10-18 09:22:16.984788931 +0200 @@ -0,0 +1,112 @@ +/* begin license * + * + * "Meresco Lucene" is a set of components and tools to integrate Lucene (based on PyLucene) into Meresco + * + * Copyright (C) 2013 Seecr (Seek You Too B.V.) http://seecr.nl + * Copyright (C) 2013 Stichting Bibliotheek.nl (BNL) http://www.bibliotheek.nl + * + * This file is part of "Meresco Lucene" + * + * "Meresco Lucene" is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * "Meresco Lucene" is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with "Meresco Lucene"; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * end license */ + +package org.meresco.lucene; + +import java.io.IOException; +import java.util.BitSet; +import java.util.Map; +import java.util.WeakHashMap; + +import org.apache.lucene.index.AtomicReader; +import org.apache.lucene.index.AtomicReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.Filter; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.DocIdBitSet; +import org.apache.lucene.util.OpenBitSet; + + +public class KeyFilter extends Filter { + private String keyName; + public OpenBitSet keySet; + private CachingKeyCollector keyCollector; + private Map docSetCache = new WeakHashMap(); + + KeyFilter(CachingKeyCollector cachingKeyCollector, String keyName) { + this.keyCollector = cachingKeyCollector; + this.keyName = keyName; + } + + void reset() { + OpenBitSet keySet = this.keyCollector.getCollectedKeys(); + if (keySet.equals(this.keySet)) + return; + this.keySet = keySet; + this.docSetCache.clear(); + } + + /** + * Intersect this filter with another c.q. perform a logical 'and'. This is + * much faster than using both filters in a BooleanFilter. Note that this + * changes the internal state until it is reset, which happens when it is + * (re-)pulled from the cache. + * + * @param f + * The other KeyFilter that will be intersected. It is not + * changed. + */ + public void intersect(KeyFilter f) { + this.keySet.and(f.keySet); + } + + /** + * Create a union of this filter and another. This performs an logical 'or'. + * Avoid BooleanFilter and use this method instead, as it is almost three + * times faster. + * + * @param f + * The other KeyFilter that will be united. It is not changed. + */ + public void unite(KeyFilter f) { + this.keySet.or(f.keySet); + + } + + @Override + public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs) throws IOException { + AtomicReader reader = context.reader(); + Object coreKey = reader.getCoreCacheKey(); + DocIdSet docSet = this.docSetCache.get(coreKey); + if (docSet == null) { + docSet = this.createDocIdSet(reader); + this.docSetCache.put(coreKey, docSet); + } + return docSet; + } + + private DocIdSet createDocIdSet(AtomicReader reader) throws IOException { + NumericDocValues keyValues = reader.getNumericDocValues(this.keyName); + if (keyValues == null) + throw new RuntimeException("no keys found for field " + this.keyName); + OpenBitSet docBitSet = new OpenBitSet(); + for (int docId = 0; docId < reader.maxDoc(); docId++) + if (this.keySet.get((int) keyValues.get(docId))) + docBitSet.set(docId); + docBitSet.trimTrailingZeros(); + return docBitSet; + } +}