Index: src/test/org/apache/lucene/search/TestSort.java =================================================================== --- src/test/org/apache/lucene/search/TestSort.java (revision 162003) +++ src/test/org/apache/lucene/search/TestSort.java (working copy) @@ -344,12 +344,9 @@ HashMap scoresA = getScores (full.search (queryA)); // we'll test searching locally, remote and multi - // note: the multi test depends on each separate index containing - // the same documents as our local index, so the computed normalization - // will be the same. so we make a multi searcher over two equal document - // sets - not realistic, but necessary for testing. MultiSearcher remote = new MultiSearcher (new Searchable[] { getRemote() }); - MultiSearcher multi = new MultiSearcher (new Searchable[] { full, full }); + // multi is not really multi, but tests the MultiSearcher handling + MultiSearcher multi = new MultiSearcher (new Searchable[] { full }); // change sorting and make sure relevancy stays the same Index: src/java/org/apache/lucene/search/RemoteSearchable.java =================================================================== --- src/java/org/apache/lucene/search/RemoteSearchable.java (revision 162003) +++ src/java/org/apache/lucene/search/RemoteSearchable.java (working copy) @@ -44,6 +44,11 @@ local.search(query, filter, results); } + public void search(Weight weight, Filter filter, HitCollector results) + throws IOException { + local.search(weight, filter, results); + } + public void close() throws IOException { local.close(); } @@ -52,6 +57,11 @@ return local.docFreq(term); } + + public int[] docFreqs(Term[] terms) throws IOException { + return local.docFreqs(terms); + } + public int maxDoc() throws IOException { return local.maxDoc(); } @@ -60,11 +70,20 @@ return local.search(query, filter, n); } + public TopDocs search(Weight weight, Filter filter, int n) throws IOException { + return local.search(weight, filter, n); + } + public TopFieldDocs search (Query query, Filter filter, int n, Sort sort) throws IOException { return local.search (query, filter, n, sort); } + public TopFieldDocs search (Weight weight, Filter filter, int n, Sort sort) + throws IOException { + return local.search (weight, filter, n, sort); + } + public Document doc(int i) throws IOException { return local.doc(i); } @@ -77,6 +96,10 @@ return local.explain(query, doc); } + public Explanation explain(Weight weight, int doc) throws IOException { + return local.explain(weight, doc); + } + /** Exports a searcher for the index in args[0] named * "//localhost/Searchable". */ public static void main(String args[]) throws Exception { Index: src/java/org/apache/lucene/search/Similarity.java =================================================================== --- src/java/org/apache/lucene/search/Similarity.java (revision 162003) +++ src/java/org/apache/lucene/search/Similarity.java (working copy) @@ -17,6 +17,7 @@ */ import java.io.IOException; +import java.io.Serializable; import java.util.Collection; import java.util.Iterator; @@ -84,7 +85,7 @@ * @see IndexWriter#setSimilarity(Similarity) * @see Searcher#setSimilarity(Similarity) */ -public abstract class Similarity { +public abstract class Similarity implements Serializable { /** The Similarity implementation used by default. */ private static Similarity defaultImpl = new DefaultSimilarity(); Index: src/java/org/apache/lucene/search/Searcher.java =================================================================== --- src/java/org/apache/lucene/search/Searcher.java (revision 162003) +++ src/java/org/apache/lucene/search/Searcher.java (working copy) @@ -18,6 +18,8 @@ import java.io.IOException; +import org.apache.lucene.index.Term; + /** An abstract base class for search implementations. * Implements some common utility methods. */ @@ -93,4 +95,15 @@ public Similarity getSimilarity() { return this.similarity; } + + + // inherit javadoc + public int[] docFreqs(Term[] terms) throws IOException { + int[] result = new int[terms.length]; + for (int i = 0; i < terms.length; i++) { + result[i] = docFreq(terms[i]); + } + return result; + } + } Index: src/java/org/apache/lucene/search/spans/SpanQuery.java =================================================================== --- src/java/org/apache/lucene/search/spans/SpanQuery.java (revision 162003) +++ src/java/org/apache/lucene/search/spans/SpanQuery.java (working copy) @@ -37,7 +37,7 @@ /** Returns a collection of all terms matched by this query.*/ public abstract Collection getTerms(); - protected Weight createWeight(Searcher searcher) { + protected Weight createWeight(Searcher searcher) throws IOException { return new SpanWeight(this, searcher); } Index: src/java/org/apache/lucene/search/spans/SpanWeight.java =================================================================== --- src/java/org/apache/lucene/search/spans/SpanWeight.java (revision 162003) +++ src/java/org/apache/lucene/search/spans/SpanWeight.java (working copy) @@ -32,7 +32,7 @@ import org.apache.lucene.search.Similarity; class SpanWeight implements Weight { - private Searcher searcher; + private Similarity similarity; private float value; private float idf; private float queryNorm; @@ -41,17 +41,19 @@ private Collection terms; private SpanQuery query; - public SpanWeight(SpanQuery query, Searcher searcher) { - this.searcher = searcher; + public SpanWeight(SpanQuery query, Searcher searcher) + throws IOException { + this.similarity = query.getSimilarity(searcher); this.query = query; this.terms = query.getTerms(); + + idf = this.query.getSimilarity(searcher).idf(terms, searcher); } public Query getQuery() { return query; } public float getValue() { return value; } public float sumOfSquaredWeights() throws IOException { - idf = this.query.getSimilarity(searcher).idf(terms, searcher); queryWeight = idf * query.getBoost(); // compute query weight return queryWeight * queryWeight; // square it } @@ -64,7 +66,7 @@ public Scorer scorer(IndexReader reader) throws IOException { return new SpanScorer(query.getSpans(reader), this, - query.getSimilarity(searcher), + similarity, reader.norms(query.getField())); } @@ -81,7 +83,7 @@ Term term = (Term)i.next(); docFreqs.append(term.text()); docFreqs.append("="); - docFreqs.append(searcher.docFreq(term)); + docFreqs.append(reader.docFreq(term)); if (i.hasNext()) { docFreqs.append(" "); Index: src/java/org/apache/lucene/search/MultiPhraseQuery.java =================================================================== --- src/java/org/apache/lucene/search/MultiPhraseQuery.java (revision 162003) +++ src/java/org/apache/lucene/search/MultiPhraseQuery.java (working copy) @@ -107,27 +107,30 @@ } private class MultiPhraseWeight implements Weight { - private Searcher searcher; + private Similarity similarity; private float value; private float idf; private float queryNorm; private float queryWeight; - public MultiPhraseWeight(Searcher searcher) { - this.searcher = searcher; - } + public MultiPhraseWeight(Searcher searcher) + throws IOException { + this.similarity = getSimilarity(searcher); - public Query getQuery() { return MultiPhraseQuery.this; } - public float getValue() { return value; } - - public float sumOfSquaredWeights() throws IOException { + // compute idf Iterator i = termArrays.iterator(); while (i.hasNext()) { Term[] terms = (Term[])i.next(); - for (int j=0; j"+filter; Index: src/java/org/apache/lucene/search/Searchable.java =================================================================== --- src/java/org/apache/lucene/search/Searchable.java (revision 162003) +++ src/java/org/apache/lucene/search/Searchable.java (working copy) @@ -48,10 +48,19 @@ * @param filter if non-null, a bitset used to eliminate some documents * @param results to receive hits * @throws BooleanQuery.TooManyClauses + * + * @deprecated */ void search(Query query, Filter filter, HitCollector results) throws IOException; + /** Expert: Low-level search implementation. + * Identical to {@link #search(Query, Filter, HitCollector)}, but takes + * a Weight instead of a query. + */ + void search(Weight weight, Filter filter, HitCollector results) + throws IOException; + /** Frees resources associated with this Searcher. * Be careful not to call this method while you are still using objects * like {@link Hits}. @@ -64,6 +73,12 @@ */ int docFreq(Term term) throws IOException; + /** Expert: For each term in the terms array, calculates the number of + * documents containing term. Returns an array with these + * document frequencies. Used to minimize number of remote calls. + */ + int[] docFreqs(Term[] terms) throws IOException; + /** Expert: Returns one greater than the largest possible document number. * Called by search code to compute term weights. * @see IndexReader#maxDoc() @@ -78,9 +93,17 @@ *

Applications should usually call {@link Searcher#search(Query)} or * {@link Searcher#search(Query,Filter)} instead. * @throws BooleanQuery.TooManyClauses + * + * @deprecated */ TopDocs search(Query query, Filter filter, int n) throws IOException; + /** Expert: Low-level search implementation. + * Identical to {@link #search(Query, Filter, int)}, but takes + * a Weight instead of a query. + */ + TopDocs search(Weight weight, Filter filter, int n) throws IOException; + /** Expert: Returns the stored fields of document i. * Called by {@link HitCollector} implementations. * @see IndexReader#document(int) @@ -103,6 +126,12 @@ */ Explanation explain(Query query, int doc) throws IOException; + /** + * Identical to {@link #search(Query, Filter, HitCollector)}, but takes + * a Weight instead of a query. + */ + Explanation explain(Weight weight, int doc) throws IOException; + /** Expert: Low-level search implementation with arbitrary sorting. Finds * the top n hits for query, applying * filter if non-null, and sorting the hits by the criteria in @@ -111,7 +140,16 @@ *

Applications should usually call {@link * Searcher#search(Query,Filter,Sort)} instead. * @throws BooleanQuery.TooManyClauses + * + * @deprecated */ TopFieldDocs search(Query query, Filter filter, int n, Sort sort) throws IOException; + + /** Expert: Low-level search implementation. + * Identical to {@link #search(Query, Filter, int, Sort)}, but takes + * a Weight instead of a query. + */ + TopFieldDocs search(Weight weight, Filter filter, int n, Sort sort) + throws IOException; } Index: src/java/org/apache/lucene/search/Query.java =================================================================== --- src/java/org/apache/lucene/search/Query.java (revision 162003) +++ src/java/org/apache/lucene/search/Query.java (working copy) @@ -20,6 +20,7 @@ import java.util.HashSet; import java.util.Iterator; +import java.util.Set; import org.apache.lucene.index.IndexReader; @@ -81,7 +82,7 @@ * *

Only implemented by primitive queries, which re-write to themselves. */ - protected Weight createWeight(Searcher searcher) { + protected Weight createWeight(Searcher searcher) throws IOException { throw new UnsupportedOperationException(); } @@ -107,6 +108,19 @@ * {@link #createWeight(Searcher)} implementatation. */ public Query combine(Query[] queries) { + for (int i = 0; i < queries.length; i++) { + if (!this.equals(queries[i])) { + throw new IllegalArgumentException(); + } + } + return this; + } + + /** + * Expert: adds all terms occuring in this query to the terms set + */ + public void extractTerms(Set terms) { + // needs to be implemented by query subclasses throw new UnsupportedOperationException(); } Index: src/java/org/apache/lucene/search/BooleanQuery.java =================================================================== --- src/java/org/apache/lucene/search/BooleanQuery.java (revision 162003) +++ src/java/org/apache/lucene/search/BooleanQuery.java (working copy) @@ -17,6 +17,8 @@ */ import java.io.IOException; +import java.util.Iterator; +import java.util.Set; import java.util.Vector; import java.util.Arrays; @@ -153,16 +155,16 @@ } private class BooleanWeight implements Weight { - protected Searcher searcher; + protected Similarity similarity; protected Vector weights = new Vector(); - public BooleanWeight(Searcher searcher) { - this.searcher = searcher; + public BooleanWeight(Searcher searcher) + throws IOException { + this.similarity = getSimilarity(searcher); for (int i = 0 ; i < clauses.size(); i++) { BooleanClause c = (BooleanClause)clauses.elementAt(i); weights.add(c.getQuery().createWeight(searcher)); } - //System.out.println("Creating " + getClass().getName()); } public Query getQuery() { return BooleanQuery.this; } @@ -213,7 +215,7 @@ if (allRequired && noneBoolean) { // ConjunctionScorer is okay ConjunctionScorer result = - new ConjunctionScorer(getSimilarity(searcher)); + new ConjunctionScorer(similarity); for (int i = 0 ; i < weights.size(); i++) { Weight w = (Weight)weights.elementAt(i); Scorer subScorer = w.scorer(reader); @@ -225,7 +227,7 @@ } // Use good-old BooleanScorer instead. - BooleanScorer result = new BooleanScorer(getSimilarity(searcher)); + BooleanScorer result = new BooleanScorer(similarity); for (int i = 0 ; i < weights.size(); i++) { BooleanClause c = (BooleanClause)clauses.elementAt(i); @@ -269,7 +271,7 @@ if (coord == 1) // only one clause matched sumExpl = sumExpl.getDetails()[0]; // eliminate wrapper - float coordFactor = getSimilarity(searcher).coord(coord, maxCoord); + float coordFactor = similarity.coord(coord, maxCoord); if (coordFactor == 1.0f) // coord is no-op return sumExpl; // eliminate wrapper else { @@ -286,13 +288,16 @@ private class BooleanWeight2 extends BooleanWeight { /* Merge into BooleanWeight in case the 1.4 BooleanScorer is dropped */ - public BooleanWeight2(Searcher searcher) { super(searcher); } + public BooleanWeight2(Searcher searcher) + throws IOException { + super(searcher); + } /** @return An alternative Scorer that uses and provides skipTo(), * and scores documents in document number order. */ public Scorer scorer(IndexReader reader) throws IOException { - BooleanScorer2 result = new BooleanScorer2(getSimilarity(searcher)); + BooleanScorer2 result = new BooleanScorer2(similarity); for (int i = 0 ; i < weights.size(); i++) { BooleanClause c = (BooleanClause)clauses.elementAt(i); @@ -319,7 +324,7 @@ return useScorer14; } - protected Weight createWeight(Searcher searcher) { + protected Weight createWeight(Searcher searcher) throws IOException { return getUseScorer14() ? (Weight) new BooleanWeight(searcher) : (Weight) new BooleanWeight2(searcher); } @@ -358,7 +363,19 @@ return this; // no clauses rewrote } + // inherit javadoc + public void extractTerms(Set terms) { + for (Iterator i = clauses.iterator(); i.hasNext();) { + BooleanClause clause = (BooleanClause) i.next(); + clause.getQuery().extractTerms(terms); + } + } + // inherit javadoc + public Query combine(Query[] queries) { + return Query.mergeBooleanQueries(queries); + } + public Object clone() { BooleanQuery clone = (BooleanQuery)super.clone(); clone.clauses = (Vector)this.clauses.clone(); Index: src/java/org/apache/lucene/search/PhraseQuery.java =================================================================== --- src/java/org/apache/lucene/search/PhraseQuery.java (revision 162003) +++ src/java/org/apache/lucene/search/PhraseQuery.java (working copy) @@ -17,6 +17,7 @@ */ import java.io.IOException; +import java.util.Set; import java.util.Vector; import org.apache.lucene.index.Term; @@ -100,14 +101,17 @@ } private class PhraseWeight implements Weight { - private Searcher searcher; + private Similarity similarity; private float value; private float idf; private float queryNorm; private float queryWeight; - public PhraseWeight(Searcher searcher) { - this.searcher = searcher; + public PhraseWeight(Searcher searcher) + throws IOException { + this.similarity = getSimilarity(searcher); + + idf = similarity.idf(terms, searcher); } public String toString() { return "weight(" + PhraseQuery.this + ")"; } @@ -116,7 +120,6 @@ public float getValue() { return value; } public float sumOfSquaredWeights() throws IOException { - idf = getSimilarity(searcher).idf(terms, searcher); queryWeight = idf * getBoost(); // compute query weight return queryWeight * queryWeight; // square it } @@ -140,11 +143,11 @@ } if (slop == 0) // optimize exact case - return new ExactPhraseScorer(this, tps, getPositions(), getSimilarity(searcher), + return new ExactPhraseScorer(this, tps, getPositions(), similarity, reader.norms(field)); else return - new SloppyPhraseScorer(this, tps, getPositions(), getSimilarity(searcher), slop, + new SloppyPhraseScorer(this, tps, getPositions(), similarity, slop, reader.norms(field)); } @@ -168,7 +171,7 @@ docFreqs.append(term.text()); docFreqs.append("="); - docFreqs.append(searcher.docFreq(term)); + docFreqs.append(reader.docFreq(term)); query.append(term.text()); } @@ -228,7 +231,7 @@ } } - protected Weight createWeight(Searcher searcher) { + protected Weight createWeight(Searcher searcher) throws IOException { if (terms.size() == 1) { // optimize one-term case Term term = (Term)terms.elementAt(0); Query termQuery = new TermQuery(term); @@ -238,6 +241,12 @@ return new PhraseWeight(searcher); } + /** + * @see org.apache.lucene.search.Query#extractTerms(java.util.Set) + */ + public void extractTerms(Set queryTerms) { + queryTerms.addAll(terms); + } /** Prints a user-readable version of this query. */ public String toString(String f) { Index: src/java/org/apache/lucene/search/TermQuery.java =================================================================== --- src/java/org/apache/lucene/search/TermQuery.java (revision 162003) +++ src/java/org/apache/lucene/search/TermQuery.java (working copy) @@ -17,6 +17,8 @@ */ import java.io.IOException; +import java.util.Set; + import org.apache.lucene.index.Term; import org.apache.lucene.index.TermDocs; import org.apache.lucene.index.IndexReader; @@ -28,14 +30,17 @@ private Term term; private class TermWeight implements Weight { + private Similarity similarity; private Searcher searcher; private float value; private float idf; private float queryNorm; private float queryWeight; - public TermWeight(Searcher searcher) { - this.searcher = searcher; + public TermWeight(Searcher searcher) + throws IOException { + this.similarity = getSimilarity(searcher); + idf = similarity.idf(term, searcher); // compute idf } public String toString() { return "weight(" + TermQuery.this + ")"; } @@ -44,7 +49,6 @@ public float getValue() { return value; } public float sumOfSquaredWeights() throws IOException { - idf = getSimilarity(searcher).idf(term, searcher); // compute idf queryWeight = idf * getBoost(); // compute query weight return queryWeight * queryWeight; // square it } @@ -52,16 +56,16 @@ public void normalize(float queryNorm) { this.queryNorm = queryNorm; queryWeight *= queryNorm; // normalize query weight - value = queryWeight * idf; // idf for document + value = queryWeight * idf; // idf for document } public Scorer scorer(IndexReader reader) throws IOException { TermDocs termDocs = reader.termDocs(term); - + if (termDocs == null) return null; - - return new TermScorer(this, termDocs, getSimilarity(searcher), + + return new TermScorer(this, termDocs, similarity, reader.norms(term.field())); } @@ -72,7 +76,7 @@ result.setDescription("weight("+getQuery()+" in "+doc+"), product of:"); Explanation idfExpl = - new Explanation(idf, "idf(docFreq=" + searcher.docFreq(term) + ")"); + new Explanation(idf, "idf(docFreq=" + reader.docFreq(term) + ")"); // explain query weight Explanation queryExpl = new Explanation(); @@ -82,16 +86,16 @@ if (getBoost() != 1.0f) queryExpl.addDetail(boostExpl); queryExpl.addDetail(idfExpl); - + Explanation queryNormExpl = new Explanation(queryNorm,"queryNorm"); queryExpl.addDetail(queryNormExpl); - + queryExpl.setValue(boostExpl.getValue() * idfExpl.getValue() * queryNormExpl.getValue()); result.addDetail(queryExpl); - + // explain field weight String field = term.field(); Explanation fieldExpl = new Explanation(); @@ -113,7 +117,7 @@ fieldExpl.setValue(tfExpl.getValue() * idfExpl.getValue() * fieldNormExpl.getValue()); - + result.addDetail(fieldExpl); // combine them @@ -134,10 +138,14 @@ /** Returns the term of this query. */ public Term getTerm() { return term; } - protected Weight createWeight(Searcher searcher) { + protected Weight createWeight(Searcher searcher) throws IOException { return new TermWeight(searcher); } + public void extractTerms(Set terms) { + terms.add(getTerm()); + } + /** Prints a user-readable version of this query. */ public String toString(String field) { StringBuffer buffer = new StringBuffer(); Index: src/java/org/apache/lucene/search/MultiSearcher.java =================================================================== --- src/java/org/apache/lucene/search/MultiSearcher.java (revision 162003) +++ src/java/org/apache/lucene/search/MultiSearcher.java (working copy) @@ -17,6 +17,10 @@ */ import java.io.IOException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; import org.apache.lucene.document.Document; import org.apache.lucene.index.Term; @@ -27,6 +31,93 @@ * or {@link #search(Query,Filter)} methods. */ public class MultiSearcher extends Searcher { + /** + * Document Frequency cache acting as a Dummy-Searcher. + * This class is no full-fledged Searcher, but only supports + * the methods necessary to initialize Weights. + */ + private static class CachedDfSource extends Searcher { + private Map dfMap; // Map from Terms to corresponding doc freqs + private int maxDoc; // document count + + public CachedDfSource(Map dfMap, int maxDoc) { + this.dfMap = dfMap; + this.maxDoc = maxDoc; + } + + public int docFreq(Term term) { + int df; + try { + df = ((Integer) dfMap.get(term)).intValue(); + } catch (NullPointerException e) { + throw new IllegalArgumentException("df for term " + term.text() + + " not available"); + } + return df; + } + + public int[] docFreqs(Term[] terms) throws IOException { + int[] result = new int[terms.length]; + for (int i = 0; i < terms.length; i++) { + result[i] = docFreq(terms[i]); + } + return result; + } + + public int maxDoc() { + return maxDoc; + } + + public Query rewrite(Query query) throws IOException { + // this is a bit of a hack. We know that a query which + // creates a Weight based on this Dummy-Searcher is + // always already rewritten (see preparedWeight()). + // Therefore we just return the unmodified query here + return query; + } + + public void close() throws IOException { + throw new UnsupportedOperationException(); + } + + public Document doc(int i) throws IOException{ + throw new UnsupportedOperationException(); + } + + public Explanation explain(Query query,int doc) throws IOException{ + throw new UnsupportedOperationException(); + } + + public Explanation explain(Weight weight,int doc) throws IOException { + throw new UnsupportedOperationException(); + } + + public void search(Query query, Filter filter, HitCollector results) throws IOException { + throw new UnsupportedOperationException(); + } + + public void search(Weight weight, Filter filter, HitCollector results) throws IOException { + throw new UnsupportedOperationException(); + } + + public TopDocs search(Query query,Filter filter,int n) throws IOException { + throw new UnsupportedOperationException(); + } + + public TopDocs search(Weight weight,Filter filter,int n) throws IOException { + throw new UnsupportedOperationException(); + } + + public TopFieldDocs search(Query query,Filter filter,int n,Sort sort) throws IOException { + throw new UnsupportedOperationException(); + } + + public TopFieldDocs search(Weight weight,Filter filter,int n,Sort sort) throws IOException { + throw new UnsupportedOperationException(); + } + }; + + private Searchable[] searchables; private int[] starts; private int maxDoc = 0; @@ -114,11 +205,18 @@ public TopDocs search(Query query, Filter filter, int nDocs) throws IOException { + Weight weight = prepareWeight(query); + return search(weight, filter, nDocs); + } + + public TopDocs search(Weight weight, Filter filter, int nDocs) + throws IOException { + HitQueue hq = new HitQueue(nDocs); int totalHits = 0; for (int i = 0; i < searchables.length; i++) { // search each searcher - TopDocs docs = searchables[i].search(query, filter, nDocs); + TopDocs docs = searchables[i].search(weight, filter, nDocs); totalHits += docs.totalHits; // update totalHits ScoreDoc[] scoreDocs = docs.scoreDocs; for (int j = 0; j < scoreDocs.length; j++) { // merge scoreDocs into hq @@ -139,11 +237,17 @@ public TopFieldDocs search (Query query, Filter filter, int n, Sort sort) throws IOException { + Weight weight = prepareWeight(query); + return search(weight, filter, n, sort); + } + + public TopFieldDocs search (Weight weight, Filter filter, int n, Sort sort) + throws IOException { FieldDocSortedHitQueue hq = null; int totalHits = 0; for (int i = 0; i < searchables.length; i++) { // search each searcher - TopFieldDocs docs = searchables[i].search (query, filter, n, sort); + TopFieldDocs docs = searchables[i].search (weight, filter, n, sort); if (hq == null) hq = new FieldDocSortedHitQueue (docs.fields, n); totalHits += docs.totalHits; // update totalHits ScoreDoc[] scoreDocs = docs.scoreDocs; @@ -166,11 +270,18 @@ // inherit javadoc public void search(Query query, Filter filter, final HitCollector results) throws IOException { + Weight weight = prepareWeight(query); + search(weight, filter, results); + } + + // inherit javadoc + public void search(Weight weight, Filter filter, final HitCollector results) + throws IOException { for (int i = 0; i < searchables.length; i++) { final int start = starts[i]; - searchables[i].search(query, filter, new HitCollector() { + searchables[i].search(weight, filter, new HitCollector() { public void collect(int doc, float score) { results.collect(doc + start, score); } @@ -184,12 +295,62 @@ for (int i = 0; i < searchables.length; i++) { queries[i] = searchables[i].rewrite(original); } - return original.combine(queries); + return queries[0].combine(queries); } public Explanation explain(Query query, int doc) throws IOException { + Weight weight = prepareWeight(query); + return explain(weight, doc); + } + + + public Explanation explain(Weight weight, int doc) throws IOException { int i = subSearcher(doc); // find searcher index - return searchables[i].explain(query,doc-starts[i]); // dispatch to searcher + return searchables[i].explain(weight,doc-starts[i]); // dispatch to searcher } + /** + * Distributed query processing is done in the following steps: + * 1. rewrite query + * 2. extract necessary terms + * 3. collect dfs for these terms from the Searchables + * 4. create query weight using aggregate dfs. + * 5. distribute that weight to Searchables + * 6. merge results + * + * Steps 1-4 are done here, 5+6 in the search() methods + * + * @return rewritten queries + */ + private Weight prepareWeight(Query original) throws IOException { + // step 1 + Query rewrittenQuery = rewrite(original); + + // step 2 + Set terms = new HashSet(); + rewrittenQuery.extractTerms(terms); + + // step3 + Term[] allTermsArray = new Term[terms.size()]; + terms.toArray(allTermsArray); + int[] aggregatedDfs = new int[terms.size()]; + for (int i = 0; i < searchables.length; i++) { + int[] dfs = searchables[i].docFreqs(allTermsArray); + for(int j=0; j 0"); - Scorer scorer = query.weight(this).scorer(reader); + Scorer scorer = weight.scorer(reader); if (scorer == null) return new TopDocs(0, new ScoreDoc[0]); @@ -124,7 +130,14 @@ public TopFieldDocs search(Query query, Filter filter, final int nDocs, Sort sort) throws IOException { - Scorer scorer = query.weight(this).scorer(reader); + return search(query.weight(this), filter, nDocs, sort); + } + + // inherit javadoc + public TopFieldDocs search(Weight weight, Filter filter, final int nDocs, + Sort sort) + throws IOException { + Scorer scorer = weight.scorer(reader); if (scorer == null) return new TopFieldDocs(0, new ScoreDoc[0], sort.fields); @@ -153,6 +166,12 @@ // inherit javadoc public void search(Query query, Filter filter, final HitCollector results) throws IOException { + search(query.weight(this), filter, results); + } + + // inherit javadoc + public void search(Weight weight, Filter filter, + final HitCollector results) throws IOException { HitCollector collector = results; if (filter != null) { final BitSet bits = filter.bits(reader); @@ -165,7 +184,7 @@ }; } - Scorer scorer = query.weight(this).scorer(reader); + Scorer scorer = weight.scorer(reader); if (scorer == null) return; scorer.score(collector); @@ -181,7 +200,10 @@ } public Explanation explain(Query query, int doc) throws IOException { - return query.weight(this).explain(reader, doc); + return explain(query.weight(this), doc); } + public Explanation explain(Weight weight, int doc) throws IOException { + return weight.explain(reader, doc); + } } Index: src/java/org/apache/lucene/search/ParallelMultiSearcher.java =================================================================== --- src/java/org/apache/lucene/search/ParallelMultiSearcher.java (revision 162003) +++ src/java/org/apache/lucene/search/ParallelMultiSearcher.java (working copy) @@ -42,10 +42,7 @@ * TODO: parallelize this one too */ public int docFreq(Term term) throws IOException { - int docFreq = 0; - for (int i = 0; i < searchables.length; i++) - docFreq += searchables[i].docFreq(term); - return docFreq; + return super.docFreq(term); } /** @@ -53,7 +50,7 @@ * Searchable, waits for each search to complete and merge * the results back together. */ - public TopDocs search(Query query, Filter filter, int nDocs) + public TopDocs search(Weight weight, Filter filter, int nDocs) throws IOException { HitQueue hq = new HitQueue(nDocs); int totalHits = 0; @@ -64,7 +61,7 @@ msta[i] = new MultiSearcherThread( searchables[i], - query, + weight, filter, nDocs, hq, @@ -101,7 +98,7 @@ * Searchable, waits for each search to complete and merges * the results back together. */ - public TopFieldDocs search(Query query, Filter filter, int nDocs, Sort sort) + public TopFieldDocs search(Weight weight, Filter filter, int nDocs, Sort sort) throws IOException { // don't specify the fields - we'll wait to do this until we get results FieldDocSortedHitQueue hq = new FieldDocSortedHitQueue (null, nDocs); @@ -112,7 +109,7 @@ msta[i] = new MultiSearcherThread( searchables[i], - query, + weight, filter, nDocs, hq, @@ -181,11 +178,7 @@ * @see org.apache.lucene.search.Searchable#rewrite(org.apache.lucene.search.Query) */ public Query rewrite(Query original) throws IOException { - Query[] queries = new Query[searchables.length]; - for (int i = 0; i < searchables.length; i++) { - queries[i] = searchables[i].rewrite(original); - } - return original.combine(queries); + return super.rewrite(original); } } @@ -196,7 +189,7 @@ class MultiSearcherThread extends Thread { private Searchable searchable; - private Query query; + private Weight weight; private Filter filter; private int nDocs; private TopDocs docs; @@ -208,7 +201,7 @@ public MultiSearcherThread( Searchable searchable, - Query query, + Weight weight, Filter filter, int nDocs, HitQueue hq, @@ -217,7 +210,7 @@ String name) { super(name); this.searchable = searchable; - this.query = query; + this.weight = weight; this.filter = filter; this.nDocs = nDocs; this.hq = hq; @@ -227,7 +220,7 @@ public MultiSearcherThread( Searchable searchable, - Query query, + Weight weight, Filter filter, int nDocs, FieldDocSortedHitQueue hq, @@ -237,7 +230,7 @@ String name) { super(name); this.searchable = searchable; - this.query = query; + this.weight = weight; this.filter = filter; this.nDocs = nDocs; this.hq = hq; @@ -248,8 +241,8 @@ public void run() { try { - docs = (sort == null) ? searchable.search (query, filter, nDocs) - : searchable.search (query, filter, nDocs, sort); + docs = (sort == null) ? searchable.search (weight, filter, nDocs) + : searchable.search (weight, filter, nDocs, sort); } // Store the IOException for later use by the caller of this thread catch (IOException ioe) { Index: src/java/org/apache/lucene/search/PhrasePrefixQuery.java =================================================================== --- src/java/org/apache/lucene/search/PhrasePrefixQuery.java (revision 162003) +++ src/java/org/apache/lucene/search/PhrasePrefixQuery.java (working copy) @@ -108,27 +108,30 @@ } private class PhrasePrefixWeight implements Weight { - private Searcher searcher; + private Similarity similarity; private float value; private float idf; private float queryNorm; private float queryWeight; - public PhrasePrefixWeight(Searcher searcher) { - this.searcher = searcher; - } + public PhrasePrefixWeight(Searcher searcher) + throws IOException { + this.similarity = getSimilarity(searcher); - public Query getQuery() { return PhrasePrefixQuery.this; } - public float getValue() { return value; } - - public float sumOfSquaredWeights() throws IOException { + // compute idf Iterator i = termArrays.iterator(); while (i.hasNext()) { Term[] terms = (Term[])i.next(); - for (int j=0; j