Index: solr/src/java/org/apache/solr/schema/LatLonType.java =================================================================== --- solr/src/java/org/apache/solr/schema/LatLonType.java (revision 1062897) +++ solr/src/java/org/apache/solr/schema/LatLonType.java (working copy) @@ -325,7 +325,7 @@ @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { return this; } @@ -340,6 +340,7 @@ protected Map lonContext; public SpatialWeight(IndexSearcher searcher) throws IOException { + super(searcher.getDataCache(),SpatialDistanceQuery.this); this.searcher = searcher; this.latContext = latSource.newContext(searcher); this.lonContext = lonSource.newContext(searcher); @@ -347,11 +348,22 @@ lonSource.createWeight(lonContext, searcher); } - @Override - public Query getQuery() { - return SpatialDistanceQuery.this; + protected SpatialWeight(QueryDataCache dataCache, Query query, IndexSearcher searcher, + float queryNorm, float queryWeight, Map latContext, Map lonContext) + { + super(dataCache,query); + this.searcher = searcher; + this.queryNorm = queryNorm; + this.queryWeight = queryWeight; + this.latContext = latContext; + this.lonContext = lonContext; } - + + public Weight makeCopy() + { + return new SpatialWeight(dataCache,theQuery,searcher,queryNorm,queryWeight,latContext,lonContext); + } + @Override public float getValue() { return queryWeight; Index: solr/src/java/org/apache/solr/search/function/FunctionQuery.java =================================================================== --- solr/src/java/org/apache/solr/search/function/FunctionQuery.java (revision 1062897) +++ solr/src/java/org/apache/solr/search/function/FunctionQuery.java (working copy) @@ -52,7 +52,7 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { return this; } @@ -66,16 +66,27 @@ protected Map context; public FunctionWeight(IndexSearcher searcher) throws IOException { + super(searcher.getDataCache(),FunctionQuery.this); this.searcher = searcher; this.context = func.newContext(searcher); func.createWeight(context, searcher); } - @Override - public Query getQuery() { - return FunctionQuery.this; + protected FunctionWeight(QueryDataCache dataCache, Query query, IndexSearcher searcher, + float queryNorm, float queryWeight, Map context) + { + super(dataCache,query); + this.searcher = searcher; + this.queryNorm = queryNorm; + this.queryWeight = queryWeight; + this.context = context; } - + + public Weight makeCopy() + { + return new FunctionWeight(dataCache,theQuery,searcher,queryNorm,queryWeight,context); + } + @Override public float getValue() { return queryWeight; Index: solr/src/java/org/apache/solr/search/function/BoostedQuery.java =================================================================== --- solr/src/java/org/apache/solr/search/function/BoostedQuery.java (revision 1062897) +++ solr/src/java/org/apache/solr/search/function/BoostedQuery.java (working copy) @@ -41,8 +41,8 @@ public Query getQuery() { return q; } public ValueSource getValueSource() { return boostVal; } - public Query rewrite(IndexReader reader) throws IOException { - Query newQ = q.rewrite(reader); + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { + Query newQ = q.cachedRewrite(dataCache,reader); if (newQ == q) return this; BoostedQuery bq = (BoostedQuery)this.clone(); bq.q = newQ; @@ -63,16 +63,27 @@ Map fcontext; public BoostedWeight(IndexSearcher searcher) throws IOException { + super(searcher.getDataCache(),BoostedQuery.this); this.searcher = searcher; this.qWeight = q.weight(searcher); this.fcontext = boostVal.newContext(searcher); boostVal.createWeight(fcontext,searcher); } - public Query getQuery() { - return BoostedQuery.this; + protected BoostedWeight(QueryDataCache dataCache, Query query, + IndexSearcher searcher, Weight qWeight, Map fcontext) + { + super(dataCache,query); + this.searcher = searcher; + this.qWeight = qWeight.makeCopy(); + this.fcontext = fcontext; } - + + public Weight makeCopy() + { + return new BoostedWeight(dataCache,theQuery,searcher,qWeight,fcontext); + } + public float getValue() { return getBoost(); } Index: solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java =================================================================== --- solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java (revision 1062897) +++ solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java (working copy) @@ -61,17 +61,28 @@ private Map context; public ConstantWeight(IndexSearcher searcher) throws IOException { + super(searcher.getDataCache(),SolrConstantScoreQuery.this); this.similarity = searcher.getSimilarity(); this.context = ValueSource.newContext(searcher); if (filter instanceof SolrFilter) ((SolrFilter)filter).createWeight(context, searcher); } - @Override - public Query getQuery() { - return SolrConstantScoreQuery.this; + protected ConstantWeight(QueryDataCache dataCache, Query query, Similarity similarity, + float queryNorm, float queryWeight, Map context) + { + super(dataCache,query); + this.similarity = similarity; + this.queryNorm = queryNorm; + this.queryWeight = queryWeight; + this.context = context; } + public Weight makeCopy() + { + return new ConstantWeight(dataCache,theQuery,similarity,queryNorm,queryWeight,context); + } + @Override public float getValue() { return queryWeight; Index: solr/src/java/org/apache/solr/handler/component/HighlightComponent.java =================================================================== --- solr/src/java/org/apache/solr/handler/component/HighlightComponent.java (revision 1062897) +++ solr/src/java/org/apache/solr/handler/component/HighlightComponent.java (working copy) @@ -113,7 +113,7 @@ if(highlightQuery != null) { boolean rewrite = !(Boolean.valueOf(req.getParams().get(HighlightParams.USE_PHRASE_HIGHLIGHTER, "true")) && Boolean.valueOf(req.getParams().get(HighlightParams.HIGHLIGHT_MULTI_TERM, "true"))); - highlightQuery = rewrite ? highlightQuery.rewrite(req.getSearcher().getIndexReader()) : highlightQuery; + highlightQuery = rewrite ? highlightQuery.rewrite(req.getSearcher().getDataCache(),req.getSearcher().getIndexReader()) : highlightQuery; } // No highlighting if there is no query -- consider q.alt="*:* Index: lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java (revision 1062897) +++ lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java (working copy) @@ -590,7 +590,7 @@ public void testRewrite() throws IOException { PhraseQuery pq = new PhraseQuery(); pq.add(new Term("foo", "bar")); - Query rewritten = pq.rewrite(searcher.getIndexReader()); + Query rewritten = pq.rewrite(searcher.getDataCache(),searcher.getIndexReader()); assertTrue(rewritten instanceof TermQuery); } Index: lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java (revision 1062897) +++ lucene/src/test/org/apache/lucene/search/spans/TestFieldMaskingSpanQuery.java (working copy) @@ -29,6 +29,7 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.QueryUtils; +import org.apache.lucene.search.QueryDataCache; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -144,7 +145,7 @@ SpanQuery q = new FieldMaskingSpanQuery (new SpanTermQuery(new Term("last", "sally")) { @Override - public Query rewrite(IndexReader reader) { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) { return new SpanOrQuery(new SpanTermQuery(new Term("first", "sally")), new SpanTermQuery(new Term("first", "james"))); } Index: lucene/src/test/org/apache/lucene/search/function/TestCustomScoreQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/function/TestCustomScoreQuery.java (revision 1062897) +++ lucene/src/test/org/apache/lucene/search/function/TestCustomScoreQuery.java (working copy) @@ -210,14 +210,14 @@ Query q = new TermQuery(new Term(TEXT_FIELD, "first")); CustomScoreQuery original = new CustomScoreQuery(q); - CustomScoreQuery rewritten = (CustomScoreQuery) original.rewrite(s.getIndexReader()); + CustomScoreQuery rewritten = (CustomScoreQuery) original.rewrite(s.getDataCache(),s.getIndexReader()); assertTrue("rewritten query should be identical, as TermQuery does not rewrite", original == rewritten); assertTrue("no hits for query", s.search(rewritten,1).totalHits > 0); assertEquals(s.search(q,1).totalHits, s.search(rewritten,1).totalHits); q = new TermRangeQuery(TEXT_FIELD, null, null, true, true); // everything original = new CustomScoreQuery(q); - rewritten = (CustomScoreQuery) original.rewrite(s.getIndexReader()); + rewritten = (CustomScoreQuery) original.rewrite(s.getDataCache(),s.getIndexReader()); assertTrue("rewritten query should not be identical, as TermRangeQuery rewrites", original != rewritten); assertTrue("no hits for query", s.search(rewritten,1).totalHits > 0); assertEquals(s.search(q,1).totalHits, s.search(original,1).totalHits); Index: lucene/src/test/org/apache/lucene/search/JustCompileSearch.java =================================================================== --- lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (revision 1062897) +++ lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (working copy) @@ -333,6 +333,16 @@ static final class JustCompileWeight extends Weight { + public JustCompileWeight() + { + super(null,null); + } + + public Weight makeCopy() + { + return new JustCompileWeight(); + } + @Override public Explanation explain(AtomicReaderContext context, int doc) throws IOException { throw new UnsupportedOperationException(UNSUPPORTED_MSG); Index: lucene/src/java/org/apache/lucene/search/ScoringRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ScoringRewrite.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/ScoringRewrite.java (working copy) @@ -86,8 +86,8 @@ * @see #setRewriteMethod */ public final static RewriteMethod CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE = new RewriteMethod() { @Override - public Query rewrite(IndexReader reader, MultiTermQuery query) throws IOException { - final BooleanQuery bq = SCORING_BOOLEAN_QUERY_REWRITE.rewrite(reader, query); + public Query rewrite(QueryDataCache dataCache, IndexReader reader, MultiTermQuery query) throws IOException { + final BooleanQuery bq = SCORING_BOOLEAN_QUERY_REWRITE.rewrite(dataCache, reader, query); // TODO: if empty boolean query return NullQuery? if (bq.clauses().isEmpty()) return bq; @@ -108,7 +108,7 @@ protected abstract void checkMaxClauseCount(int count) throws IOException; @Override - public final Q rewrite(final IndexReader reader, final MultiTermQuery query) throws IOException { + public final Q rewrite(QueryDataCache dataCache, final IndexReader reader, final MultiTermQuery query) throws IOException { final Q result = getTopLevelQuery(); final ParallelArraysTermCollector col = new ParallelArraysTermCollector(); collectTerms(reader, query, col); Index: lucene/src/java/org/apache/lucene/search/MultiTermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MultiTermQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/MultiTermQuery.java (working copy) @@ -68,7 +68,7 @@ /** Abstract class that defines how the query is rewritten. */ public static abstract class RewriteMethod implements Serializable { - public abstract Query rewrite(IndexReader reader, MultiTermQuery query) throws IOException; + public abstract Query rewrite(QueryDataCache dataCache, IndexReader reader, MultiTermQuery query) throws IOException; } /** A rewrite method that first creates a private Filter, @@ -85,7 +85,7 @@ * @see #setRewriteMethod */ public static final RewriteMethod CONSTANT_SCORE_FILTER_REWRITE = new RewriteMethod() { @Override - public Query rewrite(IndexReader reader, MultiTermQuery query) { + public Query rewrite(QueryDataCache dataCache, IndexReader reader, MultiTermQuery query) { Query result = new ConstantScoreQuery(new MultiTermQueryWrapperFilter(query)); result.setBoost(query.getBoost()); return result; @@ -313,8 +313,8 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException { - return rewriteMethod.rewrite(reader, this); + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { + return rewriteMethod.rewrite(dataCache, reader, this); } /** Index: lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (working copy) @@ -73,9 +73,9 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { if (query != null) { - Query rewritten = query.rewrite(reader); + Query rewritten = query.cachedRewrite(dataCache,reader); if (rewritten != query) { rewritten = new ConstantScoreQuery(rewritten); rewritten.setBoost(this.getBoost()); @@ -101,14 +101,26 @@ private float queryWeight; public ConstantWeight(IndexSearcher searcher) throws IOException { + super(searcher.getDataCache(),ConstantScoreQuery.this); this.innerWeight = (query == null) ? null : query.createWeight(searcher); } - @Override - public Query getQuery() { - return ConstantScoreQuery.this; + protected ConstantWeight(QueryDataCache dataCache, Query query, Weight innerWeight, + float queryNorm, float queryWeight) + { + super(dataCache,query); + this.innerWeight = (innerWeight==null)?null:innerWeight.makeCopy(); + this.queryNorm = queryNorm; + this.queryWeight = queryWeight; } + /** Make a copy, so we can cache things. + */ + public Weight makeCopy() + { + return new ConstantWeight(dataCache,theQuery,innerWeight,queryNorm,queryWeight); + } + @Override public float getValue() { return queryWeight; Index: lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (working copy) @@ -98,20 +98,32 @@ private float queryNorm; public MatchAllDocsWeight(IndexSearcher searcher) { + super(searcher.getDataCache(),MatchAllDocsQuery.this); this.similarity = searcher.getSimilarity(); } + protected MatchAllDocsWeight(QueryDataCache dataCache, Query query, Similarity similarity, + float queryWeight, float queryNorm) + { + super(dataCache,query); + this.similarity = similarity; + this.queryWeight = queryWeight; + this.queryNorm = queryNorm; + } + + /** Make a copy, so we can cache things. + */ + public Weight makeCopy() + { + return new MatchAllDocsWeight(dataCache,theQuery,similarity,queryWeight,queryNorm); + } + @Override public String toString() { return "weight(" + MatchAllDocsQuery.this + ")"; } @Override - public Query getQuery() { - return MatchAllDocsQuery.this; - } - - @Override public float getValue() { return queryWeight; } Index: lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (working copy) @@ -101,15 +101,31 @@ /** Construct the Weight for this Query searched by searcher. Recursively construct subquery weights. */ public DisjunctionMaxWeight(IndexSearcher searcher) throws IOException { + super(searcher.getDataCache(),DisjunctionMaxQuery.this); for (Query disjunctQuery : disjuncts) { weights.add(disjunctQuery.createWeight(searcher)); } } - /** Return our associated DisjunctionMaxQuery */ - @Override - public Query getQuery() { return DisjunctionMaxQuery.this; } + protected DisjunctionMaxWeight(QueryDataCache dataCache, Query query, ArrayList weights) + { + super(dataCache,query); + this.weights = new ArrayList(); + int i = 0; + while (i < weights.size()) + { + this.weights.add(weights.get(i).makeCopy()); + i++; + } + } + /** Make a copy, so we can cache things. + */ + public Weight makeCopy() + { + return new DisjunctionMaxWeight(dataCache,theQuery,weights); + } + /** Return our boost */ @Override public float getValue() { return getBoost(); } @@ -185,11 +201,11 @@ * @param reader the IndexReader we query * @return an optimized copy of us (which may not be a copy if there is nothing to optimize) */ @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { int numDisjunctions = disjuncts.size(); if (numDisjunctions == 1) { Query singleton = disjuncts.get(0); - Query result = singleton.rewrite(reader); + Query result = singleton.cachedRewrite(dataCache,reader); if (getBoost() != 1.0f) { if (result == singleton) result = (Query)result.clone(); result.setBoost(getBoost() * result.getBoost()); @@ -199,7 +215,7 @@ DisjunctionMaxQuery clone = null; for (int i = 0 ; i < numDisjunctions; i++) { Query clause = disjuncts.get(i); - Query rewrite = clause.rewrite(reader); + Query rewrite = clause.cachedRewrite(dataCache,reader); if (rewrite != clause) { if (clone == null) clone = (DisjunctionMaxQuery)this.clone(); clone.disjuncts.set(i, rewrite); Index: lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (working copy) @@ -26,6 +26,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.Weight; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.QueryDataCache; import org.apache.lucene.util.ToStringUtils; /** @@ -107,10 +108,10 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { FieldMaskingSpanQuery clone = null; - SpanQuery rewritten = (SpanQuery) maskedQuery.rewrite(reader); + SpanQuery rewritten = (SpanQuery) maskedQuery.cachedRewrite(dataCache,reader); if (rewritten != maskedQuery) { clone = (FieldMaskingSpanQuery) this.clone(); clone.maskedQuery = rewritten; Index: lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryDataCache; import org.apache.lucene.util.ToStringUtils; import java.io.IOException; @@ -167,15 +168,15 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { SpanNotQuery clone = null; - SpanQuery rewrittenInclude = (SpanQuery) include.rewrite(reader); + SpanQuery rewrittenInclude = (SpanQuery) include.cachedRewrite(dataCache,reader); if (rewrittenInclude != include) { clone = (SpanNotQuery) this.clone(); clone.include = rewrittenInclude; } - SpanQuery rewrittenExclude = (SpanQuery) exclude.rewrite(reader); + SpanQuery rewrittenExclude = (SpanQuery) exclude.cachedRewrite(dataCache,reader); if (rewrittenExclude != exclude) { if (clone == null) clone = (SpanNotQuery) this.clone(); clone.exclude = rewrittenExclude; Index: lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java (working copy) @@ -42,6 +42,7 @@ public SpanWeight(SpanQuery query, IndexSearcher searcher) throws IOException { + super(searcher.getDataCache(),query); this.similarity = searcher.getSimilarity(); this.query = query; @@ -52,6 +53,27 @@ idf = idfExp.getIdf(); } + protected SpanWeight(QueryDataCache dataCache, SpanQuery query, Similarity similarity, float value, + float idf, float queryNorm, float queryWeight, Set terms, IDFExplanation idfExp) + { + super(dataCache,query); + this.query = query; + this.similarity = similarity; + this.value = value; + this.idf = idf; + this.queryNorm = queryNorm; + this.queryWeight = queryWeight; + this.terms = terms; + this.idfExp = idfExp; + } + + /** Make a copy, so we can cache things. + */ + public Weight makeCopy() + { + return new SpanWeight(dataCache,query,similarity,value,idf,queryNorm,queryWeight,terms,idfExp); + } + @Override public Query getQuery() { return query; } Index: lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (working copy) @@ -21,7 +21,9 @@ import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryDataCache; + import java.io.IOException; import java.util.ArrayList; import java.util.Collection; @@ -87,10 +89,10 @@ @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { SpanPositionCheckQuery clone = null; - SpanQuery rewritten = (SpanQuery) match.rewrite(reader); + SpanQuery rewritten = (SpanQuery) match.cachedRewrite(dataCache,reader); if (rewritten != match) { clone = (SpanPositionCheckQuery) this.clone(); clone.match = rewritten; Index: lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (working copy) @@ -31,7 +31,9 @@ import org.apache.lucene.util.PriorityQueue; import org.apache.lucene.util.ToStringUtils; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryDataCache; + /** Matches the union of its clauses.*/ public class SpanOrQuery extends SpanQuery implements Cloneable { private List clauses; @@ -86,11 +88,11 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { SpanOrQuery clone = null; for (int i = 0 ; i < clauses.size(); i++) { SpanQuery c = clauses.get(i); - SpanQuery query = (SpanQuery) c.rewrite(reader); + SpanQuery query = (SpanQuery) c.cachedRewrite(dataCache,reader); if (query != c) { // clause rewrote: must clone if (clone == null) clone = (SpanOrQuery) this.clone(); Index: lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (working copy) @@ -27,6 +27,7 @@ import org.apache.lucene.search.TopTermsRewrite; import org.apache.lucene.search.ScoringRewrite; import org.apache.lucene.search.BooleanClause.Occur; // javadocs only +import org.apache.lucene.search.QueryDataCache; import org.apache.lucene.util.PerReaderTermState; /** @@ -108,8 +109,8 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException { - final Query q = query.rewrite(reader); + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { + final Query q = query.cachedRewrite(dataCache,reader); if (!(q instanceof SpanQuery)) throw new UnsupportedOperationException("You can only use SpanMultiTermQueryWrapper with a suitable SpanRewriteMethod."); return q; @@ -132,7 +133,7 @@ /** Abstract class that defines how the query is rewritten. */ public static abstract class SpanRewriteMethod extends MultiTermQuery.RewriteMethod { @Override - public abstract SpanQuery rewrite(IndexReader reader, MultiTermQuery query) throws IOException; + public abstract SpanQuery rewrite(QueryDataCache dataCache, IndexReader reader, MultiTermQuery query) throws IOException; } /** @@ -163,8 +164,8 @@ }; @Override - public SpanQuery rewrite(IndexReader reader, MultiTermQuery query) throws IOException { - return delegate.rewrite(reader, query); + public SpanQuery rewrite(QueryDataCache dataCache, IndexReader reader, MultiTermQuery query) throws IOException { + return delegate.rewrite(dataCache, reader, query); } // Make sure we are still a singleton even after deserializing @@ -218,8 +219,8 @@ } @Override - public SpanQuery rewrite(IndexReader reader, MultiTermQuery query) throws IOException { - return delegate.rewrite(reader, query); + public SpanQuery rewrite(QueryDataCache dataCache, IndexReader reader, MultiTermQuery query) throws IOException { + return delegate.rewrite(dataCache, reader, query); } @Override Index: lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java (working copy) @@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryDataCache; import org.apache.lucene.util.ToStringUtils; /** Matches spans which are near one another. One can specify slop, the @@ -130,11 +131,11 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { SpanNearQuery clone = null; for (int i = 0 ; i < clauses.size(); i++) { SpanQuery c = clauses.get(i); - SpanQuery query = (SpanQuery) c.rewrite(reader); + SpanQuery query = (SpanQuery) c.cachedRewrite(dataCache,reader); if (query != c) { // clause rewrote: must clone if (clone == null) clone = (SpanNearQuery) this.clone(); Index: lucene/src/java/org/apache/lucene/search/QueryDataCache.java =================================================================== --- lucene/src/java/org/apache/lucene/search/QueryDataCache.java (revision 0) +++ lucene/src/java/org/apache/lucene/search/QueryDataCache.java (revision 0) @@ -0,0 +1,434 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.search.Weight.ScorerContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.index.IndexReader; + +import java.util.*; +import java.io.*; + +/** This class caches data about a query, and about result sets returned by the query. +* It is meant to be used strictly within the rewriting or execution of a single query +* structure, and cannot work cross-thread or cross-query. +*/ +public class QueryDataCache +{ + /** A hash for rewritten queries. */ + protected Map rewriteCache = new HashMap(); + /** A hash for weights */ + protected Map weightCache = new HashMap(); + /** A hash for base caching result sets. */ + protected Map scorerCache = new HashMap(); + + public QueryDataCache() + { + } + + /** Lookup a rewrite query. + */ + public Query lookupRewrite(Query baseQuery, IndexReader reader) + throws IOException + { + Query rval = rewriteCache.get(baseQuery); + if (rval == null) + { + rval = baseQuery; + for (Query rewrittenQuery = rval.rewrite(this,reader); rewrittenQuery != rval; + rewrittenQuery = rval.rewrite(this,reader)) + { + rval = rewrittenQuery; + } + if (rval != null) + rewriteCache.put(baseQuery,rval); + } + return rval; + } + + /** Create a cloned Weight object, if we have one. + */ + public Weight createWeight(Query baseQuery, IndexSearcher searcher) + throws IOException + { + Weight rval = weightCache.get(baseQuery); + if (rval == null) + { + // No existing weight stored, so we have to create one. + // First, do the rewrite, if it hasn't already been done. I'm not sure why this is necessary + // at this level, but it seems to be. + Query rewritten = searcher.rewrite(baseQuery); + rval = rewritten.createWeight(searcher); + // The weight we save is the weight for the rewritten query!! + weightCache.put(baseQuery,rval); + } + return rval.makeCopy(); + } + + /** Look up a result iterator. This will return a FRESH result iterator, + * based on the query specified, or null if there is no underlying result + * iterator currently registered. + */ + public Scorer createScorer(Query baseQuery, Weight w, AtomicReaderContext context, ScorerContext scorerContext) + throws IOException + { + CompositeKey ck = new CompositeKey(baseQuery,context,scorerContext); + CachingResultScorer cri = scorerCache.get(ck); + if (cri == null) + { + // Need to create a new CachingResultScorer object + cri = new CachingResultScorer(w,context,scorerContext); + scorerCache.put(ck,cri); + } + // Create a new iterator to return. + return cri.getFreshScorer(); + } + + // The strategy we use to cache Scorers is complicated by the fact that the advance() + // method doesn't give us any information about how many documents got skipped. Thus, we + // only can know certain things, like when nextDoc() is done we can tell the actual next doc + // and know there are no missing docs in-between. For advance, we can know the next doc + // after the target doc ID, which may well serve for other advance() calls too. In other + // words, walking through the scorer must happen multiple times. + + protected static class CachingResultScorer + { + protected AtomicReaderContext context; + protected ScorerContext scorerContext; + protected Weight w; + + protected Scorer scorer; + protected int startingDocID; + protected int currentDocID; + + public CachingResultScorer(Weight w, AtomicReaderContext context, ScorerContext scorerContext) + throws IOException + { + this.context = context; + this.scorerContext = scorerContext; + this.w = w; + + this.scorer = w.scorer(context,scorerContext); + if (scorer != null) + { + this.startingDocID = scorer.docID(); + this.currentDocID = startingDocID; + } + } + + public Scorer getFreshScorer() + throws IOException + { + // Null scorers are special + if (scorer == null) + return scorer; + + // Otherwise, create a back reference + return new CachedResultScorer(this); + } + + public Weight getWeight() + { + return w; + } + + public Pointer initialize() + throws IOException + { + return new Pointer(startingDocID); + } + + public int docID(Pointer pointer) + { + return pointer.getCurrentDocID(); + } + + public int nextDoc(Pointer pointer) + throws IOException + { + int currentPointer = pointer.getCurrentDocID(); + if (currentPointer == Scorer.NO_MORE_DOCS) + return currentPointer; + + if (currentDocID > currentPointer) + { + // Need to build a new scorer + scorer = w.scorer(context,scorerContext); + if (scorer == null) + throw new IllegalArgumentException("Scorer inconsistently returns null on same inputs!"); + currentDocID = startingDocID; + } + + if (currentDocID == currentPointer) + { + currentDocID = scorer.nextDoc(); + pointer.setCurrentDocID(currentDocID); + return currentDocID; + } + + // Use advance() to get us where we need to be. + currentDocID = scorer.advance(currentPointer+1); + pointer.setCurrentDocID(currentDocID); + return currentDocID; + } + + public int advance(Pointer pointer, int target) + throws IOException + { + int currentPointer = pointer.getCurrentDocID(); + if (currentPointer == Scorer.NO_MORE_DOCS) + return currentPointer; + + if (currentDocID > target) + { + // Need to build a new scorer + scorer = w.scorer(context,scorerContext); + if (scorer == null) + throw new IllegalArgumentException("Scorer inconsistently returns null on same inputs!"); + currentDocID = startingDocID; + } + + if (currentDocID == target) + { + pointer.setCurrentDocID(currentDocID); + return currentDocID; + } + + // Use advance() to get us where we need to be. + currentDocID = scorer.advance(target); + pointer.setCurrentDocID(currentDocID); + return currentDocID; + } + + public float score(Pointer pointer) throws IOException + { + // Get the scorer to the right place + int currentPointer = pointer.getCurrentDocID(); + if (currentPointer == Scorer.NO_MORE_DOCS) + return 0.0f; + + if (currentDocID > currentPointer) + { + // Need to build a new scorer + scorer = w.scorer(context,scorerContext); + if (scorer == null) + throw new IllegalArgumentException("Scorer inconsistently returns null on same inputs!"); + currentDocID = startingDocID; + } + + if (currentDocID < currentPointer) + currentDocID = scorer.advance(currentPointer); + + return scorer.score(); + } + + public float freq(Pointer pointer) throws IOException + { + // Get the scorer to the right place + int currentPointer = pointer.getCurrentDocID(); + if (currentPointer == Scorer.NO_MORE_DOCS) + return 0.0f; + + if (currentDocID > currentPointer) + { + // Need to build a new scorer + scorer = w.scorer(context,scorerContext); + if (scorer == null) + throw new IllegalArgumentException("Scorer inconsistently returns null on same inputs!"); + currentDocID = startingDocID; + } + + if (currentDocID < currentPointer) + currentDocID = scorer.advance(currentPointer); + + return scorer.freq(); + } + + } + + protected static class Result + { + public int docID; + public float score; + public float freq; + + public Result(int docID, float score, float freq) + { + this.docID = docID; + this.score = score; + this.freq = freq; + } + } + + /** This is a structure which maintains an individual pointer. + */ + protected static class Pointer + { + protected int docID; + + public Pointer(int startingDocID) + { + docID = startingDocID; + } + + public void setCurrentDocID(int docID) + { + this.docID = docID; + } + + public int getCurrentDocID() + { + return docID; + } + } + + protected static class CachedResultScorer extends Scorer + { + protected CachingResultScorer sharedStuff; + protected Pointer pointer; + + public CachedResultScorer(CachingResultScorer sharedStuff) + throws IOException + { + super(sharedStuff.getWeight()); + this.sharedStuff = sharedStuff; + this.pointer = sharedStuff.initialize(); + } + + /** + * Returns the following: + *
    + *
  • -1 or {@link #NO_MORE_DOCS} if {@link #nextDoc()} or + * {@link #advance(int)} were not called yet. + *
  • {@link #NO_MORE_DOCS} if the iterator has exhausted. + *
  • Otherwise it should return the doc ID it is currently on. + *
+ *

+ * + * @since 2.9 + */ + public int docID() + { + return sharedStuff.docID(pointer); + } + + /** + * Advances to the next document in the set and returns the doc it is + * currently on, or {@link #NO_MORE_DOCS} if there are no more docs in the + * set.
+ * + * NOTE: after the iterator has exhausted you should not call this + * method, as it may result in unpredicted behavior. + * + * @since 2.9 + */ + public int nextDoc() throws IOException + { + return sharedStuff.nextDoc(pointer); + } + + /** + * Advances to the first beyond the current whose document number is greater + * than or equal to target. Returns the current document number or + * {@link #NO_MORE_DOCS} if there are no more docs in the set. + *

+ * Behaves as if written: + * + * 

+     * int advance(int target) {
+     *   int doc;
+     *   while ((doc = nextDoc()) < target) {
+     *   }
+     *   return doc;
+     * }
+     *
+ * + * Some implementations are considerably more efficient than that. + *

+ * NOTE: certain implementations may return a different value (each + * time) if called several times in a row with the same target. + *

+ * NOTE: this method may be called with {@value #NO_MORE_DOCS} for + * efficiency by some Scorers. If your implementation cannot efficiently + * determine that it should exhaust, it is recommended that you check for that + * value in each call to this method. + *

+ * NOTE: after the iterator has exhausted you should not call this + * method, as it may result in unpredicted behavior. + *

+ * + * @since 2.9 + */ + public int advance(int target) throws IOException + { + return sharedStuff.advance(pointer,target); + } + + /** Returns the score of the current document matching the query. + * Initially invalid, until {@link #nextDoc()} or {@link #advance(int)} + * is called the first time, or when called from within + * {@link Collector#collect}. + */ + public float score() throws IOException + { + return sharedStuff.score(pointer); + } + + /** Returns number of matches for the current document. + * This returns a float (not int) because + * SloppyPhraseScorer discounts its freq according to how + * "sloppy" the match was. + * + * @lucene.experimental */ + public float freq() throws IOException { + return sharedStuff.freq(pointer); + } + + } + + protected static class CompositeKey + { + private Query baseQuery; + private AtomicReaderContext context; + private ScorerContext scorerContext; + + public CompositeKey(Query baseQuery, AtomicReaderContext context, ScorerContext scorerContext) + { + this.baseQuery = baseQuery; + this.context = context; + this.scorerContext = scorerContext; + } + + public int hashCode() + { + return baseQuery.hashCode() + context.hashCode() + scorerContext.hashCode(); + } + + public boolean equals(Object o) + { + if (!(o instanceof CompositeKey)) + return false; + CompositeKey k = (CompositeKey)o; + return baseQuery.equals(k.baseQuery) && context.equals(k.context) && scorerContext.equals(k.scorerContext); + } + } + +} Property changes on: lucene/src/java/org/apache/lucene/search/QueryDataCache.java ___________________________________________________________________ Added: svn:keywords + Id Added: svn:eol-style + native Index: lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (working copy) @@ -139,6 +139,7 @@ public MultiPhraseWeight(IndexSearcher searcher) throws IOException { + super(searcher.getDataCache(),MultiPhraseQuery.this); this.similarity = searcher.getSimilarity(); // compute idf @@ -152,8 +153,24 @@ idf = idfExp.getIdf(); } - @Override - public Query getQuery() { return MultiPhraseQuery.this; } + protected MultiPhraseWeight(QueryDataCache dataCache, Query query, Similarity similarity, + float value, float idf, IDFExplanation idfExp, float queryNorm, float queryWeight) + { + super(dataCache,query); + this.similarity = similarity; + this.value = value; + this.idf = idf; + this.idfExp = idfExp; + this.queryNorm = queryNorm; + this.queryWeight = queryWeight; + } + + /** Make a copy, so we can cache things. + */ + public Weight makeCopy() + { + return new MultiPhraseWeight(dataCache,theQuery,similarity,value,idf,idfExp,queryNorm,queryWeight); + } @Override public float getValue() { return value; } @@ -314,7 +331,7 @@ } @Override - public Query rewrite(IndexReader reader) { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) { if (termArrays.size() == 1) { // optimize one-term case Term[] terms = termArrays.get(0); BooleanQuery boq = new BooleanQuery(true); Index: lucene/src/java/org/apache/lucene/search/FilteredQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FilteredQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/FilteredQuery.java (working copy) @@ -61,116 +61,13 @@ */ @Override public Weight createWeight(final IndexSearcher searcher) throws IOException { - final Weight weight = query.createWeight (searcher); - return new Weight() { - private float value; - - // pass these methods through to enclosed query's weight - @Override - public float getValue() { return value; } - - @Override - public float sumOfSquaredWeights() throws IOException { - return weight.sumOfSquaredWeights() * getBoost() * getBoost(); - } - - @Override - public void normalize (float v) { - weight.normalize(v); - value = weight.getValue() * getBoost(); - } - - @Override - public Explanation explain (AtomicReaderContext ir, int i) throws IOException { - Explanation inner = weight.explain (ir, i); - if (getBoost()!=1) { - Explanation preBoost = inner; - inner = new Explanation(inner.getValue()*getBoost(),"product of:"); - inner.addDetail(new Explanation(getBoost(),"boost")); - inner.addDetail(preBoost); - } - Filter f = FilteredQuery.this.filter; - DocIdSet docIdSet = f.getDocIdSet(ir); - DocIdSetIterator docIdSetIterator = docIdSet == null ? DocIdSet.EMPTY_DOCIDSET.iterator() : docIdSet.iterator(); - if (docIdSetIterator == null) { - docIdSetIterator = DocIdSet.EMPTY_DOCIDSET.iterator(); - } - if (docIdSetIterator.advance(i) == i) { - return inner; - } else { - Explanation result = new Explanation - (0.0f, "failure to match filter: " + f.toString()); - result.addDetail(inner); - return result; - } - } - - // return this query - @Override - public Query getQuery() { return FilteredQuery.this; } - - // return a filtering scorer - @Override - public Scorer scorer(AtomicReaderContext context, ScorerContext scoreContext) - throws IOException { - final Scorer scorer = weight.scorer(context, ScorerContext.def()); - if (scorer == null) { - return null; - } - DocIdSet docIdSet = filter.getDocIdSet(context); - if (docIdSet == null) { - return null; - } - final DocIdSetIterator docIdSetIterator = docIdSet.iterator(); - if (docIdSetIterator == null) { - return null; - } - - return new Scorer(this) { - - private int doc = -1; - - private int advanceToCommon(int scorerDoc, int disiDoc) throws IOException { - while (scorerDoc != disiDoc) { - if (scorerDoc < disiDoc) { - scorerDoc = scorer.advance(disiDoc); - } else { - disiDoc = docIdSetIterator.advance(scorerDoc); - } - } - return scorerDoc; - } - - @Override - public int nextDoc() throws IOException { - int scorerDoc, disiDoc; - return doc = (disiDoc = docIdSetIterator.nextDoc()) != NO_MORE_DOCS - && (scorerDoc = scorer.nextDoc()) != NO_MORE_DOCS - && advanceToCommon(scorerDoc, disiDoc) != NO_MORE_DOCS ? scorer.docID() : NO_MORE_DOCS; - } - - @Override - public int docID() { return doc; } - - @Override - public int advance(int target) throws IOException { - int disiDoc, scorerDoc; - return doc = (disiDoc = docIdSetIterator.advance(target)) != NO_MORE_DOCS - && (scorerDoc = scorer.advance(disiDoc)) != NO_MORE_DOCS - && advanceToCommon(scorerDoc, disiDoc) != NO_MORE_DOCS ? scorer.docID() : NO_MORE_DOCS; - } - - @Override - public float score() throws IOException { return getBoost() * scorer.score(); } - }; - } - }; + return new FilteredWeight(searcher); } /** Rewrites the wrapped query. */ @Override - public Query rewrite(IndexReader reader) throws IOException { - Query rewritten = query.rewrite(reader); + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { + Query rewritten = query.cachedRewrite(dataCache,reader); if (rewritten != query) { FilteredQuery clone = (FilteredQuery)this.clone(); clone.query = rewritten; @@ -221,4 +118,126 @@ public int hashCode() { return query.hashCode() ^ filter.hashCode() + Float.floatToRawIntBits(getBoost()); } + + protected class FilteredWeight extends Weight + { + private final Weight weight; + private float value; + + public FilteredWeight(IndexSearcher searcher) + throws IOException + { + super(searcher.getDataCache(),FilteredQuery.this); + weight = query.cachedCreateWeight(searcher); + } + + protected FilteredWeight(QueryDataCache dataCache, Query query, Weight weight, float value) + { + super(dataCache,query); + this.weight = weight.makeCopy(); + this.value = value; + } + + public Weight makeCopy() + { + return new FilteredWeight(dataCache,theQuery,weight,value); + } + + // pass these methods through to enclosed query's weight + @Override + public float getValue() { return value; } + + @Override + public float sumOfSquaredWeights() throws IOException { + return weight.sumOfSquaredWeights() * getBoost() * getBoost(); + } + + @Override + public void normalize (float v) { + weight.normalize(v); + value = weight.getValue() * getBoost(); + } + + @Override + public Explanation explain (AtomicReaderContext ir, int i) throws IOException { + Explanation inner = weight.explain (ir, i); + if (getBoost()!=1) { + Explanation preBoost = inner; + inner = new Explanation(inner.getValue()*getBoost(),"product of:"); + inner.addDetail(new Explanation(getBoost(),"boost")); + inner.addDetail(preBoost); + } + Filter f = FilteredQuery.this.filter; + DocIdSet docIdSet = f.getDocIdSet(ir); + DocIdSetIterator docIdSetIterator = docIdSet == null ? DocIdSet.EMPTY_DOCIDSET.iterator() : docIdSet.iterator(); + if (docIdSetIterator == null) { + docIdSetIterator = DocIdSet.EMPTY_DOCIDSET.iterator(); + } + if (docIdSetIterator.advance(i) == i) { + return inner; + } else { + Explanation result = new Explanation + (0.0f, "failure to match filter: " + f.toString()); + result.addDetail(inner); + return result; + } + } + + // return a filtering scorer + @Override + public Scorer scorer(AtomicReaderContext context, ScorerContext scoreContext) + throws IOException { + final Scorer scorer = weight.scorer(context, ScorerContext.def()); + if (scorer == null) { + return null; + } + DocIdSet docIdSet = filter.getDocIdSet(context); + if (docIdSet == null) { + return null; + } + final DocIdSetIterator docIdSetIterator = docIdSet.iterator(); + if (docIdSetIterator == null) { + return null; + } + + return new Scorer(this) { + + private int doc = -1; + + private int advanceToCommon(int scorerDoc, int disiDoc) throws IOException { + while (scorerDoc != disiDoc) { + if (scorerDoc < disiDoc) { + scorerDoc = scorer.advance(disiDoc); + } else { + disiDoc = docIdSetIterator.advance(scorerDoc); + } + } + return scorerDoc; + } + + @Override + public int nextDoc() throws IOException { + int scorerDoc, disiDoc; + return doc = (disiDoc = docIdSetIterator.nextDoc()) != NO_MORE_DOCS + && (scorerDoc = scorer.nextDoc()) != NO_MORE_DOCS + && advanceToCommon(scorerDoc, disiDoc) != NO_MORE_DOCS ? scorer.docID() : NO_MORE_DOCS; + } + + @Override + public int docID() { return doc; } + + @Override + public int advance(int target) throws IOException { + int disiDoc, scorerDoc; + return doc = (disiDoc = docIdSetIterator.advance(target)) != NO_MORE_DOCS + && (scorerDoc = scorer.advance(disiDoc)) != NO_MORE_DOCS + && advanceToCommon(scorerDoc, disiDoc) != NO_MORE_DOCS ? scorer.docID() : NO_MORE_DOCS; + } + + @Override + public float score() throws IOException { return getBoost() * scorer.score(); } + }; + } + } + } Index: lucene/src/java/org/apache/lucene/search/Query.java =================================================================== --- lucene/src/java/org/apache/lucene/search/Query.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/Query.java (working copy) @@ -81,6 +81,13 @@ return toString(""); } + public Weight cachedCreateWeight(IndexSearcher searcher) + throws IOException + { + // Use this query as a key to building a weight wrapper that refers to a shared weight object. + return searcher.getDataCache().createWeight(this,searcher); + } + /** * Expert: Constructs an appropriate Weight implementation for this query. * @@ -91,12 +98,13 @@ throw new UnsupportedOperationException(); } + /** * Expert: Constructs and initializes a Weight for a top-level query. */ public Weight weight(IndexSearcher searcher) throws IOException { Query query = searcher.rewrite(this); - Weight weight = query.createWeight(searcher); + Weight weight = query.cachedCreateWeight(searcher); float sum = weight.sumOfSquaredWeights(); float norm = searcher.getSimilarity().queryNorm(sum); if (Float.isInfinite(norm) || Float.isNaN(norm)) @@ -105,14 +113,21 @@ return weight; } + /** Expert: called to re-write queries into primitive queries, using a temporary + * cache to reduce overhead for queries that share subqueries. + */ + public Query cachedRewrite(QueryDataCache qdc, IndexReader reader) throws IOException { + return qdc.lookupRewrite(this,reader); + } /** Expert: called to re-write queries into primitive queries. For example, * a PrefixQuery will be rewritten into a BooleanQuery that consists * of TermQuerys. */ - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache qdc, IndexReader reader) throws IOException { return this; } + /** * Expert: adds all terms occurring in this query to the terms set. Only Index: lucene/src/java/org/apache/lucene/search/BooleanQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/BooleanQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/BooleanQuery.java (working copy) @@ -169,6 +169,7 @@ public BooleanWeight(IndexSearcher searcher, boolean disableCoord) throws IOException { + super(searcher.getDataCache(),BooleanQuery.this); this.similarity = searcher.getSimilarity(); this.disableCoord = disableCoord; weights = new ArrayList(clauses.size()); @@ -179,9 +180,29 @@ } } - @Override - public Query getQuery() { return BooleanQuery.this; } + protected BooleanWeight(QueryDataCache dataCache, Query query, Similarity similarity, ArrayList weights, + int maxCoord, boolean disableCoord) + { + super(dataCache,query); + this.similarity = similarity; + this.weights = new ArrayList(); + int i = 0; + while (i < weights.size()) + { + this.weights.add(weights.get(i).makeCopy()); + i++; + } + this.maxCoord = maxCoord; + this.disableCoord = disableCoord; + } + /** Make a copy, so we can cache things. + */ + public Weight makeCopy() + { + return new BooleanWeight(dataCache,theQuery,similarity,weights,maxCoord,disableCoord); + } + @Override public float getValue() { return getBoost(); } @@ -229,7 +250,7 @@ for (Iterator wIter = weights.iterator(); wIter.hasNext();) { Weight w = wIter.next(); BooleanClause c = cIter.next(); - if (w.scorer(context, ScorerContext.def().scoreDocsInOrder(true).topScorer(true)) == null) { + if (w.cachedScorer(context, ScorerContext.def().scoreDocsInOrder(true).topScorer(true)) == null) { if (c.isRequired()) { fail = true; Explanation r = new Explanation(0.0f, "no match on required clause (" + c.getQuery().toString() + ")"); @@ -299,7 +320,7 @@ Iterator cIter = clauses.iterator(); for (Weight w : weights) { BooleanClause c = cIter.next(); - Scorer subScorer = w.scorer(context, ScorerContext.def()); + Scorer subScorer = w.cachedScorer(context, ScorerContext.def()); if (subScorer == null) { if (c.isRequired()) { return null; @@ -359,12 +380,12 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { if (minNrShouldMatch == 0 && clauses.size() == 1) { // optimize 1-clause queries BooleanClause c = clauses.get(0); if (!c.isProhibited()) { // just return clause - Query query = c.getQuery().rewrite(reader); // rewrite first + Query query = c.getQuery().cachedRewrite(dataCache,reader); // rewrite first if (getBoost() != 1.0f) { // incorporate boost if (query == c.getQuery()) // if rewrite was no-op @@ -379,7 +400,7 @@ BooleanQuery clone = null; // recursively rewrite for (int i = 0 ; i < clauses.size(); i++) { BooleanClause c = clauses.get(i); - Query query = c.getQuery().rewrite(reader); + Query query = c.getQuery().cachedRewrite(dataCache,reader); if (query != c.getQuery()) { // clause rewrote: must clone if (clone == null) clone = (BooleanQuery)this.clone(); Index: lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java (working copy) @@ -53,7 +53,7 @@ /*(non-Javadoc) @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader) */ @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { return this; } @@ -68,14 +68,23 @@ float queryWeight; public ValueSourceWeight(IndexSearcher searcher) { + super(searcher.getDataCache(),ValueSourceQuery.this); } - /*(non-Javadoc) @see org.apache.lucene.search.Weight#getQuery() */ - @Override - public Query getQuery() { - return ValueSourceQuery.this; + protected ValueSourceWeight(QueryDataCache dataCache, Query query, float queryNorm, float queryWeight) + { + super(dataCache,query); + this.queryNorm = queryNorm; + this.queryWeight = queryWeight; } + /** Make a copy, so we can cache things. + */ + public Weight makeCopy() + { + return new ValueSourceWeight(dataCache,theQuery,queryNorm,queryWeight); + } + /*(non-Javadoc) @see org.apache.lucene.search.Weight#getValue() */ @Override public float getValue() { Index: lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java (working copy) @@ -30,6 +30,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.search.Scorer; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.QueryDataCache; import org.apache.lucene.util.ToStringUtils; /** @@ -88,17 +89,17 @@ /*(non-Javadoc) @see org.apache.lucene.search.Query#rewrite(org.apache.lucene.index.IndexReader) */ @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { CustomScoreQuery clone = null; - final Query sq = subQuery.rewrite(reader); + final Query sq = subQuery.cachedRewrite(dataCache,reader); if (sq != subQuery) { clone = (CustomScoreQuery) clone(); clone.subQuery = sq; } for(int i = 0; i < valSrcQueries.length; i++) { - final ValueSourceQuery v = (ValueSourceQuery) valSrcQueries[i].rewrite(reader); + final ValueSourceQuery v = (ValueSourceQuery) valSrcQueries[i].cachedRewrite(dataCache,reader); if (v != valSrcQueries[i]) { if (clone == null) clone = (CustomScoreQuery) clone(); clone.valSrcQueries[i] = v; @@ -187,20 +188,37 @@ boolean qStrict; public CustomWeight(IndexSearcher searcher) throws IOException { + super(searcher.getDataCache(),CustomScoreQuery.this); this.subQueryWeight = subQuery.weight(searcher); this.valSrcWeights = new Weight[valSrcQueries.length]; for(int i = 0; i < valSrcQueries.length; i++) { - this.valSrcWeights[i] = valSrcQueries[i].createWeight(searcher); + this.valSrcWeights[i] = valSrcQueries[i].cachedCreateWeight(searcher); } this.qStrict = strict; } - /*(non-Javadoc) @see org.apache.lucene.search.Weight#getQuery() */ - @Override - public Query getQuery() { - return CustomScoreQuery.this; + protected CustomWeight(QueryDataCache dataCache, Query query, + Weight subQueryWeight, Weight[] valSrcWeights, boolean qStrict) + { + super(dataCache,query); + this.subQueryWeight = subQueryWeight.makeCopy(); + this.valSrcWeights = new Weight[valSrcWeights.length]; + int i = 0; + while (i < valSrcWeights.length) + { + this.valSrcWeights[i] = valSrcWeights[i].makeCopy(); + i++; + } + this.qStrict = qStrict; } + /** Make a copy, so we can cache things. + */ + public Weight makeCopy() + { + return new CustomWeight(dataCache,theQuery,subQueryWeight,valSrcWeights,qStrict); + } + /*(non-Javadoc) @see org.apache.lucene.search.Weight#getValue() */ @Override public float getValue() { Index: lucene/src/java/org/apache/lucene/search/PhraseQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/PhraseQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/PhraseQuery.java (working copy) @@ -111,13 +111,13 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { if (terms.size() == 1) { TermQuery tq = new TermQuery(terms.get(0)); tq.setBoost(getBoost()); return tq; } else - return super.rewrite(reader); + return super.cachedRewrite(dataCache,reader); } static class PostingsAndFreq implements Comparable { @@ -146,19 +146,36 @@ public PhraseWeight(IndexSearcher searcher) throws IOException { + super(searcher.getDataCache(),PhraseQuery.this); this.similarity = searcher.getSimilarity(); idfExp = similarity.idfExplain(terms, searcher); idf = idfExp.getIdf(); } + protected PhraseWeight(QueryDataCache dataCache, Query query, Similarity similarity, float value, float idf, + float queryNorm, float queryWeight, IDFExplanation idfExp) + { + super(dataCache,query); + this.similarity = similarity; + this.value = value; + this.idf = idf; + this.queryNorm = queryNorm; + this.queryWeight = queryWeight; + this.idfExp = idfExp; + } + + /** Make a copy, so we can cache things. + */ + public Weight makeCopy() + { + return new PhraseWeight(dataCache,theQuery,similarity,value,idf,queryNorm,queryWeight,idfExp); + } + @Override public String toString() { return "weight(" + PhraseQuery.this + ")"; } @Override - public Query getQuery() { return PhraseQuery.this; } - - @Override public float getValue() { return value; } @Override @@ -317,7 +334,7 @@ Term term = terms.get(0); Query termQuery = new TermQuery(term); termQuery.setBoost(getBoost()); - return termQuery.createWeight(searcher); + return termQuery.cachedCreateWeight(searcher); } return new PhraseWeight(searcher); } Index: lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/TopTermsRewrite.java (working copy) @@ -60,7 +60,7 @@ protected abstract int getMaxSize(); @Override - public final Q rewrite(final IndexReader reader, final MultiTermQuery query) throws IOException { + public final Q rewrite(QueryDataCache dataCache, final IndexReader reader, final MultiTermQuery query) throws IOException { final int maxSize = Math.min(size, getMaxSize()); final PriorityQueue stQueue = new PriorityQueue(); collectTerms(reader, query, new TermCollector() { Index: lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/ConstantScoreAutoRewrite.java (working copy) @@ -82,7 +82,7 @@ } @Override - public Query rewrite(final IndexReader reader, final MultiTermQuery query) throws IOException { + public Query rewrite(QueryDataCache dataCache, final IndexReader reader, final MultiTermQuery query) throws IOException { // Get the enum and start visiting terms. If we // exhaust the enum before hitting either of the @@ -95,7 +95,7 @@ collectTerms(reader, query, col); final int size = col.pendingTerms.size(); if (col.hasCutOff) { - return MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE.rewrite(reader, query); + return MultiTermQuery.CONSTANT_SCORE_FILTER_REWRITE.rewrite(dataCache, reader, query); } else if (size == 0) { return getTopLevelQuery(); } else { Index: lucene/src/java/org/apache/lucene/search/TermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TermQuery.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/TermQuery.java (working copy) @@ -52,6 +52,7 @@ public TermWeight(IndexSearcher searcher, PerReaderTermState termStates, int docFreq) throws IOException { + super(searcher.getDataCache(),TermQuery.this); assert termStates != null : "PerReaderTermState must not be null"; this.termStates = termStates; this.similarity = searcher.getSimilarity(); @@ -63,13 +64,30 @@ idf = idfExp.getIdf(); } + protected TermWeight(QueryDataCache dataCache, Query query, Similarity similarity, float value, float idf, float queryNorm, float queryWeight, IDFExplanation idfExp, + PerReaderTermState termStates) + { + super(dataCache,query); + this.similarity = similarity; + this.value = value; + this.idf = idf; + this.queryNorm = queryNorm; + this.queryWeight = queryWeight; + this.idfExp = idfExp; + this.termStates = termStates; + } + + /** Make a copy, so we can cache things. + */ + public Weight makeCopy() + { + return new TermWeight(dataCache,theQuery,similarity,value,idf,queryNorm,queryWeight,idfExp,termStates); + } + @Override public String toString() { return "weight(" + TermQuery.this + ")"; } @Override - public Query getQuery() { return TermQuery.this; } - - @Override public float getValue() { return value; } @Override Index: lucene/src/java/org/apache/lucene/search/IndexSearcher.java =================================================================== --- lucene/src/java/org/apache/lucene/search/IndexSearcher.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/IndexSearcher.java (working copy) @@ -66,6 +66,9 @@ protected final ReaderContext readerContext; protected final AtomicReaderContext[] leafContexts; + // This is where we keep common sub-expression information. + protected QueryDataCache dataCache = new QueryDataCache(); + // These are only used for multi-threaded search private final ExecutorService executor; protected final IndexSearcher[] subSearchers; @@ -503,7 +506,8 @@ for (int i = 0; i < leafContexts.length; i++) { // search each subreader collector.setNextReader(leafContexts[i]); scorerContext = scorerContext.scoreDocsInOrder(!collector.acceptsDocsOutOfOrder()); - Scorer scorer = weight.scorer(leafContexts[i], scorerContext); + // Mint a new scorer, but make it based on what we have already cached for the given parameters. + Scorer scorer = weight.cachedScorer(leafContexts[i], scorerContext); if (scorer != null) { scorer.score(collector); } @@ -521,7 +525,7 @@ assert filter != null; - Scorer scorer = weight.scorer(context, ScorerContext.def()); + Scorer scorer = weight.cachedScorer(context, ScorerContext.def()); if (scorer == null) { return; } @@ -566,12 +570,7 @@ * @throws BooleanQuery.TooManyClauses */ public Query rewrite(Query original) throws IOException { - Query query = original; - for (Query rewrittenQuery = query.rewrite(reader); rewrittenQuery != query; - rewrittenQuery = query.rewrite(reader)) { - query = rewrittenQuery; - } - return query; + return original.cachedRewrite(dataCache,reader); } /** Returns an Explanation that describes how doc scored against @@ -625,6 +624,15 @@ } /** + * + * get query data hash for the evaluation. + */ + public QueryDataCache getDataCache() + { + return dataCache; + } + + /** * creates a weight for query * @return new weight */ Index: lucene/src/java/org/apache/lucene/search/Weight.java =================================================================== --- lucene/src/java/org/apache/lucene/search/Weight.java (revision 1062897) +++ lucene/src/java/org/apache/lucene/search/Weight.java (working copy) @@ -56,6 +56,19 @@ */ public abstract class Weight implements Serializable { + protected Query theQuery; + protected QueryDataCache dataCache; + + protected Weight(QueryDataCache dataCache, Query theQuery) + { + this.theQuery = theQuery; + this.dataCache = dataCache; + } + + /** Make a copy, so we can cache things. + */ + public abstract Weight makeCopy(); + /** * An explanation of the score computation for the named document. * @@ -67,14 +80,32 @@ public abstract Explanation explain(AtomicReaderContext context, int doc) throws IOException; /** The query that this concerns. */ - public abstract Query getQuery(); + public Query getQuery() + { + return theQuery; + } /** The weight for this query. */ public abstract float getValue(); + /** The data cache. + */ + public QueryDataCache getCache() + { + return dataCache; + } + /** Assigns the query normalization factor to this. */ public abstract void normalize(float norm); + public Scorer cachedScorer(AtomicReaderContext context, ScorerContext scorerContext) + throws IOException + { + // Share a cached scorer. + //return dataCache.createScorer(theQuery,this,context,scorerContext); + return scorer(context,scorerContext); + } + /** * Returns a {@link Scorer} which scores documents in/out-of order according * to scoreDocsInOrder. Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java =================================================================== --- lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (revision 1062897) +++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (working copy) @@ -41,6 +41,7 @@ import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.search.spans.Spans; +import org.apache.lucene.search.QueryDataCache; import org.apache.lucene.util.StringHelper; /** @@ -151,7 +152,7 @@ } if (mtq.getField() != null) { IndexReader ir = getLeafContextForField(mtq.getField()).reader; - extract(query.rewrite(ir), terms); + extract(query.rewrite(new QueryDataCache(),ir), terms); } } else if (query instanceof MultiPhraseQuery) { final MultiPhraseQuery mpq = (MultiPhraseQuery) query; @@ -235,7 +236,7 @@ final boolean mustRewriteQuery = mustRewriteQuery(spanQuery); if (mustRewriteQuery) { for (final String field : fieldNames) { - final SpanQuery rewrittenQuery = (SpanQuery) spanQuery.rewrite(getLeafContextForField(field).reader); + final SpanQuery rewrittenQuery = (SpanQuery) spanQuery.rewrite(new QueryDataCache(),getLeafContextForField(field).reader); queries.put(field, rewrittenQuery); rewrittenQuery.extractTerms(nonWeightedTerms); } Index: lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThisQuery.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThisQuery.java (revision 1062897) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThisQuery.java (working copy) @@ -29,6 +29,7 @@ import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryDataCache; import org.apache.lucene.search.similar.MoreLikeThis; /** @@ -61,7 +62,7 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { MoreLikeThis mlt=new MoreLikeThis(reader); Index: lucene/contrib/queries/src/java/org/apache/lucene/search/BoostingQuery.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/BoostingQuery.java (revision 1062897) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/BoostingQuery.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.QueryDataCache; /** * The BoostingQuery class can be used to effectively demote results that match a given query. @@ -53,7 +54,7 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException { + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { BooleanQuery result = new BooleanQuery() { @Override public Weight createWeight(IndexSearcher searcher) throws IOException { Index: lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (revision 1062897) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (working copy) @@ -33,6 +33,7 @@ import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.PriorityQueue; +import org.apache.lucene.search.QueryDataCache; /** * Fuzzifies ALL terms provided as strings and then picks the best n differentiating terms. @@ -250,7 +251,7 @@ } @Override - public Query rewrite(IndexReader reader) throws IOException + public Query rewrite(QueryDataCache dataCache, IndexReader reader) throws IOException { if(rewrittenQuery!=null) { Index: lucene/contrib/queries/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java (revision 1062897) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java (working copy) @@ -26,6 +26,7 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.search.QueryDataCache; /** * Rewrites MultiTermQueries into a filter, using the FieldCache for term enumeration. @@ -40,7 +41,7 @@ public final class FieldCacheRewriteMethod extends MultiTermQuery.RewriteMethod { @Override - public Query rewrite(IndexReader reader, MultiTermQuery query) { + public Query rewrite(QueryDataCache dataCache, IndexReader reader, MultiTermQuery query) { Query result = new ConstantScoreQuery(new MultiTermQueryFieldCacheWrapperFilter(query)); result.setBoost(query.getBoost()); return result;