Index: lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (revision 1079707) +++ lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (revision ) @@ -18,19 +18,19 @@ */ import java.io.IOException; - -import java.util.List; -import java.util.Collection; import java.util.ArrayList; +import java.util.Collection; import java.util.Iterator; +import java.util.List; import java.util.Set; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.PriorityQueue; import org.apache.lucene.util.ToStringUtils; -import org.apache.lucene.search.Query; /** Matches the union of its clauses.*/ public class SpanOrQuery extends SpanQuery implements Cloneable { @@ -163,9 +163,9 @@ } @Override - public Spans getSpans(final AtomicReaderContext context) throws IOException { + public Spans getSpans(final AtomicReaderContext context, final Weight.ScorerContext scorerContext) throws IOException { if (clauses.size() == 1) // optimize 1-clause case - return (clauses.get(0)).getSpans(context); + return (clauses.get(0)).getSpans(context, scorerContext); return new Spans() { private SpanQueue queue = null; @@ -174,8 +174,8 @@ queue = new SpanQueue(clauses.size()); Iterator i = clauses.iterator(); while (i.hasNext()) { - Spans spans = i.next().getSpans(context); + Spans spans = i.next().getSpans(context, scorerContext); - if ( ((target == -1) && spans.next()) + if (((target == -1) && spans.next()) || ((target != -1) && spans.skipTo(target))) { queue.add(spans); } Index: lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (revision 1169470) +++ lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (revision ) @@ -80,7 +80,7 @@ @Override public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException { - return new PayloadTermSpanScorer((TermSpans) query.getSpans(context), + return new PayloadTermSpanScorer((TermSpans) query.getSpans(context, scorerContext), this, similarity.sloppyDocScorer(stats, query.getField(), context)); } @@ -194,17 +194,11 @@ payloadExpl.setValue(scorer.getPayloadScore()); // combined ComplexExplanation result = new ComplexExplanation(); - if (includeSpanScore) { - result.addDetail(expl); - result.addDetail(payloadExpl); - result.setValue(expl.getValue() * payloadExpl.getValue()); - result.setDescription("btq, product of:"); + result.addDetail(expl); + result.addDetail(payloadExpl); + result.setValue(expl.getValue() * payloadExpl.getValue()); + result.setDescription("btq, product of:"); - } else { - result.addDetail(payloadExpl); - result.setValue(payloadExpl.getValue()); - result.setDescription("btq(includeSpanScore=false), result of:"); - } - result.setMatch(true); // LUCENE-1303 + result.setMatch(expl.getValue() == 0 ? Boolean.FALSE : Boolean.TRUE); // LUCENE-1303 return result; } } Index: lucene/src/test-framework/org/apache/lucene/search/CheckHits.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/search/CheckHits.java (revision 1169470) +++ lucene/src/test-framework/org/apache/lucene/search/CheckHits.java (revision ) @@ -101,10 +101,9 @@ for (int i = -1; i < 2; i++) { actual.clear(); - IndexSearcher s = QueryUtils.wrapUnderlyingReader - (random, searcher, i); + IndexSearcher s = QueryUtils.wrapUnderlyingReader(random, searcher, i); s.search(query, c); - Assert.assertEquals("Wrap Reader " + i + ": " + + Assert.assertEquals("Wrap Reader " + i + ": s=" + s + " query=" + query.toString(defaultFieldName), correct, actual); FieldCache.DEFAULT.purge(s.getIndexReader()); // our wrapping can create insanity otherwise Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java =================================================================== --- lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (revision 1138058) +++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (revision ) @@ -249,10 +249,11 @@ AtomicReaderContext context = getLeafContextForField(field); final Spans spans; + Weight.ScorerContext scorerContext = Weight.ScorerContext.def().acceptOnlyDocs(context.reader.getLiveDocs()); if (mustRewriteQuery) { - spans = queries.get(field).getSpans(context); + spans = queries.get(field).getSpans(context, scorerContext); } else { - spans = spanQuery.getSpans(context); + spans = spanQuery.getSpans(context, scorerContext); } Index: lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java (revision 1062775) +++ lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java (revision ) @@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.ToStringUtils; import java.io.IOException; @@ -71,76 +72,76 @@ public Object clone() { SpanNotQuery spanNotQuery = new SpanNotQuery((SpanQuery)include.clone(),(SpanQuery) exclude.clone()); spanNotQuery.setBoost(getBoost()); - return spanNotQuery; + return spanNotQuery; } @Override - public Spans getSpans(final AtomicReaderContext context) throws IOException { + public Spans getSpans(final AtomicReaderContext context, final Weight.ScorerContext scorerContext) throws IOException { return new Spans() { - private Spans includeSpans = include.getSpans(context); + private Spans includeSpans = include.getSpans(context, scorerContext); - private boolean moreInclude = true; + private boolean moreInclude = true; - private Spans excludeSpans = exclude.getSpans(context); + private Spans excludeSpans = exclude.getSpans(context, scorerContext); - private boolean moreExclude = excludeSpans.next(); + private boolean moreExclude = excludeSpans.next(); - @Override - public boolean next() throws IOException { - if (moreInclude) // move to next include - moreInclude = includeSpans.next(); + @Override + public boolean next() throws IOException { + if (moreInclude) // move to next include + moreInclude = includeSpans.next(); - while (moreInclude && moreExclude) { + while (moreInclude && moreExclude) { - if (includeSpans.doc() > excludeSpans.doc()) // skip exclude - moreExclude = excludeSpans.skipTo(includeSpans.doc()); + if (includeSpans.doc() > excludeSpans.doc()) // skip exclude + moreExclude = excludeSpans.skipTo(includeSpans.doc()); - while (moreExclude // while exclude is before - && includeSpans.doc() == excludeSpans.doc() - && excludeSpans.end() <= includeSpans.start()) { - moreExclude = excludeSpans.next(); // increment exclude - } + while (moreExclude // while exclude is before + && includeSpans.doc() == excludeSpans.doc() + && excludeSpans.end() <= includeSpans.start()) { + moreExclude = excludeSpans.next(); // increment exclude + } - if (!moreExclude // if no intersection - || includeSpans.doc() != excludeSpans.doc() - || includeSpans.end() <= excludeSpans.start()) - break; // we found a match + if (!moreExclude // if no intersection + || includeSpans.doc() != excludeSpans.doc() + || includeSpans.end() <= excludeSpans.start()) + break; // we found a match - moreInclude = includeSpans.next(); // intersected: keep scanning - } - return moreInclude; - } + moreInclude = includeSpans.next(); // intersected: keep scanning + } + return moreInclude; + } - @Override - public boolean skipTo(int target) throws IOException { - if (moreInclude) // skip include - moreInclude = includeSpans.skipTo(target); + @Override + public boolean skipTo(int target) throws IOException { + if (moreInclude) // skip include + moreInclude = includeSpans.skipTo(target); - if (!moreInclude) - return false; + if (!moreInclude) + return false; - if (moreExclude // skip exclude - && includeSpans.doc() > excludeSpans.doc()) - moreExclude = excludeSpans.skipTo(includeSpans.doc()); + if (moreExclude // skip exclude + && includeSpans.doc() > excludeSpans.doc()) + moreExclude = excludeSpans.skipTo(includeSpans.doc()); - while (moreExclude // while exclude is before - && includeSpans.doc() == excludeSpans.doc() - && excludeSpans.end() <= includeSpans.start()) { - moreExclude = excludeSpans.next(); // increment exclude - } + while (moreExclude // while exclude is before + && includeSpans.doc() == excludeSpans.doc() + && excludeSpans.end() <= includeSpans.start()) { + moreExclude = excludeSpans.next(); // increment exclude + } - if (!moreExclude // if no intersection - || includeSpans.doc() != excludeSpans.doc() - || includeSpans.end() <= excludeSpans.start()) - return true; // we found a match + if (!moreExclude // if no intersection + || includeSpans.doc() != excludeSpans.doc() + || includeSpans.end() <= excludeSpans.start()) + return true; // we found a match - return next(); // scan to next match - } + return next(); // scan to next match + } - @Override - public int doc() { return includeSpans.doc(); } - @Override - public int start() { return includeSpans.start(); } - @Override - public int end() { return includeSpans.end(); } + @Override + public int doc() { return includeSpans.doc(); } + @Override + public int start() { return includeSpans.start(); } + @Override + public int end() { return includeSpans.end(); } // TODO: Remove warning after API has been finalized @Override Index: lucene/src/java/org/apache/lucene/search/TermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TermQuery.java (revision 1169470) +++ lucene/src/java/org/apache/lucene/search/TermQuery.java (revision ) @@ -74,24 +74,26 @@ @Override public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException { + final String field = term.field(); + final IndexReader reader = context.reader; assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context); - final TermsEnum termsEnum = getTermsEnum(context); - if (termsEnum == null) { + final TermState state = termStates.get(context.ord); + if (state == null) { // term is not present in that reader + assert termNotInReader(reader, field, term.bytes()) : "no termstate found but term exists in reader"; return null; } - // TODO should we reuse the DocsEnum here? - final DocsEnum docs = termsEnum.docs(context.reader.getLiveDocs(), null); + final DocsEnum docs = reader.termDocsEnum(scorerContext.acceptOnlyDocs, field, term.bytes(), state); assert docs != null; return new TermScorer(this, docs, createDocScorer(context)); } - + /** * Creates an {@link ExactDocScorer} for this {@link TermWeight}*/ ExactDocScorer createDocScorer(AtomicReaderContext context) throws IOException { return similarity.exactDocScorer(stats, term.field(), context); } - + /** * Returns a {@link TermsEnum} positioned at this weights Term or null if * the term does not exist in the given context @@ -110,7 +112,6 @@ private boolean termNotInReader(IndexReader reader, String field, BytesRef bytes) throws IOException { // only called from assert - //System.out.println("TQ.termNotInReader reader=" + reader + " term=" + field + ":" + bytes.utf8ToString()); final Terms terms = reader.terms(field); return terms == null || terms.docFreq(bytes) == 0; } Index: lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java (revision 1169470) +++ lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java (revision ) @@ -18,6 +18,7 @@ */ import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; @@ -27,6 +28,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; +import org.junit.Ignore; + import java.io.IOException; /** This class only tests some basic functionality in CSQ, the main parts are mostly @@ -130,5 +133,34 @@ if (directory != null) directory.close(); } } - + + // nocommit -- enable this test once we fixed CSQ + @Ignore + public void testConstantScoreQueryAndFilter() throws Exception { + Directory d = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random, d); + Document doc = new Document(); + doc.add(newField("field", "a", StringField.TYPE_UNSTORED)); + w.addDocument(doc); + doc = new Document(); + doc.add(newField("field", "b", StringField.TYPE_UNSTORED)); + w.addDocument(doc); + IndexReader r = w.getReader(); + w.close(); + + // Only matches first doc + //Query q = new ConstantScoreQuery(new TermQuery(new Term("field", "a"))); + Filter fa = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "b"))), CachingWrapperFilter.DeletesMode.RECACHE); + Query q = new ConstantScoreQuery(fa); + + // Only matches 2nd doc + Filter fb = new CachingWrapperFilter(new QueryWrapperFilter(new TermQuery(new Term("field", "b"))), CachingWrapperFilter.DeletesMode.RECACHE); + + IndexSearcher s = new IndexSearcher(r); + assertEquals(0, s.search(q, fb, 1).totalHits); + + r.close(); + d.close(); -} + } + +} Index: lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (revision 1171082) +++ lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (revision ) @@ -1286,6 +1286,12 @@ * with one that returns null for getSequentialSubReaders. */ public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) throws IOException { + // nocommit fix IS wrapper to enable "low" filtering for + // search methods taking Filter + + // nocommit if r has deletions can we randomly pretend + // it does not and then pass filter to search methods + // instead? if (random.nextBoolean()) { if (maybeWrap && rarely()) { r = new SlowMultiReaderWrapper(r); Index: lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java (revision 1143415) +++ lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java (revision ) @@ -505,7 +505,7 @@ /** Returns the current numeric parser ({@code null} for {@code T} is {@code String}} */ public FieldCache.Parser getParser() { return parser; } - static abstract class FieldCacheDocIdSet extends DocIdSet { + static abstract class FieldCacheDocIdSet extends DocIdSet implements Bits { private final IndexReader reader; private final boolean canIgnoreDeletedDocs; @@ -530,6 +530,26 @@ } @Override + public Bits getRandomAccessBits() { + return this; + } + + @Override + public boolean bitsIncludesDeletedDocs() { + return canIgnoreDeletedDocs; + } + + @Override + public boolean get(int docID) { + return matchDoc(docID); + } + + @Override + public int length() { + return reader.maxDoc(); + } + + @Override public DocIdSetIterator iterator() throws IOException { final Bits liveDocs = canIgnoreDeletedDocs ? null : reader.getLiveDocs(); Index: lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java =================================================================== --- lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java (revision 1062775) +++ lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java (revision ) @@ -23,9 +23,9 @@ import java.util.Iterator; import java.util.List; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexReader.ReaderContext; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; @@ -35,6 +35,7 @@ import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.Weight; import org.apache.lucene.search.spans.SpanNearQuery; import org.apache.lucene.search.spans.SpanOrQuery; import org.apache.lucene.search.spans.SpanQuery; @@ -176,7 +177,7 @@ throws IOException { final AtomicReaderContext[] leaves = ReaderUtil.leaves(context); for (AtomicReaderContext atomicReaderContext : leaves) { - final Spans spans = query.getSpans(atomicReaderContext); + final Spans spans = query.getSpans(atomicReaderContext, Weight.ScorerContext.def().acceptOnlyDocs(atomicReaderContext.reader.getLiveDocs())); while (spans.next() == true) { if (spans.isPayloadAvailable()) { Collection payload = spans.getPayload(); Index: lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java (revision 1062775) +++ lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java (revision ) @@ -18,18 +18,16 @@ */ import java.io.IOException; - - -import java.util.List; import java.util.ArrayList; import java.util.Iterator; +import java.util.List; import java.util.Set; - -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.search.Query; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.ToStringUtils; /** Matches spans which are near one another. One can specify slop, the @@ -117,16 +115,16 @@ } @Override - public Spans getSpans(final AtomicReaderContext context) throws IOException { + public Spans getSpans(final AtomicReaderContext context, Weight.ScorerContext scorerContext) throws IOException { if (clauses.size() == 0) // optimize 0-clause case - return new SpanOrQuery(getClauses()).getSpans(context); + return new SpanOrQuery(getClauses()).getSpans(context, scorerContext); if (clauses.size() == 1) // optimize 1-clause case - return clauses.get(0).getSpans(context); + return clauses.get(0).getSpans(context, scorerContext); return inOrder - ? (Spans) new NearSpansOrdered(this, context, collectPayloads) - : (Spans) new NearSpansUnordered(this, context); + ? (Spans) new NearSpansOrdered(this, context, scorerContext, collectPayloads) + : (Spans) new NearSpansUnordered(this, context, scorerContext); } @Override Index: lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (revision 1145594) +++ lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (revision ) @@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.ToStringUtils; import java.io.IOException; @@ -81,9 +82,9 @@ } @Override - public Spans getSpans(final AtomicReaderContext context) throws IOException { + public Spans getSpans(final AtomicReaderContext context, final Weight.ScorerContext scorerContext) throws IOException { final IndexReader reader = context.reader; - final DocsAndPositionsEnum postings = reader.termPositionsEnum(reader.getLiveDocs(), + final DocsAndPositionsEnum postings = reader.termPositionsEnum(scorerContext.acceptOnlyDocs, term.field(), term.bytes()); Index: lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (revision 1144158) +++ lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (revision ) @@ -38,9 +38,9 @@ private final int maxDoc; private final Bits liveDocs; - MatchAllScorer(IndexReader reader, Weight w, float score) throws IOException { + MatchAllScorer(IndexReader reader, Bits acceptOnlyDocs, Weight w, float score) throws IOException { super(w); - liveDocs = reader.getLiveDocs(); + liveDocs = acceptOnlyDocs; this.score = score; maxDoc = reader.maxDoc(); } @@ -105,7 +105,7 @@ @Override public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException { - return new MatchAllScorer(context.reader, this, queryWeight); + return new MatchAllScorer(context.reader, scorerContext.acceptOnlyDocs, this, queryWeight); } @Override Index: lucene/src/java/org/apache/lucene/util/OpenBitSet.java =================================================================== --- lucene/src/java/org/apache/lucene/util/OpenBitSet.java (revision 1143558) +++ lucene/src/java/org/apache/lucene/util/OpenBitSet.java (revision ) @@ -126,7 +126,12 @@ } /** Returns the current capacity in bits (1 greater than the index of the last bit) */ - public long capacity() { return bits.length << 6; } + public long capacity() { + return bits.length << 6; + // nocommit -- cannot do this in general: numBits isn't + // always maintained when asserts are off + // return (int) numBits; + } /** * Returns the current capacity of this set. Included for @@ -139,6 +144,9 @@ @Override public int length() { return bits.length << 6; + // nocommit -- cannot do this in general: numBits isn't + // always maintained when asserts are off + // return (int) numBits; } /** Returns true if there are no set bits */ @@ -902,6 +910,35 @@ // empty sets from returning 0, which is too common. return (int)((h>>32) ^ h) + 0x98761234; } + + // nocommit -- default to false? + private boolean includesDeletions = true; + + /** @lucene.experimental */ + public void setIncludesDeletedDocs(boolean v) { + includesDeletions = v; -} + } + /** @lucene.experimental */ + @Override + public boolean bitsIncludesDeletedDocs() { + return includesDeletions; + } + // nocommit -- default to false? + // nocommit -- would be nice if during testing we could + // default this randomly: + private boolean useRandomAccess = true; + + /** @lucene.experimental */ + public void setUseRandomAccess(boolean v) { + useRandomAccess = v; + } + + @Override + public Bits getRandomAccessBits() { + return useRandomAccess ? this : null; + } +} + + Index: lucene/src/java/org/apache/lucene/search/PhraseQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/PhraseQuery.java (revision 1169470) +++ lucene/src/java/org/apache/lucene/search/PhraseQuery.java (revision ) @@ -120,7 +120,7 @@ @Override public Query rewrite(IndexReader reader) throws IOException { - if (terms.isEmpty()) { + if (terms.isEmpty()) { // nocommit - why did Mike remove this? BooleanQuery bq = new BooleanQuery(); bq.setBoost(getBoost()); return bq; @@ -215,7 +215,6 @@ public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException { assert !terms.isEmpty(); final IndexReader reader = context.reader; - final Bits liveDocs = reader.getLiveDocs(); PostingsAndFreq[] postingsFreqs = new PostingsAndFreq[terms.size()]; for (int i = 0; i < terms.size(); i++) { final Term t = terms.get(i); @@ -224,14 +223,14 @@ assert termNotInReader(reader, field, t.bytes()) : "no termstate found but term exists in reader"; return null; } - DocsAndPositionsEnum postingsEnum = reader.termPositionsEnum(liveDocs, + DocsAndPositionsEnum postingsEnum = reader.termPositionsEnum(scorerContext.acceptOnlyDocs, t.field(), t.bytes(), state); // PhraseQuery on a field that did not index // positions. if (postingsEnum == null) { - assert (reader.termDocsEnum(liveDocs, t.field(), t.bytes(), state) != null) : "termstate found but no term exists in reader"; + assert (reader.termDocsEnum(scorerContext.acceptOnlyDocs, t.field(), t.bytes(), state) != null) : "termstate found but no term exists in reader"; // term does exist, but has no positions throw new IllegalStateException("field \"" + t.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + t.text() + ")"); } Index: lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java =================================================================== --- lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java (revision 1143415) +++ lucene/src/java/org/apache/lucene/search/CachingSpanFilter.java (revision ) @@ -16,12 +16,13 @@ */ -import org.apache.lucene.index.IndexReader; +import java.io.IOException; + import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.OpenBitSet; -import java.io.IOException; - /** * Wraps another SpanFilter's result and caches it. The purpose is to allow * filters to simply filter, and then wrap with this class to add caching. @@ -83,6 +84,10 @@ missCount++; result = filter.bitSpans(context); + // nocommit hackish: + if (result.getDocIdSet() instanceof OpenBitSet) { + ((OpenBitSet) result.getDocIdSet()).setIncludesDeletedDocs(cache.deletesMode == CachingWrapperFilter.DeletesMode.RECACHE); + } cache.put(coreKey, delCoreKey, result); return result; Index: lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (revision 1145239) +++ lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (revision ) @@ -147,6 +147,7 @@ } while (termsEnum.next() != null); // System.out.println(" done termCount=" + termCount); + // nocommit set random access here? query.incTotalNumberOfTerms(termCount); return bitSet; } else { Index: lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (revision 1169470) +++ lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (revision ) @@ -148,7 +148,7 @@ @Override public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException { - return new PayloadNearSpanScorer(query.getSpans(context), this, + return new PayloadNearSpanScorer(query.getSpans(context, scorerContext), this, similarity, similarity.sloppyDocScorer(stats, query.getField(), context)); } Index: lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (revision 1144158) +++ lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (revision ) @@ -19,14 +19,15 @@ import java.io.IOException; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanClause.Occur; // javadocs only import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.TopTermsRewrite; import org.apache.lucene.search.ScoringRewrite; -import org.apache.lucene.search.BooleanClause.Occur; // javadocs only +import org.apache.lucene.search.TopTermsRewrite; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.TermContext; /** @@ -89,7 +90,7 @@ } @Override - public Spans getSpans(AtomicReaderContext context) throws IOException { + public Spans getSpans(AtomicReaderContext context, Weight.ScorerContext scorerContext) throws IOException { throw new UnsupportedOperationException("Query should have been rewritten"); } Index: lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java =================================================================== --- lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java (revision 1145239) +++ lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java (revision ) @@ -23,7 +23,8 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.OpenBitSetDISI; import org.apache.lucene.util.Bits; /** @@ -76,7 +77,7 @@ // after de-serialize transient Map cache; - private final DeletesMode deletesMode; + final DeletesMode deletesMode; public FilterCache(DeletesMode deletesMode) { this.deletesMode = deletesMode; @@ -173,7 +174,7 @@ * by the wrapped Filter. *

This implementation returns the given {@link DocIdSet}, if {@link DocIdSet#isCacheable} * returns true, else it copies the {@link DocIdSetIterator} into - * an {@link FixedBitSet}. + * an {@link OpenBitSetDISI}. */ protected DocIdSet docIdSetToCache(DocIdSet docIdSet, IndexReader reader) throws IOException { if (docIdSet == null) { @@ -186,15 +187,9 @@ // null is allowed to be returned by iterator(), // in this case we wrap with the empty set, // which is cacheable. - if (it == null) { - return DocIdSet.EMPTY_DOCIDSET; - } else { - final FixedBitSet bits = new FixedBitSet(reader.maxDoc()); - bits.or(it); - return bits; + return (it == null) ? DocIdSet.EMPTY_DOCIDSET : new OpenBitSetDISI(it, reader.maxDoc()); - } - } + } + } - } // for testing int hitCount, missCount; @@ -216,6 +211,20 @@ // cache miss docIdSet = docIdSetToCache(filter.getDocIdSet(context), reader); + if (docIdSet instanceof OpenBitSet) { + final OpenBitSet obs = (OpenBitSet) docIdSet; + final int setCount = (int) obs.cardinality(); + + // nocommit how/where to make this hardwired 1% + // settable...? + + if (setCount > 0 && (reader.maxDoc()/setCount) < 100) { + obs.setUseRandomAccess(true); + obs.setIncludesDeletedDocs(cache.deletesMode != DeletesMode.IGNORE); + System.out.println("DORAND " + (cache.deletesMode != DeletesMode.IGNORE)); + } + } + if (docIdSet != null) { cache.put(coreKey, delCoreKey, docIdSet); } Index: lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java (revision 1079707) +++ lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java (revision ) @@ -17,16 +17,17 @@ * limitations under the License. */ -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.util.PriorityQueue; - import java.io.IOException; import java.util.ArrayList; import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.HashSet; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.PriorityQueue; + /** * Similar to {@link NearSpansOrdered}, but for the unordered case. * @@ -131,7 +132,7 @@ } - public NearSpansUnordered(SpanNearQuery query, AtomicReaderContext context) + public NearSpansUnordered(SpanNearQuery query, AtomicReaderContext context, Weight.ScorerContext scorerContext) throws IOException { this.query = query; this.slop = query.getSlop(); @@ -141,7 +142,7 @@ subSpans = new Spans[clauses.length]; for (int i = 0; i < clauses.length; i++) { SpansCell cell = - new SpansCell(clauses[i].getSpans(context), i); + new SpansCell(clauses[i].getSpans(context, scorerContext), i); ordered.add(cell); subSpans[i] = cell.spans; } Index: lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (revision 1062775) +++ lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (revision ) @@ -92,8 +92,8 @@ // ...this is done to be more consistent with things like SpanFirstQuery @Override - public Spans getSpans(AtomicReaderContext context) throws IOException { - return maskedQuery.getSpans(context); + public Spans getSpans(AtomicReaderContext context, Weight.ScorerContext scorerContext) throws IOException { + return maskedQuery.getSpans(context, scorerContext); } @Override Index: lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (revision 1144158) +++ lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (revision ) @@ -129,6 +129,9 @@ public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException { final DocIdSetIterator disi; if (filter != null) { + // nocommit we lose scorerContext.acceptOnlyDocs!! + //if (scorerContext.acceptOnlyDocs != context.reader.getLiveDocs()) { + //} assert query == null; final DocIdSet dis = filter.getDocIdSet(context); if (dis == null) { Index: lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (revision 1065327) +++ lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (revision ) @@ -17,17 +17,18 @@ */ -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.Query; - import java.io.IOException; import java.util.ArrayList; import java.util.Collection; import java.util.Set; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Weight; + /** * * @@ -81,8 +82,8 @@ protected abstract AcceptStatus acceptPosition(Spans spans) throws IOException; @Override - public Spans getSpans(final AtomicReaderContext context) throws IOException { - return new PositionCheckSpan(context); + public Spans getSpans(final AtomicReaderContext context, Weight.ScorerContext scorerContext) throws IOException { + return new PositionCheckSpan(context, scorerContext); } @@ -106,8 +107,8 @@ protected class PositionCheckSpan extends Spans { private Spans spans; - public PositionCheckSpan(AtomicReaderContext context) throws IOException { - spans = match.getSpans(context); + public PositionCheckSpan(AtomicReaderContext context, Weight.ScorerContext scorerContext) throws IOException { + spans = match.getSpans(context, scorerContext); } @Override @@ -173,4 +174,4 @@ } } -} \ No newline at end of file +} Index: lucene/src/java/org/apache/lucene/util/AndBits.java =================================================================== --- lucene/src/java/org/apache/lucene/util/AndBits.java (revision ) +++ lucene/src/java/org/apache/lucene/util/AndBits.java (revision ) @@ -0,0 +1,45 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @lucene.experimental + */ + +public final class AndBits implements Bits { + private final Bits bits1; + private final Bits bits2; + + public AndBits(Bits bits1, Bits bits2) { + this.bits1 = bits1; + this.bits2 = bits2; + // TODO: unfortunately OpenBitSet lies about its length + // (rounds up to nearest N = 0 (mod 64)): + //if (bits1.length() != bits2.length()) { + //throw new IllegalArgumentException("bit lengths differ: " + bits1.length() + " vs " + bits2.length()); + //} + } + + public boolean get(int index) { + return bits1.get(index) && bits2.get(index); + } + + public int length() { + return Math.min(bits1.length(), bits2.length()); + } +} Index: lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java (revision 1140989) +++ lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java (revision ) @@ -256,7 +256,7 @@ try { if (scorer == null) { Weight w = s.createNormalizedWeight(q); - scorer = w.scorer(readerContextArray[leafPtr], ScorerContext.def()); + scorer = w.scorer(readerContextArray[leafPtr], ScorerContext.def().acceptOnlyDocs(readerContextArray[leafPtr].reader.getLiveDocs())); } int op = order[(opidx[0]++) % order.length]; @@ -301,7 +301,7 @@ final IndexReader previousReader = lastReader[0]; IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); Weight w = indexSearcher.createNormalizedWeight(q); - Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def()); + Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def().acceptOnlyDocs(previousReader.getLiveDocs())); if (scorer != null) { boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); @@ -327,7 +327,7 @@ final IndexReader previousReader = lastReader[0]; IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader, false); Weight w = indexSearcher.createNormalizedWeight(q); - Scorer scorer = w.scorer((AtomicReaderContext)previousReader.getTopReaderContext(), ScorerContext.def()); + Scorer scorer = w.scorer((AtomicReaderContext)previousReader.getTopReaderContext(), ScorerContext.def().acceptOnlyDocs(previousReader.getLiveDocs())); if (scorer != null) { boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); @@ -358,7 +358,7 @@ long startMS = System.currentTimeMillis(); for (int i=lastDoc[0]+1; i<=doc; i++) { Weight w = s.createNormalizedWeight(q); - Scorer scorer = w.scorer(context[leafPtr], ScorerContext.def()); + Scorer scorer = w.scorer(context[leafPtr], ScorerContext.def().acceptOnlyDocs(context[leafPtr].reader.getLiveDocs())); Assert.assertTrue("query collected "+doc+" but skipTo("+i+") says no more docs!",scorer.advance(i) != DocIdSetIterator.NO_MORE_DOCS); Assert.assertEquals("query collected "+doc+" but skipTo("+i+") got to "+scorer.docID(),doc,scorer.docID()); float skipToScore = scorer.score(); @@ -385,7 +385,7 @@ final IndexReader previousReader = lastReader[0]; IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); Weight w = indexSearcher.createNormalizedWeight(q); - Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def()); + Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def().acceptOnlyDocs(previousReader.getLiveDocs())); if (scorer != null) { boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); @@ -409,7 +409,7 @@ final IndexReader previousReader = lastReader[0]; IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); Weight w = indexSearcher.createNormalizedWeight(q); - Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def()); + Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def().acceptOnlyDocs(previousReader.getLiveDocs())); if (scorer != null) { boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); Index: lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java =================================================================== --- lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java (revision 1136568) +++ lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java (revision ) @@ -56,7 +56,7 @@ return new DocIdSet() { @Override public DocIdSetIterator iterator() throws IOException { - return weight.scorer(privateContext, ScorerContext.def()); + return weight.scorer(privateContext, ScorerContext.def().acceptOnlyDocs(context.reader.getLiveDocs())); } @Override public boolean isCacheable() { return false; } Index: lucene/src/java/org/apache/lucene/search/Weight.java =================================================================== --- lucene/src/java/org/apache/lucene/search/Weight.java (revision 1169470) +++ lucene/src/java/org/apache/lucene/search/Weight.java (revision ) @@ -19,10 +19,9 @@ import java.io.IOException; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexReader.ReaderContext; -import org.apache.lucene.search.similarities.SimilarityProvider; +import org.apache.lucene.util.Bits; /** * Expert: Calculate query weights and build query scorers. @@ -139,8 +138,11 @@ */ public final boolean topScorer; + /** If non-null, the returned scorer should filter + * according to this {@link Bits} instance. */ + public final Bits acceptOnlyDocs; - + - private static final ScorerContext DEFAULT_CONTEXT = new ScorerContext(true, false); + private static final ScorerContext DEFAULT_CONTEXT = new ScorerContext(true, false, null); /** * Returns a default {@link ScorerContext} template initialized with: @@ -153,9 +155,10 @@ return DEFAULT_CONTEXT; } - private ScorerContext(boolean scoreDocsInOrder, boolean topScorer) { + private ScorerContext(boolean scoreDocsInOrder, boolean topScorer, Bits acceptOnlyDocs) { this.scoreDocsInOrder = scoreDocsInOrder; this.topScorer = topScorer; + this.acceptOnlyDocs = acceptOnlyDocs; } /** @@ -169,7 +172,7 @@ if (this.scoreDocsInOrder == scoreDocsInOrder) { return this; } - return new ScorerContext(scoreDocsInOrder, topScorer); + return new ScorerContext(scoreDocsInOrder, topScorer, acceptOnlyDocs); } /** @@ -183,7 +186,21 @@ if (this.topScorer == topScorer) { return this; } - return new ScorerContext(scoreDocsInOrder, topScorer); + return new ScorerContext(scoreDocsInOrder, topScorer, acceptOnlyDocs); } + + /** + * Creates and returns a copy of this context with the given value for + * {@link #topScorer} and returns a new instance of + * {@link ScorerContext} iff the given value differs from the + * {@link #topScorer}. Otherwise, this method has no effect and + * returns this instance. + */ + public ScorerContext acceptOnlyDocs(Bits acceptOnlyDocs) { + if (this.acceptOnlyDocs == acceptOnlyDocs) { + return this; - } + } + return new ScorerContext(scoreDocsInOrder, topScorer, acceptOnlyDocs); -} + } + } +} Index: lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java =================================================================== --- lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (revision 1169470) +++ lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (revision ) @@ -82,7 +82,7 @@ } @Override - public Spans getSpans(AtomicReaderContext context) throws IOException { + public Spans getSpans(AtomicReaderContext context, Weight.ScorerContext scorerContext) throws IOException { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } Index: lucene/src/java/org/apache/lucene/search/IndexSearcher.java =================================================================== --- lucene/src/java/org/apache/lucene/search/IndexSearcher.java (revision 1173430) +++ lucene/src/java/org/apache/lucene/search/IndexSearcher.java (revision ) @@ -42,6 +42,8 @@ import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NIOFSDirectory; // javadoc +import org.apache.lucene.util.AndBits; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.ThreadInterruptedException; @@ -276,7 +278,7 @@ } /** Finds the top n - * hits for query where all results are after a previous + * hits for query where all results are after a previous * result (after). *

* By passing the bottom result from a previous page as after, @@ -288,7 +290,7 @@ public TopDocs searchAfter(ScoreDoc after, Query query, int n) throws IOException { return searchAfter(after, query, null, n); } - + /** Finds the top n * hits for query, applying filter if non-null, * where all results are after a previous result (after). @@ -302,7 +304,7 @@ public TopDocs searchAfter(ScoreDoc after, Query query, Filter filter, int n) throws IOException { return search(createNormalizedWeight(query), filter, after, n); } - + /** Finds the top n * hits for query. * @@ -569,13 +571,14 @@ // TODO: should we make this // threaded...? the Collector could be sync'd? - ScorerContext scorerContext = ScorerContext.def().scoreDocsInOrder(true).topScorer(true); // always use single thread: if (filter == null) { + ScorerContext scorerContext = ScorerContext.def().scoreDocsInOrder(true).topScorer(true); for (int i = 0; i < leaves.length; i++) { // search each subreader collector.setNextReader(leaves[i]); - scorerContext = scorerContext.scoreDocsInOrder(!collector.acceptsDocsOutOfOrder()); - Scorer scorer = weight.scorer(leaves[i], scorerContext); + Scorer scorer = weight.scorer(leaves[i], + scorerContext.scoreDocsInOrder(!collector.acceptsDocsOutOfOrder()) + .acceptOnlyDocs(leaves[i].reader.getLiveDocs())); if (scorer != null) { scorer.score(collector); } @@ -592,47 +595,85 @@ final Filter filter, final Collector collector) throws IOException { assert filter != null; + //System.out.println("IS.searchWithFilter r=" + context.reader); - + - Scorer scorer = weight.scorer(context, ScorerContext.def()); - if (scorer == null) { - return; - } - - int docID = scorer.docID(); - assert docID == -1 || docID == DocIdSetIterator.NO_MORE_DOCS; - // CHECKME: use ConjunctionScorer here? DocIdSet filterDocIdSet = filter.getDocIdSet(context); if (filterDocIdSet == null) { // this means the filter does not accept any documents. return; } - + + ScorerContext scorerContext = ScorerContext.def(); + + Bits liveDocs = context.reader.getLiveDocs(); + //System.out.println(" liveDocs=" + liveDocs); + + Bits acceptOnlyDocs = filterDocIdSet.getRandomAccessBits(); + //System.out.println(" accOnlyDocs=" + acceptOnlyDocs); + + if (acceptOnlyDocs != null) { + + // Filter by random-access: we push the filter all the + // way down to the atomic scorers, so the bits are + // applied just like deleted docs: + if (liveDocs != null && !filterDocIdSet.bitsIncludesDeletedDocs()) { + // TODO: we could swap order of these two if we knew + // which is more restrictive?: + acceptOnlyDocs = new AndBits(acceptOnlyDocs, liveDocs); + //System.out.println(" and w/ liveDocs"); + } else { + //System.out.println(" use accOnlyDocs"); + } + + scorerContext = scorerContext.scoreDocsInOrder(!collector.acceptsDocsOutOfOrder()).topScorer(true).acceptOnlyDocs(acceptOnlyDocs); + } else if (liveDocs != null) { + scorerContext = scorerContext.acceptOnlyDocs(liveDocs); + } + + Scorer scorer = weight.scorer(context, scorerContext); + if (scorer == null) { + return; + } + + if (acceptOnlyDocs != null) { + //System.out.println("IS: filter down low f=" + scorerContext.acceptOnlyDocs); + // Filter "down low": + collector.setNextReader(context); + scorer.score(collector); + } else { + //System.out.println("IS: filter up high"); + // Filter "up high": + int docID = scorer.docID(); + assert docID == -1 || docID == DocIdSetIterator.NO_MORE_DOCS; - DocIdSetIterator filterIter = filterDocIdSet.iterator(); - if (filterIter == null) { - // this means the filter does not accept any documents. - return; - } - int filterDoc = filterIter.nextDoc(); - int scorerDoc = scorer.advance(filterDoc); - + DocIdSetIterator filterIter = filterDocIdSet.iterator(); + if (filterIter == null) { + // this means the filter does not accept any documents. + return; + } + int filterDoc = filterIter.nextDoc(); + int scorerDoc = scorer.advance(filterDoc); + + // Filter by iteration: - collector.setScorer(scorer); - while (true) { - if (scorerDoc == filterDoc) { - // Check if scorer has exhausted, only before collecting. - if (scorerDoc == DocIdSetIterator.NO_MORE_DOCS) { - break; - } + collector.setScorer(scorer); + while (true) { + if (scorerDoc == filterDoc) { + // Check if scorer has exhausted, only before collecting. + if (scorerDoc == DocIdSetIterator.NO_MORE_DOCS) { + break; + } + //System.out.println(" c=" + scorerDoc); - collector.collect(scorerDoc); - filterDoc = filterIter.nextDoc(); - scorerDoc = scorer.advance(filterDoc); - } else if (scorerDoc > filterDoc) { - filterDoc = filterIter.advance(scorerDoc); - } else { - scorerDoc = scorer.advance(filterDoc); - } - } - } + collector.collect(scorerDoc); + filterDoc = filterIter.nextDoc(); + scorerDoc = scorer.advance(filterDoc); + } else if (scorerDoc > filterDoc) { + filterDoc = filterIter.advance(scorerDoc); + } else { + scorerDoc = scorer.advance(filterDoc); + } + } + } + } /** Expert: called to re-write queries into primitive queries. * @throws BooleanQuery.TooManyClauses Index: lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java (revision 1062775) +++ lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java (revision ) @@ -28,7 +28,7 @@ public abstract class SpanQuery extends Query { /** Expert: Returns the matches for this query in an index. Used internally * to search for spans. */ - public abstract Spans getSpans(AtomicReaderContext context) throws IOException; + public abstract Spans getSpans(AtomicReaderContext context, Weight.ScorerContext scorerContext) throws IOException; /** Returns the name of the field matched by this query.*/ public abstract String getField(); Index: lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (revision 1169470) +++ lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (revision ) @@ -167,8 +167,9 @@ public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException { assert !termArrays.isEmpty(); final IndexReader reader = context.reader; - final Bits liveDocs = reader.getLiveDocs(); - + + final Bits liveDocs = scorerContext.acceptOnlyDocs; + PhraseQuery.PostingsAndFreq[] postingsFreqs = new PhraseQuery.PostingsAndFreq[termArrays.size()]; for (int pos=0; pos 1) { - postingsEnum = new UnionDocsAndPositionsEnum(reader, terms); + postingsEnum = new UnionDocsAndPositionsEnum(scorerContext.acceptOnlyDocs, reader, terms); // coarse -- this overcounts since a given doc can // have more than one terms: @@ -188,7 +189,7 @@ } } else { final Term term = terms[0]; - postingsEnum = reader.termPositionsEnum(liveDocs, + postingsEnum = reader.termPositionsEnum(scorerContext.acceptOnlyDocs, term.field(), term.bytes()); @@ -249,11 +250,7 @@ @Override public Query rewrite(IndexReader reader) { - if (termArrays.isEmpty()) { - BooleanQuery bq = new BooleanQuery(); - bq.setBoost(getBoost()); - return bq; - } else if (termArrays.size() == 1) { // optimize one-term case + if (termArrays.size() == 1) { // optimize one-term case Term[] terms = termArrays.get(0); BooleanQuery boq = new BooleanQuery(true); for (int i=0; i docsEnums = new LinkedList(); - final Bits liveDocs = indexReader.getLiveDocs(); + for (int i = 0; i < terms.length; i++) { - DocsAndPositionsEnum postings = indexReader.termPositionsEnum(liveDocs, + DocsAndPositionsEnum postings = indexReader.termPositionsEnum(acceptOnlyDocs, terms[i].field(), terms[i].bytes()); if (postings != null) { docsEnums.add(postings); } else { - if (indexReader.termDocsEnum(liveDocs, terms[i].field(), terms[i].bytes()) != null) { + if (indexReader.termDocsEnum(acceptOnlyDocs, terms[i].field(), terms[i].bytes()) != null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + terms[i].field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + terms[i].text() + ")"); } Index: lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java =================================================================== --- lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java (revision 1145239) +++ lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java (revision ) @@ -20,6 +20,7 @@ import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.Spans; import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.util.OpenBitSet; import java.io.IOException; import java.util.ArrayList; @@ -61,7 +62,7 @@ public SpanFilterResult bitSpans(AtomicReaderContext context) throws IOException { final FixedBitSet bits = new FixedBitSet(context.reader.maxDoc()); - Spans spans = query.getSpans(context); + Spans spans = query.getSpans(context, Weight.ScorerContext.def().acceptOnlyDocs(context.reader.getLiveDocs())); List tmp = new ArrayList(20); int currentDoc = -1; SpanFilterResult.PositionInfo currentInfo = null; Index: lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java (revision 1099041) +++ lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java (revision ) @@ -17,18 +17,19 @@ * limitations under the License. */ -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.util.ArrayUtil; - import java.io.IOException; import java.util.ArrayList; +import java.util.Collection; import java.util.Comparator; import java.util.HashSet; import java.util.LinkedList; import java.util.List; -import java.util.Collection; import java.util.Set; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.ArrayUtil; + /** A Spans that is formed from the ordered subspans of a SpanNearQuery * where the subspans do not overlap and have a maximum slop between them. *

@@ -77,11 +78,11 @@ private SpanNearQuery query; private boolean collectPayloads = true; - public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context) throws IOException { - this(spanNearQuery, context, true); + public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context, Weight.ScorerContext scorerContext) throws IOException { + this(spanNearQuery, context, scorerContext, true); } - public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context, boolean collectPayloads) + public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context, Weight.ScorerContext scorerContext, boolean collectPayloads) throws IOException { if (spanNearQuery.getClauses().length < 2) { throw new IllegalArgumentException("Less than 2 clauses: " @@ -94,7 +95,7 @@ matchPayload = new LinkedList(); subSpansByDoc = new Spans[clauses.length]; for (int i = 0; i < clauses.length; i++) { - subSpans[i] = clauses[i].getSpans(context); + subSpans[i] = clauses[i].getSpans(context, scorerContext); subSpansByDoc[i] = subSpans[i]; // used in toSameDoc() } query = spanNearQuery; // kept for toString() only. Index: lucene/src/java/org/apache/lucene/search/DocIdSet.java =================================================================== --- lucene/src/java/org/apache/lucene/search/DocIdSet.java (revision 830661) +++ lucene/src/java/org/apache/lucene/search/DocIdSet.java (revision ) @@ -19,6 +19,8 @@ import java.io.IOException; +import org.apache.lucene.util.Bits; + /** * A DocIdSet contains a set of doc ids. Implementing classes must * only implement {@link #iterator} to provide access to the set. @@ -46,6 +48,11 @@ public boolean isCacheable() { return true; } + + @Override + public Bits getRandomAccessBits() { + return null; + } }; /** Provides a {@link DocIdSetIterator} to access the set. @@ -64,4 +71,23 @@ public boolean isCacheable() { return false; } + + /** Only used if {@link #getRandomAccessBits} returns non-null result; + * return true if the Bits instance has already factored + * in deletions. */ + // nocommit better name...? somewhere else...? move to + // subclass...? + public boolean bitsIncludesDeletedDocs() { + return false; -} + } + + /** Return a Bits impl if this DocIdSet should be applied + * via random-access (because the underlying bit set + * implementation supports random access, and the filter + * is dense enough), instead of {@link DocIdSetIterator}. */ + // nocommit better name...? somewhere else...? move to + // subclass...? + public Bits getRandomAccessBits() { + return null; + } +} Index: lucene/src/java/org/apache/lucene/search/BooleanQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/BooleanQuery.java (revision 1169470) +++ lucene/src/java/org/apache/lucene/search/BooleanQuery.java (revision ) @@ -310,9 +310,10 @@ List prohibited = new ArrayList(); List optional = new ArrayList(); Iterator cIter = clauses.iterator(); + final ScorerContext subScorerContext = ScorerContext.def().acceptOnlyDocs(scorerContext.acceptOnlyDocs); for (Weight w : weights) { BooleanClause c = cIter.next(); - Scorer subScorer = w.scorer(context, ScorerContext.def()); + Scorer subScorer = w.scorer(context, subScorerContext); if (subScorer == null) { if (c.isRequired()) { return null; Index: lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java =================================================================== --- lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java (revision 1169470) +++ lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java (revision ) @@ -68,7 +68,7 @@ @Override public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException { - return new SpanScorer(query.getSpans(context), this, similarity.sloppyDocScorer(stats, query.getField(), context)); + return new SpanScorer(query.getSpans(context, scorerContext), this, similarity.sloppyDocScorer(stats, query.getField(), context)); } @Override Index: lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java =================================================================== --- lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java (revision 1065327) +++ lucene/src/test/org/apache/lucene/search/spans/MultiSpansWrapper.java (revision ) @@ -24,6 +24,7 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexReader.ReaderContext; +import org.apache.lucene.search.Weight; import org.apache.lucene.util.ReaderUtil; /** @@ -48,8 +49,8 @@ public static Spans wrap(ReaderContext topLevelReaderContext, SpanQuery query) throws IOException { AtomicReaderContext[] leaves = ReaderUtil.leaves(topLevelReaderContext); - if(leaves.length == 1) { + if (leaves.length == 1) { - return query.getSpans(leaves[0]); + return query.getSpans(leaves[0], Weight.ScorerContext.def().acceptOnlyDocs(leaves[0].reader.getLiveDocs())); } return new MultiSpansWrapper(leaves, query); } @@ -60,14 +61,14 @@ return false; } if (current == null) { - current = query.getSpans(leaves[leafOrd]); + current = query.getSpans(leaves[leafOrd], Weight.ScorerContext.def().acceptOnlyDocs(leaves[leafOrd].reader.getLiveDocs())); } while(true) { if (current.next()) { return true; } if (++leafOrd < leaves.length) { - current = query.getSpans(leaves[leafOrd]); + current = query.getSpans(leaves[leafOrd], Weight.ScorerContext.def().acceptOnlyDocs(leaves[leafOrd].reader.getLiveDocs())); } else { current = null; break; @@ -85,17 +86,17 @@ int subIndex = ReaderUtil.subIndex(target, leaves); assert subIndex >= leafOrd; if (subIndex != leafOrd) { - current = query.getSpans(leaves[subIndex]); + current = query.getSpans(leaves[subIndex], Weight.ScorerContext.def().acceptOnlyDocs(leaves[leafOrd].reader.getLiveDocs())); leafOrd = subIndex; } else if (current == null) { - current = query.getSpans(leaves[leafOrd]); + current = query.getSpans(leaves[leafOrd], Weight.ScorerContext.def().acceptOnlyDocs(leaves[leafOrd].reader.getLiveDocs())); } while (true) { if (current.skipTo(target - leaves[leafOrd].docBase)) { return true; } if (++leafOrd < leaves.length) { - current = query.getSpans(leaves[leafOrd]); + current = query.getSpans(leaves[leafOrd], Weight.ScorerContext.def().acceptOnlyDocs(leaves[leafOrd].reader.getLiveDocs())); } else { current = null; break;