Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsAndPositionsEnum.java --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsAndPositionsEnum.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsAndPositionsEnum.java Sun Jun 26 14:21:07 2011 -0400 @@ -23,13 +23,13 @@ public class InstantiatedDocsAndPositionsEnum extends DocsAndPositionsEnum { private int upto; private int posUpto; - private Bits skipDocs; + private Bits keepDocs; private InstantiatedTerm term; protected InstantiatedTermDocumentInformation currentDoc; private final BytesRef payload = new BytesRef(); - public InstantiatedDocsAndPositionsEnum reset(Bits skipDocs, InstantiatedTerm term) { - this.skipDocs = skipDocs; + public InstantiatedDocsAndPositionsEnum reset(Bits keepDocs, InstantiatedTerm term) { + this.keepDocs = keepDocs; this.term = term; upto = -1; return this; @@ -47,7 +47,7 @@ return NO_MORE_DOCS; } else { currentDoc = term.getAssociatedDocuments()[upto]; - if (skipDocs == null || !skipDocs.get(currentDoc.getDocument().getDocumentNumber())) { + if (keepDocs == null || keepDocs.get(currentDoc.getDocument().getDocumentNumber())) { posUpto = -1; return docID(); } else { @@ -69,7 +69,7 @@ } currentDoc = term.getAssociatedDocuments()[upto]; - if (skipDocs != null && skipDocs.get(currentDoc.getDocument().getDocumentNumber())) { + if (keepDocs != null && !keepDocs.get(currentDoc.getDocument().getDocumentNumber())) { return nextDoc(); } else { posUpto = -1; Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsEnum.java --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsEnum.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedDocsEnum.java Sun Jun 26 14:21:07 2011 -0400 @@ -21,12 +21,12 @@ public class InstantiatedDocsEnum extends DocsEnum { private int upto; - private Bits skipDocs; + private Bits keepDocs; private InstantiatedTerm term; protected InstantiatedTermDocumentInformation currentDoc; - public InstantiatedDocsEnum reset(Bits skipDocs, InstantiatedTerm term) { - this.skipDocs = skipDocs; + public InstantiatedDocsEnum reset(Bits keepDocs, InstantiatedTerm term) { + this.keepDocs = keepDocs; this.term = term; upto = -1; return this; @@ -44,7 +44,7 @@ return NO_MORE_DOCS; } else { currentDoc = term.getAssociatedDocuments()[upto]; - if (skipDocs == null || !skipDocs.get(currentDoc.getDocument().getDocumentNumber())) { + if (keepDocs == null || keepDocs.get(currentDoc.getDocument().getDocumentNumber())) { return docID(); } else { return nextDoc(); @@ -65,7 +65,7 @@ } currentDoc = term.getAssociatedDocuments()[upto]; - if (skipDocs != null && skipDocs.get(currentDoc.getDocument().getDocumentNumber())) { + if (keepDocs != null && !keepDocs.get(currentDoc.getDocument().getDocumentNumber())) { return nextDoc(); } else { return docID(); Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndex.java Sun Jun 26 14:21:07 2011 -0400 @@ -182,9 +182,9 @@ } // create documents - final Bits delDocs = MultiFields.getDeletedDocs(sourceIndexReader); + final Bits notDelDocs = MultiFields.getNotDeletedDocs(sourceIndexReader); for (int i = 0; i < sourceIndexReader.maxDoc(); i++) { - if (delDocs != null && delDocs.get(i)) { + if (notDelDocs != null && !notDelDocs.get(i)) { deletedDocuments.set(i); } else { InstantiatedDocument document = new InstantiatedDocument(); @@ -254,7 +254,7 @@ // create term-document informations for (InstantiatedTerm term : orderedTerms) { DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(sourceIndexReader, - MultiFields.getDeletedDocs(sourceIndexReader), + MultiFields.getNotDeletedDocs(sourceIndexReader), term.getTerm().field(), new BytesRef(term.getTerm().text())); int position = 0; Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedTermsEnum.java Sun Jun 26 14:21:07 2011 -0400 @@ -118,19 +118,19 @@ } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) { + public DocsEnum docs(Bits keepDocs, DocsEnum reuse) { if (reuse == null || !(reuse instanceof InstantiatedDocsEnum)) { reuse = new InstantiatedDocsEnum(); } - return ((InstantiatedDocsEnum) reuse).reset(skipDocs, terms[upto]); + return ((InstantiatedDocsEnum) reuse).reset(keepDocs, terms[upto]); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) { + public DocsAndPositionsEnum docsAndPositions(Bits keepDocs, DocsAndPositionsEnum reuse) { if (reuse == null || !(reuse instanceof InstantiatedDocsAndPositionsEnum)) { reuse = new InstantiatedDocsAndPositionsEnum(); } - return ((InstantiatedDocsAndPositionsEnum) reuse).reset(skipDocs, terms[upto]); + return ((InstantiatedDocsAndPositionsEnum) reuse).reset(keepDocs, terms[upto]); } @Override Index: lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java --- lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java Sun Jun 26 14:21:07 2011 -0400 @@ -138,8 +138,8 @@ testTermEnum.seekCeil(new BytesRef(t.text())); assertEquals(aprioriTermEnum.term(), testTermEnum.term()); - DocsEnum aprioriTermDocs = aprioriTermEnum.docs(MultiFields.getDeletedDocs(aprioriReader), null); - DocsEnum testTermDocs = testTermEnum.docs(MultiFields.getDeletedDocs(testReader), null); + DocsEnum aprioriTermDocs = aprioriTermEnum.docs(MultiFields.getNotDeletedDocs(aprioriReader), null); + DocsEnum testTermDocs = testTermEnum.docs(MultiFields.getNotDeletedDocs(testReader), null); assertEquals(aprioriTermDocs.nextDoc(), testTermDocs.nextDoc()); assertEquals(aprioriTermDocs.freq(), testTermDocs.freq()); @@ -186,8 +186,8 @@ assertEquals(aprioriTermEnum.next(), testTermEnum.next()); - aprioriTermDocs = aprioriTermEnum.docs(MultiFields.getDeletedDocs(aprioriReader), aprioriTermDocs); - testTermDocs = testTermEnum.docs(MultiFields.getDeletedDocs(testReader), testTermDocs); + aprioriTermDocs = aprioriTermEnum.docs(MultiFields.getNotDeletedDocs(aprioriReader), aprioriTermDocs); + testTermDocs = testTermEnum.docs(MultiFields.getNotDeletedDocs(testReader), testTermDocs); while (aprioriTermDocs.nextDoc() != DocsEnum.NO_MORE_DOCS) { assertTrue(testTermDocs.nextDoc() != DocsEnum.NO_MORE_DOCS); @@ -309,13 +309,13 @@ assertEquals(air.numDocs(), tir.numDocs()); assertEquals(air.numDeletedDocs(), tir.numDeletedDocs()); - final Bits aDelDocs = MultiFields.getDeletedDocs(air); - final Bits tDelDocs = MultiFields.getDeletedDocs(tir); - assertTrue((aDelDocs != null && tDelDocs != null) || - (aDelDocs == null && tDelDocs == null)); - if (aDelDocs != null) { + final Bits aNotDelDocs = MultiFields.getNotDeletedDocs(air); + final Bits tNotDelDocs = MultiFields.getNotDeletedDocs(tir); + assertTrue((aNotDelDocs != null && tNotDelDocs != null) || + (aNotDelDocs == null && tNotDelDocs == null)); + if (aNotDelDocs != null) { for (int d =0; d 1) { // unset potential duplicates - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(notDelDocs, docs); int doc = docs.nextDoc(); if (doc != DocsEnum.NO_MORE_DOCS) { if (keepMode == KM_USE_FIRST_OCCURRENCE) { Index: lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java --- lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java Sun Jun 26 14:21:07 2011 -0400 @@ -63,7 +63,7 @@ OpenBitSet result=new OpenBitSet(reader.maxDoc()); Fields fields = reader.fields(); BytesRef br = new BytesRef(); - Bits delDocs = reader.getDeletedDocs(); + Bits notDelDocs = reader.getNotDeletedDocs(); if (fields != null) { String lastField = null; Terms termsC = null; @@ -80,7 +80,7 @@ if (terms != null) { br.copy(term.bytes()); if (termsEnum.seekCeil(br) == TermsEnum.SeekStatus.FOUND) { - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(notDelDocs, docs); while(docs.nextDoc() != DocsEnum.NO_MORE_DOCS) { result.set(docs.docID()); } Index: lucene/contrib/queries/src/test/org/apache/lucene/search/DuplicateFilterTest.java --- lucene/contrib/queries/src/test/org/apache/lucene/search/DuplicateFilterTest.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/contrib/queries/src/test/org/apache/lucene/search/DuplicateFilterTest.java Sun Jun 26 14:21:07 2011 -0400 @@ -139,7 +139,7 @@ Document d=searcher.doc(hits[i].doc); String url=d.get(KEY_FIELD); DocsEnum td = MultiFields.getTermDocsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getNotDeletedDocs(reader), KEY_FIELD, new BytesRef(url)); int lastDoc=0; @@ -163,7 +163,7 @@ Document d=searcher.doc(hits[i].doc); String url=d.get(KEY_FIELD); DocsEnum td = MultiFields.getTermDocsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getNotDeletedDocs(reader), KEY_FIELD, new BytesRef(url)); int lastDoc=0; Index: lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java --- lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java Sun Jun 26 14:21:07 2011 -0400 @@ -46,7 +46,7 @@ @Override public DocIdSet getDocIdSet(final AtomicReaderContext context) throws IOException { - final Bits delDocs = context.reader.getDeletedDocs(); + final Bits notDelDocs = context.reader.getNotDeletedDocs(); final List area = shape.getArea(); final int sz = area.size(); @@ -58,7 +58,7 @@ return new DocIdSet() { @Override public DocIdSetIterator iterator() throws IOException { - return context.reader.termDocsEnum(delDocs, fieldName, bytesRef); + return context.reader.termDocsEnum(notDelDocs, fieldName, bytesRef); } @Override @@ -71,7 +71,7 @@ for (int i =0; i< sz; i++) { double boxId = area.get(i).doubleValue(); NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(boxId), 0, bytesRef); - final DocsEnum docsEnum = context.reader.termDocsEnum(delDocs, fieldName, bytesRef); + final DocsEnum docsEnum = context.reader.termDocsEnum(notDelDocs, fieldName, bytesRef); if (docsEnum == null) continue; // iterate through all documents // which have this boxId Index: lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java --- lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java Sun Jun 26 14:21:07 2011 -0400 @@ -399,7 +399,7 @@ // System.out.println(" term=" + term); if (termsEnum.seekExact(term.bytes(), false)) { - DocsEnum docsEnum = termsEnum.docs(reader.getDeletedDocs(), docs); + DocsEnum docsEnum = termsEnum.docs(reader.getNotDeletedDocs(), docs); if (docsEnum != null) { while (true) { Index: lucene/src/java/org/apache/lucene/index/CheckIndex.java --- lucene/src/java/org/apache/lucene/index/CheckIndex.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/CheckIndex.java Sun Jun 26 14:21:07 2011 -0400 @@ -654,7 +654,7 @@ final Status.TermIndexStatus status = new Status.TermIndexStatus(); final int maxDoc = reader.maxDoc(); - final Bits delDocs = reader.getDeletedDocs(); + final Bits acceptDocs = reader.getNotDeletedDocs(); final IndexSearcher is = new IndexSearcher(reader); @@ -712,8 +712,8 @@ final int docFreq = terms.docFreq(); status.totFreq += docFreq; - docs = terms.docs(delDocs, docs); - postings = terms.docsAndPositions(delDocs, postings); + docs = terms.docs(acceptDocs, docs); + postings = terms.docsAndPositions(acceptDocs, postings); if (hasOrd) { long ord = -1; @@ -815,7 +815,7 @@ if (hasPositions) { for(int idx=0;idx<7;idx++) { final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8); - postings = terms.docsAndPositions(delDocs, postings); + postings = terms.docsAndPositions(acceptDocs, postings); final int docID = postings.advance(skipDocID); if (docID == DocsEnum.NO_MORE_DOCS) { break; @@ -851,7 +851,7 @@ } else { for(int idx=0;idx<7;idx++) { final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8); - docs = terms.docs(delDocs, docs); + docs = terms.docs(acceptDocs, docs); final int docID = docs.advance(skipDocID); if (docID == DocsEnum.NO_MORE_DOCS) { break; @@ -919,7 +919,7 @@ throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed"); } - docs = terms.docs(delDocs, docs); + docs = terms.docs(acceptDocs, docs); if (docs == null) { throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]); } @@ -967,9 +967,9 @@ } // Scan stored fields for all documents - final Bits delDocs = reader.getDeletedDocs(); + final Bits acceptDocs = reader.getNotDeletedDocs(); for (int j = 0; j < info.docCount; ++j) { - if (delDocs == null || !delDocs.get(j)) { + if (acceptDocs == null || acceptDocs.get(j)) { status.docCount++; Document doc = reader.document(j); status.totFields += doc.getFields().size(); @@ -1059,9 +1059,9 @@ infoStream.print(" test: term vectors........"); } - final Bits delDocs = reader.getDeletedDocs(); + final Bits acceptDocs = reader.getNotDeletedDocs(); for (int j = 0; j < info.docCount; ++j) { - if (delDocs == null || !delDocs.get(j)) { + if (acceptDocs == null || acceptDocs.get(j)) { status.docCount++; TermFreqVector[] tfv = reader.getTermFreqVectors(j); if (tfv != null) { Index: lucene/src/java/org/apache/lucene/index/DocTermOrds.java --- lucene/src/java/org/apache/lucene/index/DocTermOrds.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/DocTermOrds.java Sun Jun 26 14:21:07 2011 -0400 @@ -249,7 +249,7 @@ boolean testedOrd = false; - final Bits delDocs = MultiFields.getDeletedDocs(reader); + final Bits acceptDocs = MultiFields.getNotDeletedDocs(reader); // we need a minimum of 9 bytes, but round up to 12 since the space would // be wasted with most allocators anyway. @@ -312,7 +312,7 @@ final int df = te.docFreq(); if (df <= maxTermDocFreq) { - docsEnum = te.docs(delDocs, docsEnum); + docsEnum = te.docs(acceptDocs, docsEnum); final DocsEnum.BulkReadResult bulkResult = docsEnum.getBulkResult(); @@ -653,13 +653,13 @@ } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { - return termsEnum.docs(skipDocs, reuse); + public DocsEnum docs(Bits acceptDocs, DocsEnum reuse) throws IOException { + return termsEnum.docs(acceptDocs, reuse); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { - return termsEnum.docsAndPositions(skipDocs, reuse); + public DocsAndPositionsEnum docsAndPositions(Bits acceptDocs, DocsAndPositionsEnum reuse) throws IOException { + return termsEnum.docsAndPositions(acceptDocs, reuse); } @Override Index: lucene/src/java/org/apache/lucene/index/FilterIndexReader.java --- lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Sun Jun 26 14:21:07 2011 -0400 @@ -87,13 +87,13 @@ } @Override - public DocsEnum docs(Bits skipDocs, BytesRef text, DocsEnum reuse) throws IOException { - return in.docs(skipDocs, text, reuse); + public DocsEnum docs(Bits acceptDocs, BytesRef text, DocsEnum reuse) throws IOException { + return in.docs(acceptDocs, text, reuse); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, BytesRef text, DocsAndPositionsEnum reuse) throws IOException { - return in.docsAndPositions(skipDocs, text, reuse); + public DocsAndPositionsEnum docsAndPositions(Bits acceptDocs, BytesRef text, DocsAndPositionsEnum reuse) throws IOException { + return in.docsAndPositions(acceptDocs, text, reuse); } @Override @@ -172,13 +172,13 @@ } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { - return in.docs(skipDocs, reuse); + public DocsEnum docs(Bits acceptDocs, DocsEnum reuse) throws IOException { + return in.docs(acceptDocs, reuse); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { - return in.docsAndPositions(skipDocs, reuse); + public DocsAndPositionsEnum docsAndPositions(Bits acceptDocs, DocsAndPositionsEnum reuse) throws IOException { + return in.docsAndPositions(acceptDocs, reuse); } @Override @@ -304,6 +304,13 @@ public Bits getDeletedDocs() { return in.getDeletedDocs(); } + + /* + @Override + public Bits getNotDeletedDocs() { + return in.getNotDeletedDocs(); + } + */ @Override public TermFreqVector[] getTermFreqVectors(int docNumber) Index: lucene/src/java/org/apache/lucene/index/IndexReader.java --- lucene/src/java/org/apache/lucene/index/IndexReader.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/IndexReader.java Sun Jun 26 14:21:07 2011 -0400 @@ -29,6 +29,7 @@ import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.NotBits; import org.apache.lucene.util.ReaderUtil; // for javadocs import java.io.File; @@ -1136,7 +1137,7 @@ /** Returns {@link DocsEnum} for the specified field & * term. This may return null, if either the field or * term does not exist. */ - public DocsEnum termDocsEnum(Bits skipDocs, String field, BytesRef term) throws IOException { + public DocsEnum termDocsEnum(Bits acceptDocs, String field, BytesRef term) throws IOException { assert field != null; assert term != null; final Fields fields = fields(); @@ -1145,7 +1146,7 @@ } final Terms terms = fields.terms(field); if (terms != null) { - return terms.docs(skipDocs, term, null); + return terms.docs(acceptDocs, term, null); } else { return null; } @@ -1155,7 +1156,7 @@ * field & term. This may return null, if either the * field or term does not exist, or, positions were not * stored for this term. */ - public DocsAndPositionsEnum termPositionsEnum(Bits skipDocs, String field, BytesRef term) throws IOException { + public DocsAndPositionsEnum termPositionsEnum(Bits acceptDocs, String field, BytesRef term) throws IOException { assert field != null; assert term != null; final Fields fields = fields(); @@ -1164,7 +1165,7 @@ } final Terms terms = fields.terms(field); if (terms != null) { - return terms.docsAndPositions(skipDocs, term, null); + return terms.docsAndPositions(acceptDocs, term, null); } else { return null; } @@ -1175,7 +1176,7 @@ * {@link TermState}. This may return null, if either the field or the term * does not exists or the {@link TermState} is invalid for the underlying * implementation.*/ - public DocsEnum termDocsEnum(Bits skipDocs, String field, BytesRef term, TermState state) throws IOException { + public DocsEnum termDocsEnum(Bits acceptDocs, String field, BytesRef term, TermState state) throws IOException { assert state != null; assert field != null; final Fields fields = fields(); @@ -1184,7 +1185,7 @@ } final Terms terms = fields.terms(field); if (terms != null) { - return terms.docs(skipDocs, term, state, null); + return terms.docs(acceptDocs, term, state, null); } else { return null; } @@ -1195,7 +1196,7 @@ * {@link TermState}. This may return null, if either the field or the term * does not exists, the {@link TermState} is invalid for the underlying * implementation, or positions were not stored for this term.*/ - public DocsAndPositionsEnum termPositionsEnum(Bits skipDocs, String field, BytesRef term, TermState state) throws IOException { + public DocsAndPositionsEnum termPositionsEnum(Bits acceptDocs, String field, BytesRef term, TermState state) throws IOException { assert state != null; assert field != null; final Fields fields = fields(); @@ -1204,7 +1205,7 @@ } final Terms terms = fields.terms(field); if (terms != null) { - return terms.docsAndPositions(skipDocs, term, state, null); + return terms.docsAndPositions(acceptDocs, term, state, null); } else { return null; } @@ -1260,7 +1261,7 @@ public int deleteDocuments(Term term) throws StaleReaderException, CorruptIndexException, LockObtainFailedException, IOException { ensureOpen(); DocsEnum docs = MultiFields.getTermDocsEnum(this, - MultiFields.getDeletedDocs(this), + MultiFields.getNotDeletedDocs(this), term.field(), term.bytes()); if (docs == null) return 0; @@ -1393,8 +1394,20 @@ * The returned instance has been safely published for use by * multiple threads without additional synchronization. * @lucene.experimental */ + // nocommit remove public abstract Bits getDeletedDocs(); + // nocommit make abstract; make impls create this on init! + /** Return null if all docs are accepted. */ + public Bits getNotDeletedDocs() { + final Bits delDocs = getDeletedDocs(); + if (delDocs == null) { + return null; + } else { + return new NotBits(delDocs); + } + } + /** * Expert: return the IndexCommit that this reader has * opened. This method is only implemented by those Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java --- lucene/src/java/org/apache/lucene/index/IndexWriter.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java Sun Jun 26 14:21:07 2011 -0400 @@ -2931,9 +2931,9 @@ // Reader was skipped because it was 100% deletions continue; } - final Bits prevDelDocs = previousReader.getDeletedDocs(); + final Bits prevAcceptDocs = previousReader.getNotDeletedDocs(); final SegmentReader currentReader = merge.readers.get(i); - final Bits currentDelDocs = currentReader.getDeletedDocs(); + final Bits currentAcceptDocs = currentReader.getNotDeletedDocs(); if (previousReader.hasDeletions()) { // There were deletes on this segment when the merge @@ -2948,10 +2948,10 @@ // committed since we started the merge, so we // must merge them: for(int j=0;j delDocs = new ArrayList(); + final List notDelDocs = new ArrayList(); final List starts = new ArrayList(); try { final int maxDoc = new ReaderUtil.Gather(r) { @Override protected void add(int base, IndexReader r) throws IOException { - // record all delDocs, even if they are null - delDocs.add(r.getDeletedDocs()); + // record all notDelDocs, even if they are null + notDelDocs.add(r.getNotDeletedDocs()); starts.add(base); } }.run(); @@ -122,12 +123,12 @@ throw new RuntimeException(ioe); } - assert delDocs.size() > 0; - if (delDocs.size() == 1) { + assert notDelDocs.size() > 0; + if (notDelDocs.size() == 1) { // Only one actual sub reader -- optimize this case - result = delDocs.get(0); + result = notDelDocs.get(0); } else { - result = new MultiBits(delDocs, starts); + result = new MultiBits(notDelDocs, starts, true); } } else { @@ -137,6 +138,12 @@ return result; } + // nocommit remove + public static Bits getDeletedDocs(IndexReader r) { + final Bits acceptDocs = getNotDeletedDocs(r); + return acceptDocs == null ? null : new NotBits(acceptDocs); + } + /** This method may return null if the field does not exist.*/ public static Terms getTerms(IndexReader r, String field) throws IOException { final Fields fields = getFields(r); @@ -150,12 +157,12 @@ /** Returns {@link DocsEnum} for the specified field & * term. This may return null if the term does not * exist. */ - public static DocsEnum getTermDocsEnum(IndexReader r, Bits skipDocs, String field, BytesRef term) throws IOException { + public static DocsEnum getTermDocsEnum(IndexReader r, Bits acceptDocs, String field, BytesRef term) throws IOException { assert field != null; assert term != null; final Terms terms = getTerms(r, field); if (terms != null) { - return terms.docs(skipDocs, term, null); + return terms.docs(acceptDocs, term, null); } else { return null; } @@ -164,12 +171,12 @@ /** Returns {@link DocsAndPositionsEnum} for the specified * field & term. This may return null if the term does * not exist or positions were not indexed. */ - public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits skipDocs, String field, BytesRef term) throws IOException { + public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits acceptDocs, String field, BytesRef term) throws IOException { assert field != null; assert term != null; final Terms terms = getTerms(r, field); if (terms != null) { - return terms.docsAndPositions(skipDocs, term, null); + return terms.docsAndPositions(acceptDocs, term, null); } else { return null; } Index: lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java --- lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java Sun Jun 26 14:21:07 2011 -0400 @@ -331,7 +331,7 @@ } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits acceptDocs, DocsEnum reuse) throws IOException { final MultiDocsEnum docsEnum; if (reuse != null) { docsEnum = (MultiDocsEnum) reuse; @@ -339,11 +339,11 @@ docsEnum = new MultiDocsEnum(); } - final MultiBits multiSkipDocs; - if (skipDocs instanceof MultiBits) { - multiSkipDocs = (MultiBits) skipDocs; + final MultiBits multiAcceptDocs; + if (acceptDocs instanceof MultiBits) { + multiAcceptDocs = (MultiBits) acceptDocs; } else { - multiSkipDocs = null; + multiAcceptDocs = null; } int upto = 0; @@ -354,22 +354,22 @@ final Bits b; - if (multiSkipDocs != null) { + if (multiAcceptDocs != null) { // optimize for common case: requested skip docs is a // congruent sub-slice of MultiBits: in this case, we - // just pull the skipDocs from the sub reader, rather + // just pull the acceptDocs from the sub reader, rather // than making the inefficient // Slice(Multi(sub-readers)): - final MultiBits.SubResult sub = multiSkipDocs.getMatchingSub(entry.subSlice); + final MultiBits.SubResult sub = multiAcceptDocs.getMatchingSub(entry.subSlice); if (sub.matches) { b = sub.result; } else { // custom case: requested skip docs is foreign: // must slice it on every access - b = new BitsSlice(skipDocs, entry.subSlice); + b = new BitsSlice(acceptDocs, entry.subSlice); } - } else if (skipDocs != null) { - b = new BitsSlice(skipDocs, entry.subSlice); + } else if (acceptDocs != null) { + b = new BitsSlice(acceptDocs, entry.subSlice); } else { // no deletions b = null; @@ -392,7 +392,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits acceptDocs, DocsAndPositionsEnum reuse) throws IOException { final MultiDocsAndPositionsEnum docsAndPositionsEnum; if (reuse != null) { docsAndPositionsEnum = (MultiDocsAndPositionsEnum) reuse; @@ -400,11 +400,11 @@ docsAndPositionsEnum = new MultiDocsAndPositionsEnum(); } - final MultiBits multiSkipDocs; - if (skipDocs instanceof MultiBits) { - multiSkipDocs = (MultiBits) skipDocs; + final MultiBits multiAcceptDocs; + if (acceptDocs instanceof MultiBits) { + multiAcceptDocs = (MultiBits) acceptDocs; } else { - multiSkipDocs = null; + multiAcceptDocs = null; } int upto = 0; @@ -415,23 +415,23 @@ final Bits b; - if (multiSkipDocs != null) { + if (multiAcceptDocs != null) { // Optimize for common case: requested skip docs is a // congruent sub-slice of MultiBits: in this case, we - // just pull the skipDocs from the sub reader, rather + // just pull the acceptDocs from the sub reader, rather // than making the inefficient // Slice(Multi(sub-readers)): - final MultiBits.SubResult sub = multiSkipDocs.getMatchingSub(top[i].subSlice); + final MultiBits.SubResult sub = multiAcceptDocs.getMatchingSub(top[i].subSlice); if (sub.matches) { b = sub.result; } else { // custom case: requested skip docs is foreign: // must slice it on every access (very // inefficient) - b = new BitsSlice(skipDocs, top[i].subSlice); + b = new BitsSlice(acceptDocs, top[i].subSlice); } - } else if (skipDocs != null) { - b = new BitsSlice(skipDocs, top[i].subSlice); + } else if (acceptDocs != null) { + b = new BitsSlice(acceptDocs, top[i].subSlice); } else { // no deletions b = null; Index: lucene/src/java/org/apache/lucene/index/ParallelReader.java --- lucene/src/java/org/apache/lucene/index/ParallelReader.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/ParallelReader.java Sun Jun 26 14:21:07 2011 -0400 @@ -206,7 +206,7 @@ } } - @Override + @Override public Bits getDeletedDocs() { return MultiFields.getDeletedDocs(readers.get(0)); } Index: lucene/src/java/org/apache/lucene/index/SegmentMerger.java --- lucene/src/java/org/apache/lucene/index/SegmentMerger.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/SegmentMerger.java Sun Jun 26 14:21:07 2011 -0400 @@ -282,11 +282,12 @@ throws IOException, MergeAbortedException, CorruptIndexException { int docCount = 0; final int maxDoc = reader.maxDoc(); - final Bits delDocs = reader.getDeletedDocs(); + final Bits acceptDocs = reader.getNotDeletedDocs(); + assert acceptDocs != null; if (matchingFieldsReader != null) { // We can bulk-copy because the fieldInfos are "congruent" for (int j = 0; j < maxDoc;) { - if (delDocs.get(j)) { + if (!acceptDocs.get(j)) { // skip deleted docs ++j; continue; @@ -298,7 +299,7 @@ j++; numDocs++; if (j >= maxDoc) break; - if (delDocs.get(j)) { + if (!acceptDocs.get(j)) { j++; break; } @@ -311,7 +312,7 @@ } } else { for (int j = 0; j < maxDoc; j++) { - if (delDocs.get(j)) { + if (!acceptDocs.get(j)) { // skip deleted docs continue; } @@ -401,11 +402,11 @@ final IndexReader reader) throws IOException, MergeAbortedException { final int maxDoc = reader.maxDoc(); - final Bits delDocs = reader.getDeletedDocs(); + final Bits acceptDocs = reader.getNotDeletedDocs(); if (matchingVectorsReader != null) { // We can bulk-copy because the fieldInfos are "congruent" for (int docNum = 0; docNum < maxDoc;) { - if (delDocs.get(docNum)) { + if (!acceptDocs.get(docNum)) { // skip deleted docs ++docNum; continue; @@ -417,7 +418,7 @@ docNum++; numDocs++; if (docNum >= maxDoc) break; - if (delDocs.get(docNum)) { + if (!acceptDocs.get(docNum)) { docNum++; break; } @@ -429,7 +430,7 @@ } } else { for (int docNum = 0; docNum < maxDoc; docNum++) { - if (delDocs.get(docNum)) { + if (!acceptDocs.get(docNum)) { // skip deleted docs continue; } @@ -499,14 +500,14 @@ if (f != null) { slices.add(new ReaderUtil.Slice(docBase, maxDoc, fields.size())); fields.add(f); - bits.add(r.getDeletedDocs()); + bits.add(r.getNotDeletedDocs()); bitsStarts.add(docBase); } final PerDocValues producer = r.perDocValues(); if (producer != null) { perDocSlices.add(new ReaderUtil.Slice(docBase, maxDoc, fields.size())); perDocProducers.add(producer); - perDocBits.add(r.getDeletedDocs()); + perDocBits.add(r.getNotDeletedDocs()); perDocBitsStarts.add(docBase); } docBase += maxDoc; @@ -544,13 +545,13 @@ inputDocBase += reader.maxDoc(); if (mergeState.delCounts[i] != 0) { int delCount = 0; - final Bits delDocs = reader.getDeletedDocs(); - assert delDocs != null; + final Bits acceptDocs = reader.getNotDeletedDocs(); + assert acceptDocs != null; final int maxDoc = reader.maxDoc(); final int[] docMap = mergeState.docMaps[i] = new int[maxDoc]; int newDocID = 0; for(int j=0;j cache; - private final DeletesMode deletesMode; + final DeletesMode deletesMode; public FilterCache(DeletesMode deletesMode) { this.deletesMode = deletesMode; @@ -103,13 +105,13 @@ value = cache.get(delCoreKey); if (value == null) { - // now for core match, but dynamically AND NOT - // deletions + // now for core match, but dynamically AND + // accepted docs value = cache.get(coreKey); if (value != null) { - final Bits delDocs = reader.getDeletedDocs(); - if (delDocs != null) { - value = mergeDeletes(delDocs, value); + final Bits acceptDocs = reader.getNotDeletedDocs(); + if (acceptDocs != null) { + value = mergeAcceptDocs(acceptDocs, value); } } } @@ -118,7 +120,7 @@ return value; } - protected abstract T mergeDeletes(Bits delDocs, T value); + protected abstract T mergeAcceptDocs(Bits acceptDocs, T value); public synchronized void put(Object coreKey, Object delCoreKey, T value) { if (deletesMode == DeletesMode.IGNORE) { @@ -158,11 +160,11 @@ this.filter = filter; cache = new FilterCache(deletesMode) { @Override - public DocIdSet mergeDeletes(final Bits delDocs, final DocIdSet docIdSet) { + public DocIdSet mergeAcceptDocs(final Bits acceptDocs, final DocIdSet docIdSet) { return new FilteredDocIdSet(docIdSet) { @Override - protected boolean match(int docID) { - return !delDocs.get(docID); + protected boolean match(int docID) { + return acceptDocs.get(docID); } }; } @@ -197,7 +199,7 @@ public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException { final IndexReader reader = context.reader; final Object coreKey = reader.getCoreCacheKey(); - final Object delCoreKey = reader.hasDeletions() ? reader.getDeletedDocs() : coreKey; + final Object delCoreKey = reader.hasDeletions() ? ((NotBits) reader.getNotDeletedDocs()).getBits() : coreKey; DocIdSet docIdSet = cache.get(reader, coreKey, delCoreKey); if (docIdSet != null) { @@ -210,6 +212,20 @@ // cache miss docIdSet = docIdSetToCache(filter.getDocIdSet(context), reader); + if (docIdSet instanceof OpenBitSet) { + final OpenBitSet obs = (OpenBitSet) docIdSet; + final int setCount = (int) (obs.cardinality()); + + // nocommit how/where to make this hardwired 1% + // settable...? + + if (setCount > 0 && (reader.maxDoc()/setCount) < 100) { + obs.setUseRandomAccess(true); + obs.setIncludesDeletedDocs(cache.deletesMode == DeletesMode.RECACHE); + System.out.println("DORAND " + (cache.deletesMode == DeletesMode.RECACHE)); + } + } + if (docIdSet != null) { cache.put(coreKey, delCoreKey, docIdSet); } Index: lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java --- lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java Sun Jun 26 14:21:07 2011 -0400 @@ -136,16 +136,18 @@ if (filter != null) { assert query == null; final DocIdSet dis = filter.getDocIdSet(context); - if (dis == null) + if (dis == null) { return null; + } disi = dis.iterator(); } else { assert query != null && innerWeight != null; - disi = - innerWeight.scorer(context, scorerContext); + disi = innerWeight.scorer(context, scorerContext); } - if (disi == null) + + if (disi == null) { return null; + } return new ConstantScorer(disi, this); } Index: lucene/src/java/org/apache/lucene/search/DocIdSet.java --- lucene/src/java/org/apache/lucene/search/DocIdSet.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/DocIdSet.java Sun Jun 26 14:21:07 2011 -0400 @@ -19,6 +19,8 @@ import java.io.IOException; +import org.apache.lucene.util.Bits; + /** * A DocIdSet contains a set of doc ids. Implementing classes must * only implement {@link #iterator} to provide access to the set. @@ -46,6 +48,11 @@ public boolean isCacheable() { return true; } + + @Override + public Bits getRandomAccessBits() { + return null; + } }; /** Provides a {@link DocIdSetIterator} to access the set. @@ -54,6 +61,25 @@ * are no docs that match. */ public abstract DocIdSetIterator iterator() throws IOException; + /** Only used if {@link #getRandomAccessBits} returns non-null result; + * return true if the Bits instance has already factored + * in deletions. */ + // nocommit better name...? somewhere else...? move to + // subclass...? + public boolean bitsIncludesDeletedDocs() { + return false; + } + + /** Return a Bits impl if this DocIdSet should be applied + * via random-access (because the underlying bit set + * implementation supports random access, and the filter + * is dense enough), instead of {@link DocIdSetIterator}. */ + // nocommit better name...? somewhere else...? move to + // subclass...? + public Bits getRandomAccessBits() { + return null; + } + /** * This method is a hint for {@link CachingWrapperFilter}, if this DocIdSet * should be cached without copying it into a BitSet. The default is to return Index: lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java --- lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java Sun Jun 26 14:21:07 2011 -0400 @@ -533,9 +533,9 @@ @Override public DocIdSetIterator iterator() throws IOException { - final Bits skipDocs = canIgnoreDeletedDocs ? null : reader.getDeletedDocs(); + final Bits acceptDocs = canIgnoreDeletedDocs ? null : reader.getNotDeletedDocs(); - if (skipDocs == null) { + if (acceptDocs == null) { // Specialization optimization disregard deletions return new DocIdSetIterator() { private int doc = -1; @@ -575,7 +575,7 @@ final int maxDoc = reader.maxDoc(); // a DocIdSetIterator generating docIds by - // incrementing a variable & checking skipDocs - + // incrementing a variable & checking acceptDocs - return new DocIdSetIterator() { private int doc = -1; @Override @@ -590,14 +590,14 @@ if (doc >= maxDoc) { return doc = NO_MORE_DOCS; } - } while (skipDocs.get(doc) || !matchDoc(doc)); + } while (!acceptDocs.get(doc) || !matchDoc(doc)); return doc; } @Override public int advance(int target) { for(doc=target;doc filterDoc) { + filterDoc = filterIter.advance(scorerDoc); + } else { + scorerDoc = scorer.advance(filterDoc); } - collector.collect(scorerDoc); - filterDoc = filterIter.nextDoc(); - scorerDoc = scorer.advance(filterDoc); - } else if (scorerDoc > filterDoc) { - filterDoc = filterIter.advance(scorerDoc); - } else { - scorerDoc = scorer.advance(filterDoc); } } } Index: lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java --- lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java Sun Jun 26 14:21:07 2011 -0400 @@ -50,14 +50,14 @@ final byte[] norms; private int doc = -1; private final int maxDoc; - private final Bits delDocs; + private final Bits acceptDocs; private final Similarity similarity; MatchAllScorer(IndexReader reader, Similarity similarity, Weight w, byte[] norms) throws IOException { super(w); this.similarity = similarity; - delDocs = reader.getDeletedDocs(); + acceptDocs = reader.getNotDeletedDocs(); score = w.getValue(); maxDoc = reader.maxDoc(); this.norms = norms; @@ -71,7 +71,7 @@ @Override public int nextDoc() throws IOException { doc++; - while(delDocs != null && doc < maxDoc && delDocs.get(doc)) { + while(acceptDocs != null && doc < maxDoc && !acceptDocs.get(doc)) { doc++; } if (doc == maxDoc) { Index: lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java --- lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java Sun Jun 26 14:21:07 2011 -0400 @@ -175,7 +175,7 @@ if (termArrays.size() == 0) // optimize zero-term case return null; final IndexReader reader = context.reader; - final Bits delDocs = reader.getDeletedDocs(); + final Bits acceptDocs = reader.getNotDeletedDocs(); PhraseQuery.PostingsAndFreq[] postingsFreqs = new PhraseQuery.PostingsAndFreq[termArrays.size()]; @@ -196,12 +196,12 @@ } } else { final Term term = terms[0]; - postingsEnum = reader.termPositionsEnum(delDocs, + postingsEnum = reader.termPositionsEnum(acceptDocs, term.field(), term.bytes()); if (postingsEnum == null) { - if (reader.termDocsEnum(delDocs, term.field(), term.bytes()) != null) { + if (reader.termDocsEnum(acceptDocs, term.field(), term.bytes()) != null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + term.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + term.text() + ")"); } else { @@ -497,15 +497,15 @@ public UnionDocsAndPositionsEnum(IndexReader indexReader, Term[] terms) throws IOException { List docsEnums = new LinkedList(); - final Bits delDocs = indexReader.getDeletedDocs(); + final Bits acceptDocs = indexReader.getNotDeletedDocs(); for (int i = 0; i < terms.length; i++) { - DocsAndPositionsEnum postings = indexReader.termPositionsEnum(delDocs, + DocsAndPositionsEnum postings = indexReader.termPositionsEnum(acceptDocs, terms[i].field(), terms[i].bytes()); if (postings != null) { docsEnums.add(postings); } else { - if (indexReader.termDocsEnum(delDocs, terms[i].field(), terms[i].bytes()) != null) { + if (indexReader.termDocsEnum(acceptDocs, terms[i].field(), terms[i].bytes()) != null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + terms[i].field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + terms[i].text() + ")"); } Index: lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java --- lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Sun Jun 26 14:21:07 2011 -0400 @@ -125,13 +125,12 @@ // fill into a OpenBitSet final OpenBitSet bitSet = new OpenBitSet(context.reader.maxDoc()); int termCount = 0; - final Bits delDocs = reader.getDeletedDocs(); DocsEnum docsEnum = null; do { termCount++; // System.out.println(" iter termCount=" + termCount + " term=" + // enumerator.term().toBytesString()); - docsEnum = termsEnum.docs(delDocs, docsEnum); + docsEnum = termsEnum.docs(reader.getNotDeletedDocs(), docsEnum); final DocsEnum.BulkReadResult result = docsEnum.getBulkResult(); while (true) { final int count = docsEnum.read(); Index: lucene/src/java/org/apache/lucene/search/PhraseQuery.java --- lucene/src/java/org/apache/lucene/search/PhraseQuery.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/PhraseQuery.java Sun Jun 26 14:21:07 2011 -0400 @@ -213,16 +213,15 @@ return null; final IndexReader reader = context.reader; PostingsAndFreq[] postingsFreqs = new PostingsAndFreq[terms.size()]; - final Bits delDocs = reader.getDeletedDocs(); for (int i = 0; i < terms.size(); i++) { final Term t = terms.get(i); - DocsAndPositionsEnum postingsEnum = reader.termPositionsEnum(delDocs, + DocsAndPositionsEnum postingsEnum = reader.termPositionsEnum(scorerContext.acceptOnlyDocs, t.field(), t.bytes()); // PhraseQuery on a field that did not index // positions. if (postingsEnum == null) { - if (reader.termDocsEnum(delDocs, t.field(), t.bytes()) != null) { + if (reader.termDocsEnum(scorerContext.acceptOnlyDocs, t.field(), t.bytes()) != null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + t.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run PhraseQuery (term=" + t.text() + ")"); } else { Index: lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java --- lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/QueryWrapperFilter.java Sun Jun 26 14:21:07 2011 -0400 @@ -56,7 +56,8 @@ return new DocIdSet() { @Override public DocIdSetIterator iterator() throws IOException { - return weight.scorer(privateContext, ScorerContext.def()); + // nocommit + return weight.scorer(privateContext, ScorerContext.def().acceptOnlyDocs(context.reader.getNotDeletedDocs())); } @Override public boolean isCacheable() { return false; } Index: lucene/src/java/org/apache/lucene/search/TermQuery.java --- lucene/src/java/org/apache/lucene/search/TermQuery.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/TermQuery.java Sun Jun 26 14:21:07 2011 -0400 @@ -90,13 +90,12 @@ final String field = term.field(); final IndexReader reader = context.reader; assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context); - final TermState state = termStates - .get(context.ord); + final TermState state = termStates.get(context.ord); if (state == null) { // term is not present in that reader assert termNotInReader(reader, field, term.bytes()) : "no termstate found but term exists in reader"; return null; } - final DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), field, term.bytes(), state); + final DocsEnum docs = reader.termDocsEnum(scorerContext.acceptOnlyDocs, field, term.bytes(), state); assert docs != null; return new TermScorer(this, docs, similarity, context.reader.norms(field)); } @@ -143,7 +142,7 @@ Explanation tfExplanation = new Explanation(); int tf = 0; - DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), term.field(), term.bytes()); + DocsEnum docs = reader.termDocsEnum(context.reader.getNotDeletedDocs(), term.field(), term.bytes()); if (docs != null) { int newDoc = docs.advance(doc); if (newDoc == doc) { Index: lucene/src/java/org/apache/lucene/search/Weight.java --- lucene/src/java/org/apache/lucene/search/Weight.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/Weight.java Sun Jun 26 14:21:07 2011 -0400 @@ -19,9 +19,10 @@ import java.io.IOException; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexReader.ReaderContext; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.Bits; /** * Expert: Calculate query weights and build query scorers. @@ -140,9 +141,12 @@ * be called instead. */ public final boolean topScorer; + + /** If non-null, the returned scorer should filter + * according to this {@link Bits} instance. */ + public final Bits acceptOnlyDocs; - - private static final ScorerContext DEFAULT_CONTEXT = new ScorerContext(true, false); + private static final ScorerContext DEFAULT_CONTEXT = new ScorerContext(true, false, null); /** * Returns a default {@link ScorerContext} template initialized with: @@ -155,9 +159,10 @@ return DEFAULT_CONTEXT; } - private ScorerContext(boolean scoreDocsInOrder, boolean topScorer) { + private ScorerContext(boolean scoreDocsInOrder, boolean topScorer, Bits acceptOnlyDocs) { this.scoreDocsInOrder = scoreDocsInOrder; this.topScorer = topScorer; + this.acceptOnlyDocs = acceptOnlyDocs; } /** @@ -171,7 +176,7 @@ if (this.scoreDocsInOrder == scoreDocsInOrder) { return this; } - return new ScorerContext(scoreDocsInOrder, topScorer); + return new ScorerContext(scoreDocsInOrder, topScorer, acceptOnlyDocs); } /** @@ -185,7 +190,21 @@ if (this.topScorer == topScorer) { return this; } - return new ScorerContext(scoreDocsInOrder, topScorer); + return new ScorerContext(scoreDocsInOrder, topScorer, acceptOnlyDocs); + } + + /** + * Creates and returns a copy of this context with the given value for + * {@link #topScorer} and returns a new instance of + * {@link ScorerContext} iff the given value differs from the + * {@link #topScorer}. Otherwise, this method has no effect and + * returns this instance. + */ + public ScorerContext acceptOnlyDocs(Bits acceptOnlyDocs) { + if (this.acceptOnlyDocs == acceptOnlyDocs) { + return this; + } + return new ScorerContext(scoreDocsInOrder, topScorer, acceptOnlyDocs); } } } Index: lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java --- lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/cache/DocTermsCreator.java Sun Jun 26 14:21:07 2011 -0400 @@ -105,7 +105,7 @@ if (terms != null) { int termCount = 0; final TermsEnum termsEnum = terms.iterator(); - final Bits delDocs = MultiFields.getDeletedDocs(reader); + final Bits acceptDocs = MultiFields.getNotDeletedDocs(reader); DocsEnum docs = null; while(true) { if (termCount++ == termCountHardLimit) { @@ -120,7 +120,7 @@ break; } final long pointer = bytes.copyUsingLengthPrefix(term); - docs = termsEnum.docs(delDocs, docs); + docs = termsEnum.docs(acceptDocs, docs); while (true) { final int docID = docs.nextDoc(); if (docID == DocIdSetIterator.NO_MORE_DOCS) { Index: lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java --- lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java Sun Jun 26 14:21:07 2011 -0400 @@ -322,12 +322,12 @@ } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits acceptDocs, DocsEnum reuse) throws IOException { throw new UnsupportedOperationException(); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits acceptDocs, DocsAndPositionsEnum reuse) throws IOException { throw new UnsupportedOperationException(); } Index: lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java --- lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java Sun Jun 26 14:21:07 2011 -0400 @@ -126,7 +126,7 @@ private class ValueSourceScorer extends Scorer { private final float qWeight; private final DocValues vals; - private final Bits delDocs; + private final Bits acceptDocs; private final int maxDoc; private int doc = -1; @@ -137,14 +137,14 @@ qWeight = w.getValue(); // this is when/where the values are first created. vals = valSrc.getValues(context); - delDocs = reader.getDeletedDocs(); + acceptDocs = reader.getNotDeletedDocs(); maxDoc = reader.maxDoc(); } @Override public int nextDoc() throws IOException { doc++; - while (delDocs != null && doc < maxDoc && delDocs.get(doc)) { + while (acceptDocs != null && doc < maxDoc && !acceptDocs.get(doc)) { doc++; } if (doc == maxDoc) { Index: lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java --- lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Sun Jun 26 14:21:07 2011 -0400 @@ -83,14 +83,14 @@ @Override public Spans getSpans(final AtomicReaderContext context) throws IOException { final IndexReader reader = context.reader; - final DocsAndPositionsEnum postings = reader.termPositionsEnum(reader.getDeletedDocs(), + final DocsAndPositionsEnum postings = reader.termPositionsEnum(reader.getNotDeletedDocs(), term.field(), term.bytes()); if (postings != null) { return new TermSpans(postings, term); } else { - if (reader.termDocsEnum(reader.getDeletedDocs(), term.field(), term.bytes()) != null) { + if (reader.termDocsEnum(reader.getNotDeletedDocs(), term.field(), term.bytes()) != null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + term.field() + "\" was indexed with Field.omitTermFreqAndPositions=true; cannot run SpanTermQuery (term=" + term.text() + ")"); } else { Index: lucene/src/java/org/apache/lucene/util/AndBits.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ lucene/src/java/org/apache/lucene/util/AndBits.java Sun Jun 26 14:21:07 2011 -0400 @@ -0,0 +1,43 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @lucene.experimental + */ + +public final class AndBits implements Bits { + private final Bits bits1; + private final Bits bits2; + + public AndBits(Bits bits1, Bits bits2) { + this.bits1 = bits1; + this.bits2 = bits2; + if (bits1.length() != bits2.length()) { + throw new IllegalArgumentException("bit lengths differ: " + bits1.length() + " vs " + bits2.length()); + } + } + + public boolean get(int index) { + return bits1.get(index) && bits2.get(index); + } + + public int length() { + return bits1.length(); + } +} Index: lucene/src/java/org/apache/lucene/util/MultiBits.java --- lucene/src/java/org/apache/lucene/util/MultiBits.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/util/MultiBits.java Sun Jun 26 14:21:07 2011 -0400 @@ -34,13 +34,16 @@ // length is 1+subs.length (the last entry has the maxDoc): private final int[] starts; - public MultiBits(List bits, List starts) { + private final boolean defaultValue; + + public MultiBits(List bits, List starts, boolean defaultValue) { assert starts.size() == 1+bits.size(); this.subs = bits.toArray(Bits.EMPTY_ARRAY); this.starts = new int[starts.size()]; for(int i=0;i>32) ^ h) + 0x98761234; } + private boolean includesDeletions; + + /** @lucene.experimental */ + public void setIncludesDeletedDocs(boolean v) { + includesDeletions = v; + } + + /** @lucene.experimental */ + @Override + public boolean bitsIncludesDeletedDocs() { + return includesDeletions; + } + + private boolean useRandomAccess; + + /** @lucene.experimental */ + public void setUseRandomAccess(boolean v) { + useRandomAccess = v; + } + + @Override + public Bits getRandomAccessBits() { + return useRandomAccess ? this : null; + } } Index: lucene/src/java/org/apache/lucene/util/OpenBitSetDISI.java --- lucene/src/java/org/apache/lucene/util/OpenBitSetDISI.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/java/org/apache/lucene/util/OpenBitSetDISI.java Sun Jun 26 14:21:07 2011 -0400 @@ -19,7 +19,10 @@ import java.io.IOException; import org.apache.lucene.search.DocIdSetIterator; - + +// nocommit -- why extend OBS here? why not makea new OBS +// then use util methods to set the bits... + public class OpenBitSetDISI extends OpenBitSet { /** Construct an OpenBitSetDISI with its bits set Index: lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java --- lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Sun Jun 26 14:21:07 2011 -0400 @@ -1249,6 +1249,8 @@ * with one that returns null for getSequentialSubReaders. */ public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) throws IOException { + // nocommit fix IS wrapper to enable "low" filtering for + // search methods taking Filter if (random.nextBoolean()) { if (maybeWrap && rarely()) { r = new SlowMultiReaderWrapper(r); Index: lucene/src/test/org/apache/lucene/TestExternalCodecs.java --- lucene/src/test/org/apache/lucene/TestExternalCodecs.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/test/org/apache/lucene/TestExternalCodecs.java Sun Jun 26 14:21:07 2011 -0400 @@ -346,26 +346,26 @@ } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) { - return new RAMDocsEnum(ramField.termToDocs.get(current), skipDocs); + public DocsEnum docs(Bits keepDocs, DocsEnum reuse) { + return new RAMDocsEnum(ramField.termToDocs.get(current), keepDocs); } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse) { - return new RAMDocsAndPositionsEnum(ramField.termToDocs.get(current), skipDocs); + public DocsAndPositionsEnum docsAndPositions(Bits keepDocs, DocsAndPositionsEnum reuse) { + return new RAMDocsAndPositionsEnum(ramField.termToDocs.get(current), keepDocs); } } private static class RAMDocsEnum extends DocsEnum { private final RAMTerm ramTerm; - private final Bits skipDocs; + private final Bits keepDocs; private RAMDoc current; int upto = -1; int posUpto = 0; - public RAMDocsEnum(RAMTerm ramTerm, Bits skipDocs) { + public RAMDocsEnum(RAMTerm ramTerm, Bits keepDocs) { this.ramTerm = ramTerm; - this.skipDocs = skipDocs; + this.keepDocs = keepDocs; } @Override @@ -383,7 +383,7 @@ upto++; if (upto < ramTerm.docs.size()) { current = ramTerm.docs.get(upto); - if (skipDocs == null || !skipDocs.get(current.docID)) { + if (keepDocs == null || keepDocs.get(current.docID)) { posUpto = 0; return current.docID; } @@ -406,14 +406,14 @@ private static class RAMDocsAndPositionsEnum extends DocsAndPositionsEnum { private final RAMTerm ramTerm; - private final Bits skipDocs; + private final Bits keepDocs; private RAMDoc current; int upto = -1; int posUpto = 0; - public RAMDocsAndPositionsEnum(RAMTerm ramTerm, Bits skipDocs) { + public RAMDocsAndPositionsEnum(RAMTerm ramTerm, Bits keepDocs) { this.ramTerm = ramTerm; - this.skipDocs = skipDocs; + this.keepDocs = keepDocs; } @Override @@ -431,7 +431,7 @@ upto++; if (upto < ramTerm.docs.size()) { current = ramTerm.docs.get(upto); - if (skipDocs == null || !skipDocs.get(current.docID)) { + if (keepDocs == null || keepDocs.get(current.docID)) { posUpto = 0; return current.docID; } @@ -560,6 +560,9 @@ r.close(); s.close(); + if (VERBOSE) { + System.out.println("\nTEST: now delete 2nd doc"); + } w.deleteDocuments(new Term("id", "44")); w.optimize(); r = IndexReader.open(w, true); Index: lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java --- lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java Sun Jun 26 14:21:07 2011 -0400 @@ -73,7 +73,7 @@ IndexReader reader = writer.getReader(); DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getNotDeletedDocs(reader), "preanalyzed", new BytesRef("term1")); assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS); @@ -81,7 +81,7 @@ assertEquals(0, termPositions.nextPosition()); termPositions = MultiFields.getTermPositionsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getNotDeletedDocs(reader), "preanalyzed", new BytesRef("term2")); assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS); @@ -90,7 +90,7 @@ assertEquals(3, termPositions.nextPosition()); termPositions = MultiFields.getTermPositionsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getNotDeletedDocs(reader), "preanalyzed", new BytesRef("term3")); assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS); Index: lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java --- lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Sun Jun 26 14:21:07 2011 -0400 @@ -281,10 +281,10 @@ _TestUtil.checkIndex(dir); - final Bits delDocs = MultiFields.getDeletedDocs(reader); + final Bits notDelDocs = MultiFields.getNotDeletedDocs(reader); for(int i=0;i<35;i++) { - if (!delDocs.get(i)) { + if (notDelDocs.get(i)) { Document d = reader.document(i); List fields = d.getFields(); if (d.getField("content3") == null) { Index: lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java --- lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java Sun Jun 26 14:21:07 2011 -0400 @@ -611,6 +611,9 @@ final int N = 10; for(int pass=0;pass<2;pass++) { + if (VERBOSE) { + System.out.println("TEST: pass=" + pass); + } boolean useCompoundFile = (pass % 2) != 0; @@ -631,7 +634,7 @@ for(int i=0;i fieldable1 = doc1.getFields(); @@ -880,15 +880,15 @@ FieldsEnum fenum1 = MultiFields.getFields(index1).iterator(); FieldsEnum fenum2 = MultiFields.getFields(index1).iterator(); String field1 = null; - Bits delDocs = MultiFields.getDeletedDocs(index1); + Bits notDelDocs = MultiFields.getNotDeletedDocs(index1); while((field1=fenum1.next()) != null) { assertEquals("Different fields", field1, fenum2.next()); TermsEnum enum1 = fenum1.terms(); TermsEnum enum2 = fenum2.terms(); while(enum1.next() != null) { assertEquals("Different terms", enum1.term(), enum2.next()); - DocsAndPositionsEnum tp1 = enum1.docsAndPositions(delDocs, null); - DocsAndPositionsEnum tp2 = enum2.docsAndPositions(delDocs, null); + DocsAndPositionsEnum tp1 = enum1.docsAndPositions(notDelDocs, null); + DocsAndPositionsEnum tp2 = enum2.docsAndPositions(notDelDocs, null); while(tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java --- lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java Sun Jun 26 14:21:07 2011 -0400 @@ -282,9 +282,9 @@ assertTrue(sim.decodeNormValue(MultiNorms.norms(r1, "field1")[4]) == norm1); assertTrue(sim.decodeNormValue(MultiNorms.norms(pr1Clone, "field1")[4]) != norm1); - final Bits delDocs = MultiFields.getDeletedDocs(r1); - assertTrue(delDocs == null || !delDocs.get(10)); - assertTrue(MultiFields.getDeletedDocs(pr1Clone).get(10)); + final Bits notDelDocs = MultiFields.getNotDeletedDocs(r1); + assertTrue(notDelDocs == null || notDelDocs.get(10)); + assertFalse(MultiFields.getNotDeletedDocs(pr1Clone).get(10)); // try to update the original reader, which should throw an exception try { @@ -374,10 +374,10 @@ assertTrue(origSegmentReader.deletedDocs != clonedSegmentReader.deletedDocs); assertDocDeleted(origSegmentReader, clonedSegmentReader, 1); - final Bits delDocs = origSegmentReader.getDeletedDocs(); - assertTrue(delDocs == null || !delDocs.get(2)); // doc 2 should not be deleted + final Bits notDelDocs = origSegmentReader.getNotDeletedDocs(); + assertTrue(notDelDocs == null || notDelDocs.get(2)); // doc 2 should not be deleted // in original segmentreader - assertTrue(clonedSegmentReader.getDeletedDocs().get(2)); // doc 2 should be deleted in + assertFalse(clonedSegmentReader.getNotDeletedDocs().get(2)); // doc 2 should be deleted in // cloned segmentreader // deleting a doc from the original segmentreader should throw an exception @@ -419,7 +419,7 @@ clonedReader.close(); IndexReader r = IndexReader.open(dir1, false); - assertTrue(MultiFields.getDeletedDocs(r).get(1)); + assertFalse(MultiFields.getNotDeletedDocs(r).get(1)); r.close(); dir1.close(); } @@ -448,7 +448,7 @@ private void assertDocDeleted(SegmentReader reader, SegmentReader reader2, int doc) { - assertEquals(reader.getDeletedDocs().get(doc), reader2.getDeletedDocs().get(doc)); + assertEquals(reader.getNotDeletedDocs().get(doc), reader2.getNotDeletedDocs().get(doc)); } private void assertDelDocsRefCountEquals(int refCount, SegmentReader reader) { Index: lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java --- lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderDelete.java Sun Jun 26 14:21:07 2011 -0400 @@ -285,17 +285,17 @@ IndexReader r = new SlowMultiReaderWrapper(w.getReader()); w.close(); - assertNull(r.getDeletedDocs()); + assertNull(r.getNotDeletedDocs()); r.close(); r = new SlowMultiReaderWrapper(IndexReader.open(dir, false)); - assertNull(r.getDeletedDocs()); + assertNull(r.getNotDeletedDocs()); assertEquals(1, r.deleteDocuments(new Term("f", "doctor"))); - assertNotNull(r.getDeletedDocs()); - assertTrue(r.getDeletedDocs().get(0)); + assertNotNull(r.getNotDeletedDocs()); + assertFalse(r.getNotDeletedDocs().get(0)); assertEquals(1, r.deleteDocuments(new Term("f", "who"))); - assertTrue(r.getDeletedDocs().get(1)); + assertFalse(r.getNotDeletedDocs().get(1)); r.close(); dir.close(); } Index: lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java --- lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java Sun Jun 26 14:21:07 2011 -0400 @@ -1120,8 +1120,8 @@ r2.deleteDocument(0); // r1 should not see the delete - final Bits r1DelDocs = MultiFields.getDeletedDocs(r1); - assertFalse(r1DelDocs != null && r1DelDocs.get(0)); + final Bits r1AcceptDocs = MultiFields.getNotDeletedDocs(r1); + assertFalse(r1AcceptDocs != null && !r1AcceptDocs.get(0)); // Now r2 should have made a private copy of deleted docs: assertTrue(sr1.deletedDocs!=sr2.deletedDocs); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Sun Jun 26 14:21:07 2011 -0400 @@ -536,7 +536,7 @@ Term t = new Term("field", "a"); assertEquals(1, reader.docFreq(t)); DocsEnum td = MultiFields.getTermDocsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getNotDeletedDocs(reader), "field", new BytesRef("a")); td.nextDoc(); @@ -947,7 +947,7 @@ assertEquals(1, hits.length); DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(s.getIndexReader(), - MultiFields.getDeletedDocs(s.getIndexReader()), + MultiFields.getNotDeletedDocs(s.getIndexReader()), "field", new BytesRef("a")); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java --- lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Sun Jun 26 11:29:28 2011 -0400 +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Sun Jun 26 14:21:07 2011 -0400 @@ -484,7 +484,7 @@ // Make sure the doc that hit the exception was marked // as deleted: DocsEnum tdocs = MultiFields.getTermDocsEnum(reader, - MultiFields.getDeletedDocs(reader), + MultiFields.getNotDeletedDocs(reader), t.field(), new BytesRef(t.text())); @@ -624,10 +624,10 @@ assertEquals(expected, reader.docFreq(new Term("contents", "here"))); assertEquals(expected, reader.maxDoc()); int numDel = 0; - final Bits delDocs = MultiFields.getDeletedDocs(reader); - assertNotNull(delDocs); + final Bits acceptDocs = MultiFields.getNotDeletedDocs(reader); + assertNotNull(acceptDocs); for(int j=0;j=maxDoc) { return doc=NO_MORE_DOCS; } - if (delDocs != null && delDocs.get(doc)) continue; + if (acceptDocs != null && !acceptDocs.get(doc)) continue; if (!match()) continue; return doc; } Index: solr/src/java/org/apache/solr/search/JoinQParserPlugin.java --- solr/src/java/org/apache/solr/search/JoinQParserPlugin.java Sun Jun 26 11:29:28 2011 -0400 +++ solr/src/java/org/apache/solr/search/JoinQParserPlugin.java Sun Jun 26 14:21:07 2011 -0400 @@ -285,19 +285,19 @@ } } - Bits fromDeletedDocs = MultiFields.getDeletedDocs(fromSearcher.getIndexReader()); - Bits toDeletedDocs = fromSearcher == toSearcher ? fromDeletedDocs : MultiFields.getDeletedDocs(toSearcher.getIndexReader()); + Bits fromAcceptDocs = MultiFields.getNotDeletedDocs(fromSearcher.getIndexReader()); + Bits toAcceptDocs = fromSearcher == toSearcher ? fromAcceptDocs : MultiFields.getNotDeletedDocs(toSearcher.getIndexReader()); fromDeState = new SolrIndexSearcher.DocsEnumState(); fromDeState.fieldName = fromField; - fromDeState.deletedDocs = fromDeletedDocs; + fromDeState.acceptedDocs = fromAcceptDocs; fromDeState.termsEnum = termsEnum; fromDeState.docsEnum = null; fromDeState.minSetSizeCached = minDocFreqFrom; toDeState = new SolrIndexSearcher.DocsEnumState(); toDeState.fieldName = toField; - toDeState.deletedDocs = toDeletedDocs; + toDeState.acceptedDocs = toAcceptDocs; toDeState.termsEnum = toTermsEnum; toDeState.docsEnum = null; toDeState.minSetSizeCached = minDocFreqTo; @@ -314,7 +314,7 @@ if (freq < minDocFreqFrom) { fromTermDirectCount++; - // OK to skip deletedDocs, since we check for intersection with docs matching query + // OK to skip acceptedDocs, since we check for intersection with docs matching query fromDeState.docsEnum = fromDeState.termsEnum.docs(null, fromDeState.docsEnum); DocsEnum docsEnum = fromDeState.docsEnum; @@ -393,8 +393,8 @@ } else { toTermDirectCount++; - // need to use deletedDocs here so we don't map to any deleted ones - toDeState.docsEnum = toDeState.termsEnum.docs(toDeState.deletedDocs, toDeState.docsEnum); + // need to use acceptedDocs here so we don't map to any deleted ones + toDeState.docsEnum = toDeState.termsEnum.docs(toDeState.acceptedDocs, toDeState.docsEnum); DocsEnum docsEnum = toDeState.docsEnum; if (docsEnum instanceof MultiDocsEnum) { Index: solr/src/java/org/apache/solr/search/SolrIndexSearcher.java --- solr/src/java/org/apache/solr/search/SolrIndexSearcher.java Sun Jun 26 11:29:28 2011 -0400 +++ solr/src/java/org/apache/solr/search/SolrIndexSearcher.java Sun Jun 26 14:21:07 2011 -0400 @@ -493,7 +493,7 @@ Terms terms = fields.terms(t.field()); if (terms == null) return -1; BytesRef termBytes = t.bytes(); - DocsEnum docs = terms.docs(MultiFields.getDeletedDocs(reader), termBytes, null); + DocsEnum docs = terms.docs(MultiFields.getNotDeletedDocs(reader), termBytes, null); if (docs == null) return -1; int id = docs.nextDoc(); return id == DocIdSetIterator.NO_MORE_DOCS ? -1 : id; @@ -730,7 +730,7 @@ int bitsSet = 0; OpenBitSet obs = null; - DocsEnum docsEnum = deState.termsEnum.docs(deState.deletedDocs, deState.docsEnum); + DocsEnum docsEnum = deState.termsEnum.docs(deState.acceptedDocs, deState.docsEnum); if (deState.docsEnum == null) { deState.docsEnum = docsEnum; } @@ -819,8 +819,8 @@ Terms terms = fields.terms(t.field()); BytesRef termBytes = t.bytes(); - Bits skipDocs = reader.getDeletedDocs(); - DocsEnum docsEnum = terms==null ? null : terms.docs(skipDocs, termBytes, null); + Bits acceptDocs = reader.getNotDeletedDocs(); + DocsEnum docsEnum = terms==null ? null : terms.docs(acceptDocs, termBytes, null); if (docsEnum != null) { DocsEnum.BulkReadResult readResult = docsEnum.getBulkResult(); @@ -1617,7 +1617,7 @@ public static class DocsEnumState { public String fieldName; // currently interned for as long as lucene requires it public TermsEnum termsEnum; - public Bits deletedDocs; + public Bits acceptedDocs; public DocsEnum docsEnum; public int minSetSizeCached; Index: solr/src/java/org/apache/solr/search/function/FileFloatSource.java --- solr/src/java/org/apache/solr/search/function/FileFloatSource.java Sun Jun 26 11:29:28 2011 -0400 +++ solr/src/java/org/apache/solr/search/function/FileFloatSource.java Sun Jun 26 14:21:07 2011 -0400 @@ -243,7 +243,7 @@ DocsEnum docsEnum = null; // removing deleted docs shouldn't matter - // final Bits delDocs = MultiFields.getDeletedDocs(reader); + // final Bits acceptDocs = MultiFields.getNotDeletedDocs(reader); for (String line; (line=r.readLine())!=null;) { int delimIndex = line.indexOf(delimiter); Index: solr/src/java/org/apache/solr/search/function/FunctionQuery.java --- solr/src/java/org/apache/solr/search/function/FunctionQuery.java Sun Jun 26 11:29:28 2011 -0400 +++ solr/src/java/org/apache/solr/search/function/FunctionQuery.java Sun Jun 26 14:21:07 2011 -0400 @@ -112,7 +112,7 @@ int doc=-1; final DocValues vals; final boolean hasDeletions; - final Bits delDocs; + final Bits acceptDocs; public AllScorer(AtomicReaderContext context, FunctionWeight w) throws IOException { super(w); @@ -121,8 +121,8 @@ this.reader = context.reader; this.maxDoc = reader.maxDoc(); this.hasDeletions = reader.hasDeletions(); - this.delDocs = MultiFields.getDeletedDocs(reader); - assert !hasDeletions || delDocs != null; + this.acceptDocs = MultiFields.getNotDeletedDocs(reader); + assert !hasDeletions || acceptDocs != null; vals = func.getValues(weight.context, context); } @@ -142,7 +142,7 @@ if (doc>=maxDoc) { return doc=NO_MORE_DOCS; } - if (hasDeletions && delDocs.get(doc)) continue; + if (hasDeletions && !acceptDocs.get(doc)) continue; return doc; } } Index: solr/src/java/org/apache/solr/search/function/ValueSource.java --- solr/src/java/org/apache/solr/search/function/ValueSource.java Sun Jun 26 11:29:28 2011 -0400 +++ solr/src/java/org/apache/solr/search/function/ValueSource.java Sun Jun 26 14:21:07 2011 -0400 @@ -199,7 +199,7 @@ protected final int maxDoc; protected final DocValues values; protected boolean checkDeletes; - private final Bits delDocs; + private final Bits acceptDocs; protected ValueSourceScorer(IndexReader reader, DocValues values) { super(null); @@ -207,7 +207,7 @@ this.maxDoc = reader.maxDoc(); this.values = values; setCheckDeletes(true); - this.delDocs = MultiFields.getDeletedDocs(reader); + this.acceptDocs = MultiFields.getNotDeletedDocs(reader); } public IndexReader getReader() { @@ -219,7 +219,7 @@ } public boolean matches(int doc) { - return (!checkDeletes || !delDocs.get(doc)) && matchesValue(doc); + return (!checkDeletes || acceptDocs.get(doc)) && matchesValue(doc); } public boolean matchesValue(int doc) {