Index: src/test/org/apache/lucene/util/TestBitVector.java =================================================================== --- src/test/org/apache/lucene/util/TestBitVector.java (revision 736176) +++ src/test/org/apache/lucene/util/TestBitVector.java (working copy) @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; @@ -201,7 +202,52 @@ bv.write(d, "TESTBV"); } } + /** + * Test nextSetBit() with various bit-twiddling edge cases. + * @throws Exception + */ + public void testNextSetBit() throws Exception { + for (int i = 24; i <= 33; i++) { + BitVector bitVec = new BitVector(64); + bitVec.set(i); + assertEquals("nextSetBit for 0 is " + i, bitVec.nextSetBit(0), i); + assertEquals("nextSetBit for 1 is " + i, bitVec.nextSetBit(1), i); + for (int probe = 15; probe <= i; probe++) { + assertEquals("nextSetBit for " + probe + " is " + i, + bitVec.nextSetBit(probe), i); + } + for (int probe = i + 1; probe <= i + 9; probe++) { + assertEquals("no nextSetBit for " + probe + " when max is " + i, + bitVec.nextSetBit(probe), -1); + } + } + } + + public void testDocIdSet() throws IOException { + int[] ints = new int[] {0,1,2,3,5,8}; + BitVector bv = new BitVector(9); + for (int x=0; x < ints.length; x++) { + bv.set(ints[x]); + } + tstDocIdSet(ints, bv); + } + + private void tstDocIdSet(int[] ints, BitVector bitVector) throws IOException { + for (int i = 0; i < ints.length; i++) { + if ((i > 0) && (ints[i-1] == ints[i])) { + return; // DocNrSkipper should not skip to same document. + } + } + DocIdSetIterator m = bitVector.iterator(); + for (int i = 0; i < ints.length; i++) { + assertTrue("No end of Matcher at: " + i, m.next()); + assertEquals(ints[i], m.doc()); + } + assertTrue("End of Matcher", (! m.next())); + } + + /** * Compare two BitVectors. * This should really be an equals method on the BitVector itself. * @param bv One bit vector Index: src/java/org/apache/lucene/index/ParallelReader.java =================================================================== --- src/java/org/apache/lucene/index/ParallelReader.java (revision 736176) +++ src/java/org/apache/lucene/index/ParallelReader.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.document.FieldSelectorResult; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.search.DocIdSet; import java.io.IOException; import java.util.*; @@ -231,6 +232,12 @@ return ((IndexReader)readers.get(0)).isDeleted(n); return false; } + + DocIdSet getDeletedDocs() throws IOException { + if (readers.size() > 0) + return ((IndexReader)readers.get(0)).getDeletedDocs(); + return null; + } // delete in all readers protected void doDelete(int n) throws CorruptIndexException, IOException { Index: src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/SegmentReader.java (revision 736176) +++ src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -31,6 +31,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.DocIdSet; import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -719,7 +720,11 @@ ("attempt to access a deleted document"); return getFieldsReader().doc(n, fieldSelector); } - + + DocIdSet getDeletedDocs() throws IOException { + return deletedDocs; + } + public synchronized boolean isDeleted(int n) { return (deletedDocs != null && deletedDocs.get(n)); } Index: src/java/org/apache/lucene/index/SegmentTermDocs.java =================================================================== --- src/java/org/apache/lucene/index/SegmentTermDocs.java (revision 736176) +++ src/java/org/apache/lucene/index/SegmentTermDocs.java (working copy) @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.lucene.util.BitVector; import org.apache.lucene.store.IndexInput; +import org.apache.lucene.search.DocIdSetIterator; class SegmentTermDocs implements TermDocs { protected SegmentReader parent; @@ -27,6 +28,7 @@ protected int count; protected int df; protected BitVector deletedDocs; + DocIdSetIterator deletedDocsIt; int doc = 0; int freq; @@ -42,13 +44,22 @@ protected boolean currentFieldStoresPayloads; protected boolean currentFieldOmitTf; + private int nextDeletion; protected SegmentTermDocs(SegmentReader parent) { this.parent = parent; this.freqStream = (IndexInput) parent.freqStream.clone(); - this.deletedDocs = parent.deletedDocs; + try { + if (parent.getDeletedDocs() != null) { + this.deletedDocsIt = parent.getDeletedDocs().iterator(); + this.deletedDocs = (BitVector)parent.getDeletedDocs(); + } + } catch (IOException ex) { + throw new RuntimeException("", ex); + } this.skipInterval = parent.tis.getSkipInterval(); this.maxSkipLevels = parent.tis.getMaxSkipLevels(); + this.nextDeletion = this.deletedDocsIt == null ? Integer.MAX_VALUE : -1; } public void seek(Term term) throws IOException { @@ -75,6 +86,7 @@ void seek(TermInfo ti, Term term) throws IOException { count = 0; + nextDeletion = this.deletedDocsIt == null ? Integer.MAX_VALUE : -1; FieldInfo fi = parent.fieldInfos.fieldInfo(term.field); currentFieldOmitTf = (fi != null) ? fi.omitTf : false; currentFieldStoresPayloads = (fi != null) ? fi.storePayloads : false; @@ -121,10 +133,17 @@ } count++; - - if (deletedDocs == null || !deletedDocs.get(doc)) - break; - skippingDoc(); + if (deletedDocsIt != null) { + if (doc > nextDeletion) { + if (deletedDocsIt.skipTo(doc)) + nextDeletion = deletedDocsIt.doc(); + } + if (doc == nextDeletion) { + skippingDoc(); + continue; + } + } + break; } return true; } @@ -146,36 +165,44 @@ else freq = freqStream.readVInt(); // else read freq count++; - - if (deletedDocs == null || !deletedDocs.get(doc)) { - docs[i] = doc; - freqs[i] = freq; - ++i; + if (deletedDocsIt != null) { + if (doc > nextDeletion) { + if (deletedDocsIt.skipTo(doc)) + nextDeletion = deletedDocsIt.doc(); + } + if (doc == nextDeletion) + continue; } + docs[i] = doc; + freqs[i] = freq; + ++i; } return i; } } - + private final int readNoTf(final int[] docs, final int[] freqs, final int length) throws IOException { int i = 0; while (i < length && count < df) { // manually inlined call to next() for speed doc += freqStream.readVInt(); count++; - - if (deletedDocs == null || !deletedDocs.get(doc)) { - docs[i] = doc; - // Hardware freq to 1 when term freqs were not - // stored in the index - freqs[i] = 1; - ++i; + if (deletedDocsIt != null) { + if (doc > nextDeletion) { + if (deletedDocsIt.skipTo(doc)) + nextDeletion = deletedDocsIt.doc(); + } + if (doc == nextDeletion) + continue; } + docs[i] = doc; + freqs[i] = 1; + ++i; } return i; } - + /** Overridden by SegmentTermPositions to skip in prox stream. */ protected void skipProx(long proxPointer, int payloadLength) throws IOException {} Index: src/java/org/apache/lucene/index/MultiReader.java =================================================================== --- src/java/org/apache/lucene/index/MultiReader.java (revision 736176) +++ src/java/org/apache/lucene/index/MultiReader.java (working copy) @@ -27,6 +27,8 @@ import org.apache.lucene.index.MultiSegmentReader.MultiTermDocs; import org.apache.lucene.index.MultiSegmentReader.MultiTermEnum; import org.apache.lucene.index.MultiSegmentReader.MultiTermPositions; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.MultiDocIdSet; /** An IndexReader which reads multiple indexes, appending their content. * @@ -338,6 +340,11 @@ } } + DocIdSet getDeletedDocs() throws IOException { + DocIdSet[] sets = MultiSegmentReader.getDeletedDocsDocIdSets(subReaders); + return new MultiDocIdSet(starts, sets); + } + public Collection getFieldNames (IndexReader.FieldOption fieldNames) { ensureOpen(); return MultiSegmentReader.getFieldNames(fieldNames, this.subReaders); Index: src/java/org/apache/lucene/index/FilterIndexReader.java =================================================================== --- src/java/org/apache/lucene/index/FilterIndexReader.java (revision 736176) +++ src/java/org/apache/lucene/index/FilterIndexReader.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.search.DocIdSet; import org.apache.lucene.store.Directory; import java.io.IOException; @@ -146,7 +147,11 @@ ensureOpen(); return in.document(n, fieldSelector); } - + + DocIdSet getDeletedDocs() throws IOException { + return in.getDeletedDocs(); + } + public boolean isDeleted(int n) { // Don't call ensureOpen() here (it could affect performance) return in.isDeleted(n); Index: src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- src/java/org/apache/lucene/index/IndexReader.java (revision 736176) +++ src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.DocIdSet; import org.apache.lucene.store.*; import java.io.File; @@ -1041,6 +1042,8 @@ throw new UnsupportedOperationException("This reader does not support this method."); } + abstract DocIdSet getDeletedDocs() throws IOException; + /** * Prints the filename and size of each file within a given compound file. * Add the -extract flag to extract files to the current working directory. Index: src/java/org/apache/lucene/index/MultiSegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/MultiSegmentReader.java (revision 736176) +++ src/java/org/apache/lucene/index/MultiSegmentReader.java (working copy) @@ -26,6 +26,8 @@ import java.util.Map; import java.util.Set; +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.MultiDocIdSet; import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.store.Directory; @@ -194,7 +196,20 @@ } starts[subReaders.length] = maxDoc; } - + + static DocIdSet[] getDeletedDocsDocIdSets(IndexReader[] readers) throws IOException { + DocIdSet[] sets = new DocIdSet[readers.length]; + for (int x=0; x < readers.length; x++) { + sets[x] = readers[x].getDeletedDocs(); + } + return sets; + } + + DocIdSet getDeletedDocs() throws IOException { + DocIdSet[] sets = getDeletedDocsDocIdSets(subReaders); + return new MultiDocIdSet(starts, sets); + } + protected synchronized DirectoryIndexReader doReopen(SegmentInfos infos) throws CorruptIndexException, IOException { if (infos.size() == 1) { // The index has only one segment now, so we can't refresh the MultiSegmentReader. Index: src/java/org/apache/lucene/util/BitVector.java =================================================================== --- src/java/org/apache/lucene/util/BitVector.java (revision 736176) +++ src/java/org/apache/lucene/util/BitVector.java (working copy) @@ -18,7 +18,10 @@ */ import java.io.IOException; +import java.util.BitSet; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.DocIdSet; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -35,7 +38,7 @@ @version $Id$ */ -public final class BitVector { +public final class BitVector extends DocIdSet { private byte[] bits; private int size; @@ -46,7 +49,68 @@ size = n; bits = new byte[(size >> 3) + 1]; } + + /** Returns the index of the first bit that is set to true that occurs on or + * after the specified starting index. + */ + public final int nextSetBit(int bit) { + final int max = bits.length; + for (int pos = bit >>> 3; pos < max; pos++) { + if (bits[pos] != 0) { + // There's a non-zero bit in this byte. + final int base = pos * 8; + for (int i = 0; i < 8; i++) { + final int candidate = base + i; + if (candidate < size && candidate >= bit) { + if (get(candidate)) { + return candidate; + } + } + } + } + } + return -1; + } + + public DocIdSetIterator iterator() { + return new BitVectorDocIdSetIterator(this); + } + + private static class BitVectorDocIdSetIterator extends DocIdSetIterator { + private int docId; + private BitVector bitSet; + + BitVectorDocIdSetIterator(BitVector bitSet) { + this.bitSet = bitSet; + this.docId = -1; + } + + public int doc() { + assert docId != -1; + return docId; + } + + public boolean next() { + // (docId + 1) on next line requires -1 initial value for docNr: + return checkNextDocId(bitSet.nextSetBit(docId + 1)); + } + + public boolean skipTo(int skipDocNr) { + return checkNextDocId( bitSet.nextSetBit(skipDocNr)); + } + + private boolean checkNextDocId(int d) { + if (d == -1) { // -1 returned by BitSet.nextSetBit() when exhausted + docId = Integer.MAX_VALUE; + return false; + } else { + docId = d; + return true; + } + } + } + /** Sets the value of bit to one. */ public final void set(int bit) { if (bit >= size) {