Index: lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java --- lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Wed Jul 06 10:39:55 2011 -0400 @@ -49,7 +49,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; public class HighlighterPhraseTest extends LuceneTestCase { private static final String FIELD = "text"; @@ -119,7 +119,7 @@ final Query phraseQuery = new SpanNearQuery(new SpanQuery[] { new SpanTermQuery(new Term(FIELD, "fox")), new SpanTermQuery(new Term(FIELD, "jumped")) }, 0, true); - final OpenBitSet bitset = new OpenBitSet(); + final FastBitSet bitset = new FastBitSet(indexReader.maxDoc()); indexSearcher.search(phraseQuery, new Collector() { private int baseDoc; @@ -146,10 +146,11 @@ } }); assertEquals(1, bitset.cardinality()); + final int maxDoc = indexReader.maxDoc(); final Highlighter highlighter = new Highlighter( new SimpleHTMLFormatter(), new SimpleHTMLEncoder(), new QueryScorer(phraseQuery)); - for (int position = bitset.nextSetBit(0); position >= 0; position = bitset + for (int position = bitset.nextSetBit(0); position >= 0 && position < maxDoc-1; position = bitset .nextSetBit(position + 1)) { assertEquals(0, position); final TokenStream tokenStream = TokenSources.getTokenStream( Index: lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java --- lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/contrib/misc/src/java/org/apache/lucene/index/MultiPassIndexSplitter.java Wed Jul 06 10:39:55 2011 -0400 @@ -25,8 +25,8 @@ import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.util.FastBitSet; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.OpenBitSet; import org.apache.lucene.util.Version; /** @@ -178,7 +178,7 @@ * list of deletions. */ public static final class FakeDeleteIndexReader extends FilterIndexReader { - OpenBitSet liveDocs; + FastBitSet liveDocs; public FakeDeleteIndexReader(IndexReader in) { super(new SlowMultiReaderWrapper(in)); @@ -197,13 +197,13 @@ @Override protected void doUndeleteAll() { final int maxDoc = in.maxDoc(); - liveDocs = new OpenBitSet(maxDoc); + liveDocs = new FastBitSet(in.maxDoc()); if (in.hasDeletions()) { final Bits oldLiveDocs = in.getLiveDocs(); assert oldLiveDocs != null; // this loop is a little bit ineffective, as Bits has no nextSetBit(): for (int i = 0; i < maxDoc; i++) { - if (oldLiveDocs.get(i)) liveDocs.fastSet(i); + if (oldLiveDocs.get(i)) liveDocs.set(i); } } else { // mark all docs as valid Index: lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java --- lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/contrib/misc/src/java/org/apache/lucene/index/PKIndexSplitter.java Wed Jul 06 10:39:55 2011 -0400 @@ -27,8 +27,8 @@ import org.apache.lucene.search.TermRangeFilter; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.FastBitSet; import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.OpenBitSetDISI; import org.apache.lucene.util.Version; /** @@ -94,12 +94,12 @@ super(new SlowMultiReaderWrapper(reader)); final int maxDoc = in.maxDoc(); - final OpenBitSetDISI bits = new OpenBitSetDISI(maxDoc); + final FastBitSet bits = new FastBitSet(maxDoc); final DocIdSet docs = preserveFilter.getDocIdSet((AtomicReaderContext) in.getTopReaderContext()); if (docs != null) { final DocIdSetIterator it = docs.iterator(); if (it != null) { - bits.inPlaceOr(it); + bits.or(it); } } if (negateFilter) { @@ -113,7 +113,7 @@ for (int i = it.nextDoc(); i < maxDoc; i = it.nextDoc()) { if (!oldLiveDocs.get(i)) { // we can safely modify the current bit, as the iterator already stepped over it: - bits.fastClear(i); + bits.clear(i); } } } Index: lucene/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java --- lucene/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/contrib/queries/src/java/org/apache/lucene/search/DuplicateFilter.java Wed Jul 06 10:39:55 2011 -0400 @@ -24,7 +24,7 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.MultiFields; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; import org.apache.lucene.util.Bits; public class DuplicateFilter extends Filter @@ -84,8 +84,8 @@ } } - private OpenBitSet correctBits(IndexReader reader) throws IOException { - OpenBitSet bits = new OpenBitSet(reader.maxDoc()); //assume all are INvalid + private FastBitSet correctBits(IndexReader reader) throws IOException { + FastBitSet bits = new FastBitSet(reader.maxDoc()); //assume all are INvalid final Bits liveDocs = MultiFields.getLiveDocs(reader); Terms terms = reader.fields().terms(fieldName); if (terms != null) { @@ -119,10 +119,10 @@ return bits; } - private OpenBitSet fastBits(IndexReader reader) throws IOException + private FastBitSet fastBits(IndexReader reader) throws IOException { - OpenBitSet bits=new OpenBitSet(reader.maxDoc()); + FastBitSet bits=new FastBitSet(reader.maxDoc()); bits.set(0,reader.maxDoc()); //assume all are valid final Bits liveDocs = MultiFields.getLiveDocs(reader); Terms terms = reader.fields().terms(fieldName); Index: lucene/contrib/queries/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java --- lucene/contrib/queries/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/contrib/queries/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java Wed Jul 06 10:39:55 2011 -0400 @@ -111,6 +111,7 @@ @Override public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException { final FieldCache.DocTermsIndex fcsi = FieldCache.DEFAULT.getTermsIndex(context.reader, query.field); + // Cannot use FastBitSet because we require long index (ord): final OpenBitSet termSet = new OpenBitSet(fcsi.numOrd()); TermsEnum termsEnum = query.getTermsEnum(new Terms() { @@ -137,7 +138,7 @@ do { long ord = termsEnum.ord(); if (ord > 0) { - termSet.fastSet(ord); + termSet.set(ord); termCount++; } } while (termsEnum.next() != null); @@ -150,7 +151,7 @@ return new FieldCacheRangeFilter.FieldCacheDocIdSet(context.reader, true) { @Override boolean matchDoc(int doc) throws ArrayIndexOutOfBoundsException { - return termSet.fastGet(fcsi.getOrd(doc)); + return termSet.get(fcsi.getOrd(doc)); } }; } Index: lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java --- lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/contrib/queries/src/java/org/apache/lucene/search/TermsFilter.java Wed Jul 06 10:39:55 2011 -0400 @@ -29,7 +29,7 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.Fields; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.Bits; @@ -60,7 +60,7 @@ @Override public DocIdSet getDocIdSet(AtomicReaderContext context) throws IOException { IndexReader reader = context.reader; - OpenBitSet result=new OpenBitSet(reader.maxDoc()); + FastBitSet result=new FastBitSet(reader.maxDoc()); Fields fields = reader.fields(); BytesRef br = new BytesRef(); Bits liveDocs = reader.getLiveDocs(); Index: lucene/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java --- lucene/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/contrib/queries/src/test/org/apache/lucene/search/TermsFilterTest.java Wed Jul 06 10:39:55 2011 -0400 @@ -27,7 +27,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.index.SlowMultiReaderWrapper; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; public class TermsFilterTest extends LuceneTestCase { @@ -67,19 +67,19 @@ TermsFilter tf=new TermsFilter(); tf.addTerm(new Term(fieldName,"19")); - OpenBitSet bits = (OpenBitSet)tf.getDocIdSet(context); + FastBitSet bits = (FastBitSet)tf.getDocIdSet(context); assertEquals("Must match nothing", 0, bits.cardinality()); tf.addTerm(new Term(fieldName,"20")); - bits = (OpenBitSet)tf.getDocIdSet(context); + bits = (FastBitSet)tf.getDocIdSet(context); assertEquals("Must match 1", 1, bits.cardinality()); tf.addTerm(new Term(fieldName,"10")); - bits = (OpenBitSet)tf.getDocIdSet(context); + bits = (FastBitSet)tf.getDocIdSet(context); assertEquals("Must match 2", 2, bits.cardinality()); tf.addTerm(new Term(fieldName,"00")); - bits = (OpenBitSet)tf.getDocIdSet(context); + bits = (FastBitSet)tf.getDocIdSet(context); assertEquals("Must match 2", 2, bits.cardinality()); reader.close(); Index: lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java --- lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java Wed Jul 06 10:39:55 2011 -0400 @@ -27,7 +27,7 @@ import org.apache.lucene.util.NumericUtils; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; /** *

NOTE: This API is still in @@ -67,7 +67,7 @@ } }; } else { - final OpenBitSet bits = new OpenBitSet(context.reader.maxDoc()); + final FastBitSet bits = new FastBitSet(context.reader.maxDoc()); for (int i =0; i< sz; i++) { double boxId = area.get(i).doubleValue(); NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(boxId), 0, bytesRef); @@ -77,7 +77,7 @@ // which have this boxId int doc; while ((doc = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { - bits.fastSet(doc); + bits.set(doc); } } return bits; Index: lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java --- lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/CachingWrapperFilter.java Wed Jul 06 10:39:55 2011 -0400 @@ -23,7 +23,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.util.OpenBitSetDISI; +import org.apache.lucene.util.FastBitSet; import org.apache.lucene.util.Bits; /** @@ -186,7 +186,13 @@ // null is allowed to be returned by iterator(), // in this case we wrap with the empty set, // which is cacheable. - return (it == null) ? DocIdSet.EMPTY_DOCIDSET : new OpenBitSetDISI(it, reader.maxDoc()); + if (it == null) { + return DocIdSet.EMPTY_DOCIDSET; + } else { + final FastBitSet bits = new FastBitSet(reader.maxDoc()); + bits.or(it); + return bits; + } } } Index: lucene/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java --- lucene/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/FieldCacheTermsFilter.java Wed Jul 06 10:39:55 2011 -0400 @@ -22,7 +22,7 @@ import org.apache.lucene.index.DocsEnum; // javadoc @link import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; import org.apache.lucene.util.BytesRef; /** @@ -54,11 +54,11 @@ *

* * With each search, this filter translates the specified - * set of Terms into a private {@link OpenBitSet} keyed by + * set of Terms into a private {@link FastBitSet} keyed by * term number per unique {@link IndexReader} (normally one * reader per segment). Then, during matching, the term * number for each docID is retrieved from the cache and - * then checked for inclusion using the {@link OpenBitSet}. + * then checked for inclusion using the {@link FastBitSet}. * Since all testing is done using RAM resident data * structures, performance should be very fast, most likely * fast enough to not require further caching of the @@ -69,12 +69,12 @@ * *

* - * In contrast, TermsFilter builds up an {@link OpenBitSet}, + * In contrast, TermsFilter builds up an {@link FastBitSet}, * keyed by docID, every time it's created, by enumerating * through all matching docs using {@link DocsEnum} to seek * and scan through each term's docID list. While there is * no linear scan of all docIDs, besides the allocation of - * the underlying array in the {@link OpenBitSet}, this + * the underlying array in the {@link FastBitSet}, this * approach requires a number of "disk seeks" in proportion * to the number of terms, which can be exceptionally costly * when there are cache misses in the OS's IO cache. @@ -123,16 +123,16 @@ protected class FieldCacheTermsFilterDocIdSet extends DocIdSet { private FieldCache.DocTermsIndex fcsi; - private OpenBitSet openBitSet; + private FastBitSet bits; public FieldCacheTermsFilterDocIdSet(FieldCache.DocTermsIndex fcsi) { this.fcsi = fcsi; - openBitSet = new OpenBitSet(this.fcsi.numOrd()); + bits = new FastBitSet(this.fcsi.numOrd()); final BytesRef spare = new BytesRef(); for (int i=0;i 0) { - openBitSet.fastSet(termNumber); + bits.set(termNumber); } } } @@ -159,7 +159,7 @@ @Override public int nextDoc() { try { - while (!openBitSet.fastGet(fcsi.getOrd(++doc))) {} + while (!bits.get(fcsi.getOrd(++doc))) {} } catch (ArrayIndexOutOfBoundsException e) { doc = NO_MORE_DOCS; } @@ -170,7 +170,7 @@ public int advance(int target) { try { doc = target; - while (!openBitSet.fastGet(fcsi.getOrd(doc))) { + while (!bits.get(fcsi.getOrd(doc))) { doc++; } } catch (ArrayIndexOutOfBoundsException e) { Index: lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java --- lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Wed Jul 06 10:39:55 2011 -0400 @@ -25,8 +25,9 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.util.FastBitSet; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.OpenBitSet; /** * A wrapper for {@link MultiTermQuery}, that exposes its @@ -122,8 +123,8 @@ final TermsEnum termsEnum = query.getTermsEnum(terms); assert termsEnum != null; if (termsEnum.next() != null) { - // fill into a OpenBitSet - final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc()); + // fill into a FastBitSet + final FastBitSet bitSet = new FastBitSet(context.reader.maxDoc()); int termCount = 0; final Bits liveDocs = reader.getLiveDocs(); DocsEnum docsEnum = null; Index: lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java --- lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/SpanQueryFilter.java Wed Jul 06 10:39:55 2011 -0400 @@ -19,7 +19,7 @@ import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.Spans; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; import java.io.IOException; import java.util.ArrayList; @@ -60,7 +60,7 @@ @Override public SpanFilterResult bitSpans(AtomicReaderContext context) throws IOException { - final OpenBitSet bits = new OpenBitSet(context.reader.maxDoc()); + final FastBitSet bits = new FastBitSet(context.reader.maxDoc()); Spans spans = query.getSpans(context); List tmp = new ArrayList(20); int currentDoc = -1; Index: lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java --- lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/cache/ByteValuesCreator.java Wed Jul 06 10:39:55 2011 -0400 @@ -32,7 +32,7 @@ import org.apache.lucene.search.cache.CachedArray.ByteValues; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; public class ByteValuesCreator extends CachedArrayCreator { @@ -110,7 +110,7 @@ vals.values = new byte[maxDoc]; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; + FastBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new FastBitSet( maxDoc ) : null; DocsEnum docs = null; try { while(true) { Index: lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java --- lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/cache/CachedArrayCreator.java Wed Jul 06 10:39:55 2011 -0400 @@ -29,7 +29,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; public abstract class CachedArrayCreator extends EntryCreatorWithOptions { @@ -101,7 +101,7 @@ /** * Utility function to help check what bits are valid */ - protected Bits checkMatchAllBits( OpenBitSet valid, int numDocs, int maxDocs ) + protected Bits checkMatchAllBits( FastBitSet valid, int numDocs, int maxDocs ) { if( numDocs != maxDocs ) { if( hasOption( OPTION_CACHE_BITS ) ) { @@ -124,7 +124,7 @@ Terms terms = MultiFields.getTerms(reader, field); if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - OpenBitSet validBits = new OpenBitSet( reader.maxDoc() ); + FastBitSet validBits = new FastBitSet( reader.maxDoc() ); DocsEnum docs = null; while(true) { final BytesRef term = termsEnum.next(); Index: lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java --- lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/cache/DoubleValuesCreator.java Wed Jul 06 10:39:55 2011 -0400 @@ -32,7 +32,7 @@ import org.apache.lucene.search.cache.CachedArray.DoubleValues; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; public class DoubleValuesCreator extends CachedArrayCreator { @@ -120,7 +120,7 @@ vals.values = null; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; + FastBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new FastBitSet( maxDoc ) : null; DocsEnum docs = null; try { while(true) { Index: lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java --- lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/cache/FloatValuesCreator.java Wed Jul 06 10:39:55 2011 -0400 @@ -32,7 +32,7 @@ import org.apache.lucene.search.cache.CachedArray.FloatValues; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; public class FloatValuesCreator extends CachedArrayCreator { @@ -121,7 +121,7 @@ vals.values = null; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; + FastBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new FastBitSet( maxDoc ) : null; DocsEnum docs = null; try { while(true) { Index: lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java --- lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/cache/IntValuesCreator.java Wed Jul 06 10:39:55 2011 -0400 @@ -32,7 +32,7 @@ import org.apache.lucene.search.cache.CachedArray.IntValues; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; public class IntValuesCreator extends CachedArrayCreator { @@ -121,7 +121,7 @@ vals.values = null; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; + FastBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new FastBitSet( maxDoc ) : null; DocsEnum docs = null; try { while(true) { Index: lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java --- lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/cache/LongValuesCreator.java Wed Jul 06 10:39:55 2011 -0400 @@ -32,7 +32,7 @@ import org.apache.lucene.search.cache.CachedArray.LongValues; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; public class LongValuesCreator extends CachedArrayCreator { @@ -121,7 +121,7 @@ vals.values = null; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; + FastBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new FastBitSet( maxDoc ) : null; DocsEnum docs = null; try { while(true) { Index: lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java --- lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/java/org/apache/lucene/search/cache/ShortValuesCreator.java Wed Jul 06 10:39:55 2011 -0400 @@ -32,7 +32,7 @@ import org.apache.lucene.search.cache.CachedArray.ShortValues; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; public class ShortValuesCreator extends CachedArrayCreator { @@ -111,7 +111,7 @@ vals.values = new short[maxDoc]; if (terms != null) { final TermsEnum termsEnum = terms.iterator(); - OpenBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new OpenBitSet( maxDoc ) : null; + FastBitSet validBits = (hasOption(OPTION_CACHE_BITS)) ? new FastBitSet( maxDoc ) : null; DocsEnum docs = null; try { while(true) { Index: lucene/src/java/org/apache/lucene/util/FastBitSet.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ lucene/src/java/org/apache/lucene/util/FastBitSet.java Wed Jul 06 10:39:55 2011 -0400 @@ -0,0 +1,288 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.lucene.util; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.search.DocIdSet; +import org.apache.lucene.search.DocIdSetIterator; + +// nocommit -- need better name? + +// TODO: maybe merge with BitVector? Problem is BitVector +// caches its cardinality... + +/** BitSet of fixed length (numBits), backed by private + * long[], accessed with an int index, implementing Bits + * and DocIdSet. + * + * @lucene.internal + **/ + +// nocommit needs test + +public final class FastBitSet extends DocIdSet implements Bits { + private final long[] bits; + private int numBits; + + /** returns the number of 64 bit words it would take to hold numBits */ + public static int bits2words(int numBits) { + return (((numBits-1)>>>6)+1); + } + + public FastBitSet(int numBits) { + this.numBits = numBits; + bits = new long[bits2words(numBits)]; + } + + @Override + public DocIdSetIterator iterator() { + return new OpenBitSetIterator(bits, bits.length); + } + + @Override + public int length() { + return numBits; + } + + /** This DocIdSet implementation is cacheable. */ + @Override + public boolean isCacheable() { + return true; + } + + /** Returns number of set bits. NOTE: this visits every + * long in the backing bits array, and the result is not + * internally cached! */ + public int cardinality() { + return (int) BitUtil.pop_array(bits, 0, bits.length); + } + + public boolean get(int index) { + assert index >= 0 && index < numBits; + int i = index >> 6; // div 64 + // signed shift will keep a negative index and force an + // array-index-out-of-bounds-exception, removing the need for an explicit check. + int bit = index & 0x3f; // mod 64 + long bitmask = 1L << bit; + return (bits[i] & bitmask) != 0; + } + + public void set(int index) { + assert index >= 0 && index < numBits; + int wordNum = index >> 6; // div 64 + int bit = index & 0x3f; // mod 64 + long bitmask = 1L << bit; + bits[wordNum] |= bitmask; + } + + public boolean getAndSet(int index) { + assert index >= 0 && index < numBits; + int wordNum = index >> 6; // div 64 + int bit = index & 0x3f; // mod 64 + long bitmask = 1L << bit; + boolean val = (bits[wordNum] & bitmask) != 0; + bits[wordNum] |= bitmask; + return val; + } + + public void clear(int index) { + assert index >= 0 && index < numBits; + int wordNum = index >> 6; + int bit = index & 0x03f; + long bitmask = 1L << bit; + bits[wordNum] &= ~bitmask; + } + + public boolean getAndClear(int index) { + assert index >= 0 && index < numBits; + int wordNum = index >> 6; // div 64 + int bit = index & 0x3f; // mod 64 + long bitmask = 1L << bit; + boolean val = (bits[wordNum] & bitmask) != 0; + bits[wordNum] &= ~bitmask; + return val; + } + + /** Returns the index of the first set bit starting at the index specified. + * -1 is returned if there are no more set bits. + */ + public int nextSetBit(int index) { + assert index >= 0 && index < numBits; + int i = index >> 6; + int subIndex = index & 0x3f; // index within the word + long word = bits[i] >> subIndex; // skip all the bits to the right of index + + if (word!=0) { + return (i<<6) + subIndex + BitUtil.ntz(word); + } + + while(++i < bits.length) { + word = bits[i]; + if (word != 0) { + return (i<<6) + BitUtil.ntz(word); + } + } + + return -1; + } + + public int prevSetBit(int index) { + assert index >= 0 && index < numBits; + int i = index >> 6; + final int subIndex; + long word; + subIndex = index & 0x3f; // index within the word + word = (bits[i] << (63-subIndex)); // skip all the bits to the left of index + + if (word != 0) { + return (i << 6) + subIndex - Long.numberOfLeadingZeros(word); // See LUCENE-3197 + } + + while (--i >= 0) { + word = bits[i]; + if (word !=0 ) { + return (i << 6) + 63 - Long.numberOfLeadingZeros(word); + } + } + + return -1; + } + + /** Does in-place OR of the bits provided by the + * iterator. */ + public void or(DocIdSetIterator iter) throws IOException { + int doc; + while ((doc = iter.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { + set(doc); + } + } + + public void or(FastBitSet other) { + long[] thisArr = this.bits; + long[] otherArr = other.bits; + int pos = Math.min(thisArr.length, otherArr.length); + while (--pos >= 0) { + thisArr[pos] |= otherArr[pos]; + } + } + + // NOTE: no .isEmpty() here because that's trappy (ie, + // typically isEmpty is low cost, but this one wouldn't + // be) + + /** Flips a range of bits + * + * @param startIndex lower index + * @param endIndex one-past the last bit to flip + */ + public void flip(int startIndex, int endIndex) { + assert startIndex >= 0 && startIndex < numBits; + assert endIndex >= 0 && endIndex <= numBits; + if (endIndex <= startIndex) { + return; + } + + int startWord = startIndex >> 6; + int endWord = (endIndex-1) >> 6; + + /*** Grrr, java shifting wraps around so -1L>>>64 == -1 + * for that reason, make sure not to use endmask if the bits to flip will + * be zero in the last word (redefine endWord to be the last changed...) + long startmask = -1L << (startIndex & 0x3f); // example: 11111...111000 + long endmask = -1L >>> (64-(endIndex & 0x3f)); // example: 00111...111111 + ***/ + + long startmask = -1L << startIndex; + long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap + + if (startWord == endWord) { + bits[startWord] ^= (startmask & endmask); + return; + } + + bits[startWord] ^= startmask; + + for (int i=startWord+1; i= 0 && startIndex < numBits; + assert endIndex >= 0 && endIndex <= numBits; + if (endIndex <= startIndex) { + return; + } + + int startWord = startIndex >> 6; + int endWord = (endIndex-1) >> 6; + + long startmask = -1L << startIndex; + long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap + + if (startWord == endWord) { + bits[startWord] |= (startmask & endmask); + return; + } + + bits[startWord] |= startmask; + Arrays.fill(bits, startWord+1, endWord, -1L); + bits[endWord] |= endmask; + } + + /** Clears a range of bits. + * + * @param startIndex lower index + * @param endIndex one-past the last bit to clear + */ + public void clear(int startIndex, int endIndex) { + assert startIndex >= 0 && startIndex < numBits; + assert endIndex >= 0 && endIndex <= numBits; + if (endIndex <= startIndex) { + return; + } + + int startWord = startIndex >> 6; + int endWord = (endIndex-1) >> 6; + + long startmask = -1L << startIndex; + long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap + + // invert masks since we are clearing + startmask = ~startmask; + endmask = ~endmask; + + if (startWord == endWord) { + bits[startWord] &= (startmask | endmask); + return; + } + + bits[startWord] &= startmask; + Arrays.fill(bits, startWord+1, endWord, 0L); + bits[endWord] &= endmask; + } +} \ No newline at end of file Index: lucene/src/test/org/apache/lucene/index/TestLongPostings.java --- lucene/src/test/org/apache/lucene/index/TestLongPostings.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/test/org/apache/lucene/index/TestLongPostings.java Wed Jul 06 10:39:55 2011 -0400 @@ -30,7 +30,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; import org.apache.lucene.util._TestUtil; public class TestLongPostings extends LuceneTestCase { @@ -91,7 +91,7 @@ */ } - final OpenBitSet isS1 = new OpenBitSet(NUM_DOCS); + final FastBitSet isS1 = new FastBitSet(NUM_DOCS); for(int idx=0;idx closeables = new ArrayList(); IndexReader r = IndexReader.open(w, true); - final int numRemainingValues = (int) (numValues - deleted.cardinality()); + final int numRemainingValues = numValues - deleted.cardinality(); final int base = r.numDocs() - numRemainingValues; // for FIXED_INTS_8 we use value mod 128 - to enable testing in // one go we simply use numValues as the mod for all other INT types @@ -331,11 +331,11 @@ for (ValueType byteIndexValue : byteVariantList) { List closeables = new ArrayList(); final int bytesSize = 1 + atLeast(50); - OpenBitSet deleted = indexValues(w, numValues, byteIndexValue, + FastBitSet deleted = indexValues(w, numValues, byteIndexValue, byteVariantList, withDeletions, bytesSize); final IndexReader r = IndexReader.open(w, withDeletions); assertEquals(0, r.numDeletedDocs()); - final int numRemainingValues = (int) (numValues - deleted.cardinality()); + final int numRemainingValues = numValues - deleted.cardinality(); final int base = r.numDocs() - numRemainingValues; IndexDocValues bytesReader = getDocValues(r, byteIndexValue.name()); assertNotNull("field " + byteIndexValue.name() @@ -484,11 +484,11 @@ Index.ANALYZED_NO_NORMS, Index.NOT_ANALYZED, Index.NOT_ANALYZED_NO_NORMS, Index.NO }; - private OpenBitSet indexValues(IndexWriter w, int numValues, ValueType value, + private FastBitSet indexValues(IndexWriter w, int numValues, ValueType value, List valueVarList, boolean withDeletions, int bytesSize) throws CorruptIndexException, IOException { final boolean isNumeric = NUMERICS.contains(value); - OpenBitSet deleted = new OpenBitSet(numValues); + FastBitSet deleted = new FastBitSet(numValues); Document doc = new Document(); Index idx = IDX_VALUES[random.nextInt(IDX_VALUES.length)]; AbstractField field = random.nextBoolean() ? new IndexDocValuesField(value.name()) Index: lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java --- lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/test/org/apache/lucene/search/TestCachingWrapperFilter.java Wed Jul 06 10:39:55 2011 -0400 @@ -30,8 +30,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.OpenBitSet; -import org.apache.lucene.util.OpenBitSetDISI; +import org.apache.lucene.util.FastBitSet; import org.apache.lucene.util._TestUtil; public class TestCachingWrapperFilter extends LuceneTestCase { @@ -125,7 +124,7 @@ if (originalSet.isCacheable()) { assertEquals("Cached DocIdSet must be of same class like uncached, if cacheable", originalSet.getClass(), cachedSet.getClass()); } else { - assertTrue("Cached DocIdSet must be an OpenBitSet if the original one was not cacheable", cachedSet instanceof OpenBitSetDISI || cachedSet == DocIdSet.EMPTY_DOCIDSET); + assertTrue("Cached DocIdSet must be an FastBitSet if the original one was not cacheable", cachedSet instanceof FastBitSet || cachedSet == DocIdSet.EMPTY_DOCIDSET); } } @@ -143,11 +142,11 @@ assertDocIdSetCacheable(reader, NumericRangeFilter.newIntRange("test", Integer.valueOf(10000), Integer.valueOf(-10000), true, true), true); // is cacheable: assertDocIdSetCacheable(reader, FieldCacheRangeFilter.newIntRange("test", Integer.valueOf(10), Integer.valueOf(20), true, true), true); - // a openbitset filter is always cacheable + // a fastbitset filter is always cacheable assertDocIdSetCacheable(reader, new Filter() { @Override public DocIdSet getDocIdSet(AtomicReaderContext context) { - return new OpenBitSet(); + return new FastBitSet(context.reader.maxDoc()); } }, true); Index: lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java --- lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/test/org/apache/lucene/search/TestFilteredSearch.java Wed Jul 06 10:39:55 2011 -0400 @@ -30,7 +30,7 @@ import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; @@ -97,7 +97,7 @@ @Override public DocIdSet getDocIdSet(AtomicReaderContext context) { assert context.isAtomic; - final OpenBitSet set = new OpenBitSet(); + final FastBitSet set = new FastBitSet(context.reader.maxDoc()); int docBase = context.docBase; final int limit = docBase+context.reader.maxDoc(); for (;index < docs.length; index++) { @@ -108,7 +108,7 @@ set.set(docId-docBase); } } - return set.isEmpty()?null:set; + return set.cardinality() == 0 ? null:set; } public void reset(){ Index: lucene/src/test/org/apache/lucene/search/cache/TestEntryCreators.java --- lucene/src/test/org/apache/lucene/search/cache/TestEntryCreators.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/test/org/apache/lucene/search/cache/TestEntryCreators.java Wed Jul 06 10:39:55 2011 -0400 @@ -31,7 +31,7 @@ import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; import static org.hamcrest.CoreMatchers.*; @@ -220,7 +220,7 @@ } assertEquals( "Cached numTerms does not match : "+tester, distinctTerms.size(), cachedVals.numTerms ); assertEquals( "Cached numDocs does not match : "+tester, numDocs, cachedVals.numDocs ); - assertEquals( "Ordinal should match numDocs : "+tester, numDocs, ((OpenBitSet)cachedVals.valid).cardinality() ); + assertEquals( "Ordinal should match numDocs : "+tester, numDocs, ((FastBitSet)cachedVals.valid).cardinality() ); } } Index: lucene/src/test/org/apache/lucene/util/TestNumericUtils.java --- lucene/src/test/org/apache/lucene/util/TestNumericUtils.java Wed Jul 06 10:28:30 2011 -0400 +++ lucene/src/test/org/apache/lucene/util/TestNumericUtils.java Wed Jul 06 10:39:55 2011 -0400 @@ -184,6 +184,7 @@ private void assertLongRangeSplit(final long lower, final long upper, int precisionStep, final boolean useBitSet, final Iterable expectedBounds, final Iterable expectedShifts ) throws Exception { + // Cannot use FastBitSet since the range could be long: final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null; final Iterator neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator(); final Iterator neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator(); @@ -212,7 +213,7 @@ if (useBitSet) { // after flipping all bits in the range, the cardinality should be zero bits.flip(0,upper-lower+1); - assertTrue("The sub-range concenated should match the whole range", bits.isEmpty()); + assertEquals("The sub-range concenated should match the whole range", 0, bits.cardinality()); } } @@ -424,7 +425,7 @@ private void assertIntRangeSplit(final int lower, final int upper, int precisionStep, final boolean useBitSet, final Iterable expectedBounds, final Iterable expectedShifts ) throws Exception { - final OpenBitSet bits=useBitSet ? new OpenBitSet(upper-lower+1) : null; + final FastBitSet bits=useBitSet ? new FastBitSet(upper-lower+1) : null; final Iterator neededBounds = (expectedBounds == null) ? null : expectedBounds.iterator(); final Iterator neededShifts = (expectedShifts == null) ? null : expectedShifts.iterator(); @@ -451,8 +452,8 @@ if (useBitSet) { // after flipping all bits in the range, the cardinality should be zero - bits.flip(0,upper-lower+1); - assertTrue("The sub-range concenated should match the whole range", bits.isEmpty()); + bits.flip(0, upper-lower+1); + assertEquals("The sub-range concenated should match the whole range", 0, bits.cardinality()); } } Index: modules/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java --- modules/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java Wed Jul 06 10:28:30 2011 -0400 +++ modules/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java Wed Jul 06 10:39:55 2011 -0400 @@ -88,7 +88,6 @@ private int totalGroupCount; private int docBase; private int groupEndDocID; - //private OpenBitSet lastDocPerGroupBits; private DocIdSetIterator lastDocPerGroupBits; private Scorer scorer; private final GroupQueue groupQueue; Index: modules/join/src/java/org/apache/lucene/search/join/BlockJoinQuery.java --- modules/join/src/java/org/apache/lucene/search/join/BlockJoinQuery.java Wed Jul 06 10:28:30 2011 -0400 +++ modules/join/src/java/org/apache/lucene/search/join/BlockJoinQuery.java Wed Jul 06 10:39:55 2011 -0400 @@ -35,7 +35,7 @@ import org.apache.lucene.search.Weight; import org.apache.lucene.search.grouping.TopGroups; import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.OpenBitSet; +import org.apache.lucene.util.FastBitSet; /** * This query requires that you index @@ -45,7 +45,7 @@ * child documents must appear first, ending with the parent * document. At search time you provide a Filter * identifying the parents, however this Filter must provide - * an {@link OpenBitSet} per sub-reader. + * an {@link FastBitSet} per sub-reader. * *

Once the block index is built, use this query to wrap * any sub-query matching only child docs and join matches in that @@ -170,11 +170,11 @@ // No matches return null; } - if (!(parents instanceof OpenBitSet)) { - throw new IllegalStateException("parentFilter must return OpenBitSet; got " + parents); + if (!(parents instanceof FastBitSet)) { + throw new IllegalStateException("parentFilter must return FastBitSet; got " + parents); } - return new BlockJoinScorer(this, childScorer, (OpenBitSet) parents, firstChildDoc, scoreMode); + return new BlockJoinScorer(this, childScorer, (FastBitSet) parents, firstChildDoc, scoreMode); } @Override @@ -192,7 +192,7 @@ static class BlockJoinScorer extends Scorer { private final Scorer childScorer; - private final OpenBitSet parentBits; + private final FastBitSet parentBits; private final ScoreMode scoreMode; private int parentDoc; private float parentScore; @@ -202,7 +202,7 @@ private float[] pendingChildScores; private int childDocUpto; - public BlockJoinScorer(Weight weight, Scorer childScorer, OpenBitSet parentBits, int firstChildDoc, ScoreMode scoreMode) { + public BlockJoinScorer(Weight weight, Scorer childScorer, FastBitSet parentBits, int firstChildDoc, ScoreMode scoreMode) { super(weight); //System.out.println("Q.init firstChildDoc=" + firstChildDoc); this.parentBits = parentBits;