Index: lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java =================================================================== --- lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (revision 1564668) +++ lucene/core/src/java/org/apache/lucene/util/FixedBitSet.java (working copy) @@ -78,7 +78,68 @@ @Override public DocIdSetIterator iterator() { - return new OpenBitSetIterator(bits, wordLength); + // define locally so we don't have "enclosing acces" issue + final long[] bits = this.bits; + final int wordLength = this.wordLength; + final int numBits = this.numBits; + return new DocIdSetIterator() { + int doc = -1; + @Override + public int nextDoc() throws IOException { + if (doc == NO_MORE_DOCS || ++doc >= numBits) { + return doc = NO_MORE_DOCS; + } + int i = doc >> 6; + final int subIndex = doc & 0x3f; // index within the word + long word = bits[i] >> subIndex; // skip all the bits to the right of index + + if (word != 0) { + return doc = doc + Long.numberOfTrailingZeros(word); + } + + while (++i < wordLength) { + word = bits[i]; + if (word != 0) { + return doc = (i << 6) + Long.numberOfTrailingZeros(word); + } + } + + return doc = NO_MORE_DOCS; + } + + @Override + public int docID() { + return doc; + } + + @Override + public long cost() { + return bits.length; + } + + @Override + public int advance(int target) throws IOException { + if (doc == NO_MORE_DOCS || target >= numBits) { + return doc = NO_MORE_DOCS; + } + int i = target >> 6; + final int subIndex = target & 0x3f; // index within the word + long word = bits[i] >> subIndex; // skip all the bits to the right of index + + if (word != 0) { + return doc = target + Long.numberOfTrailingZeros(word); + } + + while (++i < wordLength) { + word = bits[i]; + if (word != 0) { + return doc = (i << 6) + Long.numberOfTrailingZeros(word); + } + } + + return doc = NO_MORE_DOCS; + } + }; } @Override @@ -166,7 +227,7 @@ long word = bits[i] >> subIndex; // skip all the bits to the right of index if (word!=0) { - return (i<<6) + subIndex + Long.numberOfTrailingZeros(word); + return index + Long.numberOfTrailingZeros(word); } while(++i < wordLength) { Index: lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java (revision 1564668) +++ lucene/facet/src/java/org/apache/lucene/facet/FacetsCollector.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.index.AtomicReaderContext; import org.apache.lucene.search.Collector; +import org.apache.lucene.search.DocIdSet; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.Filter; import org.apache.lucene.search.FilteredQuery; @@ -50,13 +51,22 @@ private AtomicReaderContext context; private Scorer scorer; - private FixedBitSet bits; private int totalHits; private float[] scores; private final boolean keepScores; private final List matchingDocs = new ArrayList(); + private MutableDocIdSet docs; /** + * Used during collection to record matching docs and then return a + * {@link DocIdSet} that contains them. + */ + protected static abstract class MutableDocIdSet { + public abstract void addDoc(int docId) throws IOException; + public abstract DocIdSet getDocs(); + } + + /** * Holds the documents that were matched in the {@link AtomicReaderContext}. * If scores were required, then {@code scores} is not null. */ @@ -66,7 +76,7 @@ public final AtomicReaderContext context; /** Which documents were seen. */ - public final FixedBitSet bits; + public final DocIdSet bits; /** Non-sparse scores array. */ public final float[] scores; @@ -75,7 +85,7 @@ public final int totalHits; /** Sole constructor. */ - public MatchingDocs(AtomicReaderContext context, FixedBitSet bits, int totalHits, float[] scores) { + public MatchingDocs(AtomicReaderContext context, DocIdSet bits, int totalHits, float[] scores) { this.context = context; this.bits = bits; this.scores = scores; @@ -93,6 +103,24 @@ public FacetsCollector(boolean keepScores) { this.keepScores = keepScores; } + + /** Creates a {@link MutableDocIdSet} to record hits. */ + protected MutableDocIdSet createHitsSet(final int maxDoc) { + return new MutableDocIdSet() { + private final FixedBitSet bits = new FixedBitSet(maxDoc); + + @Override + public void addDoc(int docId) throws IOException { + bits.set(docId); + } + + @Override + public DocIdSet getDocs() { + return bits; + } + + }; + } /** True if scores were saved. */ public boolean getKeepScores() { @@ -104,9 +132,9 @@ * visited segment. */ public List getMatchingDocs() { - if (bits != null) { - matchingDocs.add(new MatchingDocs(this.context, bits, totalHits, scores)); - bits = null; + if (docs != null) { + matchingDocs.add(new MatchingDocs(this.context, docs.getDocs(), totalHits, scores)); + docs = null; scores = null; context = null; } @@ -124,7 +152,7 @@ @Override public final void collect(int doc) throws IOException { - bits.set(doc); + docs.addDoc(doc); if (keepScores) { if (totalHits >= scores.length) { float[] newScores = new float[ArrayUtil.oversize(totalHits + 1, 4)]; @@ -143,10 +171,10 @@ @Override public final void setNextReader(AtomicReaderContext context) throws IOException { - if (bits != null) { - matchingDocs.add(new MatchingDocs(this.context, bits, totalHits, scores)); + if (docs != null) { + matchingDocs.add(new MatchingDocs(this.context, docs.getDocs(), totalHits, scores)); } - bits = new FixedBitSet(context.reader().maxDoc()); + docs = createHitsSet(context.reader().maxDoc()); totalHits = 0; if (keepScores) { scores = new float[64]; // some initial size Index: lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java (revision 1564668) +++ lucene/facet/src/java/org/apache/lucene/facet/range/DoubleRangeFacetCounts.java (working copy) @@ -30,6 +30,7 @@ import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.DoubleFieldSource; import org.apache.lucene.queries.function.valuesource.FloatFieldSource; // javadocs +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.NumericUtils; /** {@link Facets} implementation that computes counts for @@ -81,10 +82,12 @@ int missingCount = 0; for (MatchingDocs hits : matchingDocs) { FunctionValues fv = valueSource.getValues(Collections.emptyMap(), hits.context); - final int length = hits.bits.length(); - int doc = 0; + totCount += hits.totalHits; - while (doc < length && (doc = hits.bits.nextSetBit(doc)) != -1) { + DocIdSetIterator docs = hits.bits.iterator(); + + int doc; + while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { // Skip missing docs: if (fv.exists(doc)) { counter.add(NumericUtils.doubleToSortableLong(fv.doubleVal(doc))); @@ -91,7 +94,6 @@ } else { missingCount++; } - doc++; } } Index: lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java (revision 1564668) +++ lucene/facet/src/java/org/apache/lucene/facet/range/LongRangeFacetCounts.java (working copy) @@ -27,6 +27,7 @@ import org.apache.lucene.queries.function.FunctionValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.valuesource.LongFieldSource; +import org.apache.lucene.search.DocIdSetIterator; /** {@link Facets} implementation that computes counts for * dynamic long ranges from a provided {@link ValueSource}, @@ -62,10 +63,11 @@ int missingCount = 0; for (MatchingDocs hits : matchingDocs) { FunctionValues fv = valueSource.getValues(Collections.emptyMap(), hits.context); - final int length = hits.bits.length(); - int doc = 0; + totCount += hits.totalHits; - while (doc < length && (doc = hits.bits.nextSetBit(doc)) != -1) { + DocIdSetIterator docs = hits.bits.iterator(); + int doc; + while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { // Skip missing docs: if (fv.exists(doc)) { counter.add(fv.longVal(doc)); @@ -72,8 +74,6 @@ } else { missingCount++; } - - doc++; } } Index: lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java (revision 1564668) +++ lucene/facet/src/java/org/apache/lucene/facet/sortedset/SortedSetDocValuesFacetCounts.java (working copy) @@ -38,6 +38,7 @@ import org.apache.lucene.index.MultiDocValues.MultiSortedSetDocValues; import org.apache.lucene.index.ReaderUtil; import org.apache.lucene.index.SortedSetDocValues; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; /** Compute facets counts from previously @@ -175,9 +176,7 @@ continue; } - final int maxDoc = reader.maxDoc(); - assert maxDoc == hits.bits.length(); - //System.out.println(" dv=" + dv); + DocIdSetIterator docs = hits.bits.iterator(); // TODO: yet another option is to count all segs // first, only in seg-ord space, and then do a @@ -196,8 +195,8 @@ if (hits.totalHits < numSegOrds/10) { //System.out.println(" remap as-we-go"); // Remap every ord to global ord as we iterate: - int doc = 0; - while (doc < maxDoc && (doc = hits.bits.nextSetBit(doc)) != -1) { + int doc; + while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { //System.out.println(" doc=" + doc); segValues.setDocument(doc); int term = (int) segValues.nextOrd(); @@ -206,7 +205,6 @@ counts[(int) ordinalMap.getGlobalOrd(segOrd, term)]++; term = (int) segValues.nextOrd(); } - ++doc; } } else { //System.out.println(" count in seg ord first"); @@ -213,8 +211,8 @@ // First count in seg-ord space: final int[] segCounts = new int[numSegOrds]; - int doc = 0; - while (doc < maxDoc && (doc = hits.bits.nextSetBit(doc)) != -1) { + int doc; + while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { //System.out.println(" doc=" + doc); segValues.setDocument(doc); int term = (int) segValues.nextOrd(); @@ -223,7 +221,6 @@ segCounts[term]++; term = (int) segValues.nextOrd(); } - ++doc; } // Then, migrate to global ords: @@ -238,9 +235,8 @@ } else { // No ord mapping (e.g., single segment index): // just aggregate directly into counts: - - int doc = 0; - while (doc < maxDoc && (doc = hits.bits.nextSetBit(doc)) != -1) { + int doc; + while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { segValues.setDocument(doc); int term = (int) segValues.nextOrd(); while (term != SortedSetDocValues.NO_MORE_ORDS) { @@ -247,7 +243,6 @@ counts[term]++; term = (int) segValues.nextOrd(); } - ++doc; } } } Index: lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java (revision 1564668) +++ lucene/facet/src/java/org/apache/lucene/facet/taxonomy/FastTaxonomyFacetCounts.java (working copy) @@ -21,11 +21,11 @@ import java.util.List; import org.apache.lucene.facet.FacetsCollector; +import org.apache.lucene.facet.FacetsCollector.MatchingDocs; import org.apache.lucene.facet.FacetsConfig; -import org.apache.lucene.facet.FacetsCollector.MatchingDocs; import org.apache.lucene.index.BinaryDocValues; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.FixedBitSet; /** Computes facets counts, assuming the default encoding * into DocValues was used. @@ -55,12 +55,12 @@ if (dv == null) { // this reader does not have DocValues for the requested category list continue; } - FixedBitSet bits = hits.bits; - - final int length = hits.bits.length(); - int doc = 0; + BytesRef scratch = new BytesRef(); - while (doc < length && (doc = bits.nextSetBit(doc)) != -1) { + DocIdSetIterator docs = hits.bits.iterator(); + + int doc; + while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { dv.get(doc, scratch); byte[] bytes = scratch.bytes; int end = scratch.offset + scratch.length; @@ -77,7 +77,6 @@ ord = (ord << 7) | (b & 0x7F); } } - ++doc; } } Index: lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java =================================================================== --- lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java (revision 1564668) +++ lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetCounts.java (working copy) @@ -24,7 +24,7 @@ import org.apache.lucene.facet.FacetsCollector.MatchingDocs; import org.apache.lucene.facet.FacetsConfig; import org.apache.lucene.index.BinaryDocValues; -import org.apache.lucene.util.FixedBitSet; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.IntsRef; /** Reads from any {@link OrdinalsReader}; use {@link @@ -49,16 +49,14 @@ IntsRef scratch = new IntsRef(); for(MatchingDocs hits : matchingDocs) { OrdinalsReader.OrdinalsSegmentReader ords = ordinalsReader.getReader(hits.context); - FixedBitSet bits = hits.bits; - - final int length = hits.bits.length(); - int doc = 0; - while (doc < length && (doc = bits.nextSetBit(doc)) != -1) { + DocIdSetIterator docs = hits.bits.iterator(); + + int doc; + while ((doc = docs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { ords.get(doc, scratch); for(int i=0;i