Index: lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java (revision 1407932) +++ lucene/queries/src/java/org/apache/lucene/queries/BooleanFilter.java (working copy) @@ -44,7 +44,23 @@ public class BooleanFilter extends Filter implements Iterable { private final List clauses = new ArrayList(); + private final boolean passAcceptDocs; + + /** Creates a new BooleanFilter, which does not pass the accept docs down to the filter clauses. */ + public BooleanFilter() { + this(false); + } + /** Creates a new BooleanFilter. + * @param passAcceptDocs if enabled, the filter passes deleted documents down to each filter clause. + * Additionally, when applying MUST clauses, already filtered docs get excluded, too. Enabling this + * setting may slow down cheap filters because of repeated checks for deleted docs, but it may + * speed up filters that are expensive to create and can leverage the already filtered docs. + */ + public BooleanFilter(boolean passAcceptDocs) { + this.passAcceptDocs = passAcceptDocs; + } + /** * Returns the a DocIdSetIterator representing the Boolean composition * of the filters that have been added. @@ -54,11 +70,14 @@ FixedBitSet res = null; final AtomicReader reader = context.reader(); + // We filter the final bit set, if we don't pass the acceptDocs down to the filter clauses: + boolean enforceAcceptDocs = !passAcceptDocs; + boolean hasShouldClauses = false; for (final FilterClause fc : clauses) { if (fc.getOccur() == Occur.SHOULD) { hasShouldClauses = true; - final DocIdSetIterator disi = getDISI(fc.getFilter(), context); + final DocIdSetIterator disi = getDISI(fc.getFilter(), context, acceptDocs); if (disi == null) continue; if (res == null) { res = new FixedBitSet(reader.maxDoc()); @@ -74,9 +93,12 @@ if (res == null) { assert !hasShouldClauses; res = new FixedBitSet(reader.maxDoc()); - res.set(0, reader.maxDoc()); // NOTE: may set bits on deleted docs + // NOTE: may set bits on deleted docs: + res.set(0, reader.maxDoc()); + // we must filter at the end, because we set bits, which may be deleted: + enforceAcceptDocs = true; } - final DocIdSetIterator disi = getDISI(fc.getFilter(), context); + final DocIdSetIterator disi = getDISI(fc.getFilter(), context, acceptDocs); if (disi != null) { res.andNot(disi); } @@ -85,7 +107,7 @@ for (final FilterClause fc : clauses) { if (fc.getOccur() == Occur.MUST) { - final DocIdSetIterator disi = getDISI(fc.getFilter(), context); + final DocIdSetIterator disi = getDISI(fc.getFilter(), context, (res != null) ? res : acceptDocs); if (disi == null) { return DocIdSet.EMPTY_DOCIDSET; // no documents can match } @@ -98,13 +120,16 @@ } } - return res != null ? BitsFilteredDocIdSet.wrap(res, acceptDocs) : DocIdSet.EMPTY_DOCIDSET; + if (res != null) { + return enforceAcceptDocs ? BitsFilteredDocIdSet.wrap(res, acceptDocs) : res; + } else { + return DocIdSet.EMPTY_DOCIDSET; + } } - private static DocIdSetIterator getDISI(Filter filter, AtomicReaderContext context) + private DocIdSetIterator getDISI(Filter filter, AtomicReaderContext context, Bits acceptDocs) throws IOException { - // we dont pass acceptDocs, we will filter at the end using an additional filter - final DocIdSet set = filter.getDocIdSet(context, null); + final DocIdSet set = filter.getDocIdSet(context, passAcceptDocs ? acceptDocs : null); return (set == null || set == DocIdSet.EMPTY_DOCIDSET) ? null : set.iterator(); } @@ -146,12 +171,12 @@ } final BooleanFilter other = (BooleanFilter)obj; - return clauses.equals(other.clauses); + return (passAcceptDocs == other.passAcceptDocs) && clauses.equals(other.clauses); } @Override public int hashCode() { - return 657153718 ^ clauses.hashCode(); + return (passAcceptDocs ? 4532423 : 657153718) ^ clauses.hashCode(); } /** Prints a user-readable version of this Filter. */ Index: lucene/queries/src/test/org/apache/lucene/queries/BooleanFilter2Test.java =================================================================== --- lucene/queries/src/test/org/apache/lucene/queries/BooleanFilter2Test.java (revision 0) +++ lucene/queries/src/test/org/apache/lucene/queries/BooleanFilter2Test.java (working copy) @@ -0,0 +1,27 @@ +package org.apache.lucene.queries; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class BooleanFilter2Test extends BooleanFilterTest { + + @Override + protected BooleanFilter newBooleanFilter() { + return new BooleanFilter(true); + } + +} Index: lucene/queries/src/test/org/apache/lucene/queries/BooleanFilter2Test.java =================================================================== --- lucene/queries/src/test/org/apache/lucene/queries/BooleanFilter2Test.java (revision 0) +++ lucene/queries/src/test/org/apache/lucene/queries/BooleanFilter2Test.java (working copy) Property changes on: lucene/queries/src/test/org/apache/lucene/queries/BooleanFilter2Test.java ___________________________________________________________________ Added: svn:keywords ## -0,0 +1 ## +Date Author Id Revision HeadURL \ No newline at end of property Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java =================================================================== --- lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java (revision 1407932) +++ lucene/queries/src/test/org/apache/lucene/queries/BooleanFilterTest.java (working copy) @@ -43,6 +43,10 @@ public class BooleanFilterTest extends LuceneTestCase { private Directory directory; private AtomicReader reader; + + protected BooleanFilter newBooleanFilter() { + return new BooleanFilter(); + } @Override public void setUp() throws Exception { @@ -139,25 +143,25 @@ public void testShould() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getTermsFilter("price", "030"), Occur.SHOULD); tstFilterCard("Should retrieves only 1 doc", 1, booleanFilter); // same with a real DISI (no OpenBitSetIterator) - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getWrappedTermQuery("price", "030"), Occur.SHOULD); tstFilterCard("Should retrieves only 1 doc", 1, booleanFilter); } public void testShoulds() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD); booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD); tstFilterCard("Shoulds are Ored together", 5, booleanFilter); } public void testShouldsAndMustNot() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD); booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD); booleanFilter.add(getTermsFilter("inStock", "N"), Occur.MUST_NOT); @@ -167,7 +171,7 @@ tstFilterCard("Shoulds Ored but AndNots", 3, booleanFilter); // same with a real DISI (no OpenBitSetIterator) - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD); booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD); booleanFilter.add(getWrappedTermQuery("inStock", "N"), Occur.MUST_NOT); @@ -178,14 +182,14 @@ } public void testShouldsAndMust() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD); booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD); booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST); tstFilterCard("Shoulds Ored but MUST", 3, booleanFilter); // same with a real DISI (no OpenBitSetIterator) - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD); booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD); booleanFilter.add(getWrappedTermQuery("accessRights", "admin"), Occur.MUST); @@ -193,7 +197,7 @@ } public void testShouldsAndMusts() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getRangeFilter("price", "010", "020"), Occur.SHOULD); booleanFilter.add(getRangeFilter("price", "020", "030"), Occur.SHOULD); booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST); @@ -202,7 +206,7 @@ } public void testShouldsAndMustsAndMustNot() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getRangeFilter("price", "030", "040"), Occur.SHOULD); booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST); booleanFilter.add(getRangeFilter("date", "20050101", "20051231"), Occur.MUST); @@ -210,7 +214,7 @@ tstFilterCard("Shoulds Ored but MUSTs ANDED and MustNot", 0, booleanFilter); // same with a real DISI (no OpenBitSetIterator) - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getRangeFilter("price", "030", "040"), Occur.SHOULD); booleanFilter.add(getWrappedTermQuery("accessRights", "admin"), Occur.MUST); booleanFilter.add(getRangeFilter("date", "20050101", "20051231"), Occur.MUST); @@ -219,115 +223,115 @@ } public void testJustMust() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST); tstFilterCard("MUST", 3, booleanFilter); // same with a real DISI (no OpenBitSetIterator) - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getWrappedTermQuery("accessRights", "admin"), Occur.MUST); tstFilterCard("MUST", 3, booleanFilter); } public void testJustMustNot() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getTermsFilter("inStock", "N"), Occur.MUST_NOT); tstFilterCard("MUST_NOT", 4, booleanFilter); // same with a real DISI (no OpenBitSetIterator) - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getWrappedTermQuery("inStock", "N"), Occur.MUST_NOT); tstFilterCard("MUST_NOT", 4, booleanFilter); } public void testMustAndMustNot() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getTermsFilter("inStock", "N"), Occur.MUST); booleanFilter.add(getTermsFilter("price", "030"), Occur.MUST_NOT); tstFilterCard("MUST_NOT wins over MUST for same docs", 0, booleanFilter); // same with a real DISI (no OpenBitSetIterator) - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getWrappedTermQuery("inStock", "N"), Occur.MUST); booleanFilter.add(getWrappedTermQuery("price", "030"), Occur.MUST_NOT); tstFilterCard("MUST_NOT wins over MUST for same docs", 0, booleanFilter); } public void testEmpty() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); tstFilterCard("empty BooleanFilter returns no results", 0, booleanFilter); } public void testCombinedNullDocIdSets() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getTermsFilter("price", "030"), Occur.MUST); booleanFilter.add(getNullDISFilter(), Occur.MUST); tstFilterCard("A MUST filter that returns a null DIS should never return documents", 0, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getTermsFilter("price", "030"), Occur.MUST); booleanFilter.add(getNullDISIFilter(), Occur.MUST); tstFilterCard("A MUST filter that returns a null DISI should never return documents", 0, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getTermsFilter("price", "030"), Occur.SHOULD); booleanFilter.add(getNullDISFilter(), Occur.SHOULD); tstFilterCard("A SHOULD filter that returns a null DIS should be invisible", 1, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getTermsFilter("price", "030"), Occur.SHOULD); booleanFilter.add(getNullDISIFilter(), Occur.SHOULD); tstFilterCard("A SHOULD filter that returns a null DISI should be invisible", 1, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getTermsFilter("price", "030"), Occur.MUST); booleanFilter.add(getNullDISFilter(), Occur.MUST_NOT); tstFilterCard("A MUST_NOT filter that returns a null DIS should be invisible", 1, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getTermsFilter("price", "030"), Occur.MUST); booleanFilter.add(getNullDISIFilter(), Occur.MUST_NOT); tstFilterCard("A MUST_NOT filter that returns a null DISI should be invisible", 1, booleanFilter); } public void testJustNullDocIdSets() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getNullDISFilter(), Occur.MUST); tstFilterCard("A MUST filter that returns a null DIS should never return documents", 0, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getNullDISIFilter(), Occur.MUST); tstFilterCard("A MUST filter that returns a null DISI should never return documents", 0, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getNullDISFilter(), Occur.SHOULD); tstFilterCard("A single SHOULD filter that returns a null DIS should never return documents", 0, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getNullDISIFilter(), Occur.SHOULD); tstFilterCard("A single SHOULD filter that returns a null DISI should never return documents", 0, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getNullDISFilter(), Occur.MUST_NOT); tstFilterCard("A single MUST_NOT filter that returns a null DIS should be invisible", 5, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getNullDISIFilter(), Occur.MUST_NOT); tstFilterCard("A single MUST_NOT filter that returns a null DIS should be invisible", 5, booleanFilter); } public void testNonMatchingShouldsAndMusts() throws Exception { - BooleanFilter booleanFilter = new BooleanFilter(); + BooleanFilter booleanFilter = newBooleanFilter(); booleanFilter.add(getEmptyFilter(), Occur.SHOULD); booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST); tstFilterCard(">0 shoulds with no matches should return no docs", 0, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getNullDISFilter(), Occur.SHOULD); booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST); tstFilterCard(">0 shoulds with no matches should return no docs", 0, booleanFilter); - booleanFilter = new BooleanFilter(); + booleanFilter = newBooleanFilter(); booleanFilter.add(getNullDISIFilter(), Occur.SHOULD); booleanFilter.add(getTermsFilter("accessRights", "admin"), Occur.MUST); tstFilterCard(">0 shoulds with no matches should return no docs", 0, booleanFilter);