Index: src/java/org/apache/lucene/util/OpenBitSetDISI.java =================================================================== --- src/java/org/apache/lucene/util/OpenBitSetDISI.java (revision 0) +++ src/java/org/apache/lucene/util/OpenBitSetDISI.java (revision 0) @@ -0,0 +1,56 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import org.apache.lucene.search.DocIdSetIterator; + +public class OpenBitSetDISI extends OpenBitSet { + public OpenBitSetDISI(DocIdSetIterator disi) throws IOException { + inPlaceOr(disi); + } + + public void inPlaceOr(DocIdSetIterator disi) throws IOException { + while (disi.next()) { + set(disi.doc()); + } + } + + public void inPlaceAnd(DocIdSetIterator disi) throws IOException { + int index = nextSetBit(0); + int lastNotCleared = -1; + while ((index != -1) && disi.skipTo(index)) { + while ((index != -1) && (index < disi.doc())) { + fastClear(index); + index = nextSetBit(index + 1); + } + if (index == disi.doc()) { + lastNotCleared = index; + index++; + } + assert (index == -1) || (index > disi.doc()); + } + clear(lastNotCleared+1, size()); + } + + public void inPlaceNot(DocIdSetIterator disi) throws IOException { + while (disi.next() && (disi.doc() < size())) { + fastClear(disi.doc()); + } + } +} Index: contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java =================================================================== --- contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java (revision 638957) +++ contrib/queries/src/test/org/apache/lucene/search/BooleanFilterTest.java (working copy) @@ -76,13 +76,24 @@ tf.addTerm(new Term(field,text)); return tf; } + + private void tstFilterCard(String mes, int expected, Filter filt) + throws Throwable + { + DocIdSetIterator disi = filt.getDocIdSet(reader).iterator(); + int actual = 0; + while (disi.next()) { + actual++; + } + assertEquals(mes, expected, actual); + } + public void testShould() throws Throwable { BooleanFilter booleanFilter = new BooleanFilter(); booleanFilter.add(new FilterClause(getTermsFilter("price","030"),BooleanClause.Occur.SHOULD)); - BitSet bits = booleanFilter.bits(reader); - assertEquals("Should retrieves only 1 doc",1,bits.cardinality()); + tstFilterCard("Should retrieves only 1 doc",1,booleanFilter); } public void testShoulds() throws Throwable @@ -90,8 +101,7 @@ BooleanFilter booleanFilter = new BooleanFilter(); booleanFilter.add(new FilterClause(getRangeFilter("price","010", "020"),BooleanClause.Occur.SHOULD)); booleanFilter.add(new FilterClause(getRangeFilter("price","020", "030"),BooleanClause.Occur.SHOULD)); - BitSet bits = booleanFilter.bits(reader); - assertEquals("Shoulds are Ored together",5,bits.cardinality()); + tstFilterCard("Shoulds are Ored together",5,booleanFilter); } public void testShouldsAndMustNot() throws Throwable { @@ -99,12 +109,10 @@ booleanFilter.add(new FilterClause(getRangeFilter("price","010", "020"),BooleanClause.Occur.SHOULD)); booleanFilter.add(new FilterClause(getRangeFilter("price","020", "030"),BooleanClause.Occur.SHOULD)); booleanFilter.add(new FilterClause(getTermsFilter("inStock", "N"),BooleanClause.Occur.MUST_NOT)); - BitSet bits = booleanFilter.bits(reader); - assertEquals("Shoulds Ored but AndNot",4,bits.cardinality()); + tstFilterCard("Shoulds Ored but AndNot",4,booleanFilter); booleanFilter.add(new FilterClause(getTermsFilter("inStock", "Maybe"),BooleanClause.Occur.MUST_NOT)); - bits = booleanFilter.bits(reader); - assertEquals("Shoulds Ored but AndNots",3,bits.cardinality()); + tstFilterCard("Shoulds Ored but AndNots",3,booleanFilter); } public void testShouldsAndMust() throws Throwable @@ -113,8 +121,7 @@ booleanFilter.add(new FilterClause(getRangeFilter("price","010", "020"),BooleanClause.Occur.SHOULD)); booleanFilter.add(new FilterClause(getRangeFilter("price","020", "030"),BooleanClause.Occur.SHOULD)); booleanFilter.add(new FilterClause(getTermsFilter("accessRights", "admin"),BooleanClause.Occur.MUST)); - BitSet bits = booleanFilter.bits(reader); - assertEquals("Shoulds Ored but MUST",3,bits.cardinality()); + tstFilterCard("Shoulds Ored but MUST",3,booleanFilter); } public void testShouldsAndMusts() throws Throwable { @@ -123,8 +130,7 @@ booleanFilter.add(new FilterClause(getRangeFilter("price","020", "030"),BooleanClause.Occur.SHOULD)); booleanFilter.add(new FilterClause(getTermsFilter("accessRights", "admin"),BooleanClause.Occur.MUST)); booleanFilter.add(new FilterClause(getRangeFilter("date","20040101", "20041231"),BooleanClause.Occur.MUST)); - BitSet bits = booleanFilter.bits(reader); - assertEquals("Shoulds Ored but MUSTs ANDED",1,bits.cardinality()); + tstFilterCard("Shoulds Ored but MUSTs ANDED",1,booleanFilter); } public void testShouldsAndMustsAndMustNot() throws Throwable { @@ -133,33 +139,27 @@ booleanFilter.add(new FilterClause(getTermsFilter("accessRights", "admin"),BooleanClause.Occur.MUST)); booleanFilter.add(new FilterClause(getRangeFilter("date","20050101", "20051231"),BooleanClause.Occur.MUST)); booleanFilter.add(new FilterClause(getTermsFilter("inStock","N"),BooleanClause.Occur.MUST_NOT)); - BitSet bits = booleanFilter.bits(reader); - assertEquals("Shoulds Ored but MUSTs ANDED and MustNot",0,bits.cardinality()); + tstFilterCard("Shoulds Ored but MUSTs ANDED and MustNot",0,booleanFilter); } public void testJustMust() throws Throwable { BooleanFilter booleanFilter = new BooleanFilter(); booleanFilter.add(new FilterClause(getTermsFilter("accessRights", "admin"),BooleanClause.Occur.MUST)); - BitSet bits = booleanFilter.bits(reader); - assertEquals("MUST",3,bits.cardinality()); + tstFilterCard("MUST",3,booleanFilter); } public void testJustMustNot() throws Throwable { BooleanFilter booleanFilter = new BooleanFilter(); booleanFilter.add(new FilterClause(getTermsFilter("inStock","N"),BooleanClause.Occur.MUST_NOT)); - BitSet bits = booleanFilter.bits(reader); - assertEquals("MUST_NOT",4,bits.cardinality()); + tstFilterCard("MUST_NOT",4,booleanFilter); } public void testMustAndMustNot() throws Throwable { BooleanFilter booleanFilter = new BooleanFilter(); booleanFilter.add(new FilterClause(getTermsFilter("inStock","N"),BooleanClause.Occur.MUST)); booleanFilter.add(new FilterClause(getTermsFilter("price","030"),BooleanClause.Occur.MUST_NOT)); - BitSet bits = booleanFilter.bits(reader); - assertEquals("MUST_NOT wins over MUST for same docs",0,bits.cardinality()); + tstFilterCard("MUST_NOT wins over MUST for same docs",0,booleanFilter); } - - } Index: contrib/queries/src/java/org/apache/lucene/search/BooleanFilter.java =================================================================== --- contrib/queries/src/java/org/apache/lucene/search/BooleanFilter.java (revision 638957) +++ contrib/queries/src/java/org/apache/lucene/search/BooleanFilter.java (working copy) @@ -19,10 +19,10 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.BitSet; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.util.OpenBitSetDISI; /** * A container Filter that allows Boolean composition of Filters. @@ -37,184 +37,151 @@ public class BooleanFilter extends Filter { - //ArrayList of SHOULD filters - ArrayList shouldFilters = null; - //ArrayList of NOT filters - ArrayList notFilters = null; - //ArrayList of MUST filters - ArrayList mustFilters = null; + ArrayList shouldFilters = null; + ArrayList notFilters = null; + ArrayList mustFilters = null; + + private DocIdSetIterator getDISI(ArrayList filters, int index, IndexReader reader) + throws IOException + { + return ((Filter)filters.get(index)).getDocIdSet(reader).iterator(); + } - /** - * Returns the a BitSet representing the Boolean composition - * of the filters that have been added. - */ - - public BitSet bits(IndexReader reader) throws IOException - { - //create a new bitSet - BitSet returnBits = null; - - //SHOULD filters - if (shouldFilters!=null) - { - returnBits = ((Filter)shouldFilters.get(0)).bits(reader); -// avoid changing the original bitset - it may be cached - returnBits=(BitSet) returnBits.clone(); - if (shouldFilters.size() > 1) - { - for (int i = 1; i < shouldFilters.size(); i++) - { - returnBits.or(((Filter)shouldFilters.get(i)).bits(reader)); - } - } - } - - //NOT filters - if (notFilters!=null) - { - for (int i = 0; i < notFilters.size(); i++) - { - BitSet notBits=((Filter)notFilters.get(i)).bits(reader); - if(returnBits==null) - { - returnBits=(BitSet) notBits.clone(); - returnBits.flip(0,reader.maxDoc()); - } - else - { - returnBits.andNot(notBits); - } - } - } - - //MUST filters - if (mustFilters!=null) - { - for (int i = 0; i < mustFilters.size(); i++) - { - BitSet mustBits=((Filter)mustFilters.get(i)).bits(reader); - if(returnBits==null) - { - if(mustFilters.size()==1) - { - returnBits=mustBits; - - } - else - { - //don't mangle the bitset - returnBits=(BitSet) mustBits.clone(); - } - } - else - { - returnBits.and(mustBits); - } - } - } - if(returnBits==null) - { - returnBits=new BitSet(reader.maxDoc()); - } - return returnBits; - } - - /** - * Adds a new FilterClause to the Boolean Filter container - * @param filterClause A FilterClause object containing a Filter and an Occur parameter - */ - - public void add(FilterClause filterClause) - { - if (filterClause.getOccur().equals(Occur.MUST)) - { - if(mustFilters==null) - { - mustFilters=new ArrayList(); - } - mustFilters.add(filterClause.getFilter()); - } - if (filterClause.getOccur().equals(Occur.SHOULD)) - { - if(shouldFilters==null) - { - shouldFilters=new ArrayList(); - } - shouldFilters.add(filterClause.getFilter()); - } - if (filterClause.getOccur().equals(Occur.MUST_NOT)) - { - if(notFilters==null) - { - notFilters=new ArrayList(); - } - notFilters.add(filterClause.getFilter()); - } - } + /** + * Returns the a DocIdSetIterator representing the Boolean composition + * of the filters that have been added. + */ + public DocIdSet getDocIdSet(IndexReader reader) throws IOException + { + OpenBitSetDISI res = null; + + if (shouldFilters != null) { + for (int i = 0; i < shouldFilters.size(); i++) { + if (res == null) { + res = new OpenBitSetDISI(getDISI(shouldFilters, i, reader)); + } else { + res.inPlaceOr(getDISI(shouldFilters, i, reader)); + } + } + } + + if (notFilters!=null) { + for (int i = 0; i < notFilters.size(); i++) { + if (res == null) { + res = new OpenBitSetDISI(getDISI(notFilters, i, reader)); + res.flip(0, reader.maxDoc()); // NOTE: may set bits on deleted docs + } else { + res.inPlaceNot(getDISI(notFilters, i, reader)); + } + } + } + + if (mustFilters!=null) { + for (int i = 0; i < mustFilters.size(); i++) { + if (res == null) { + res = new OpenBitSetDISI(getDISI(mustFilters, i, reader)); + } else { + res.inPlaceAnd(getDISI(mustFilters, i, reader)); + } + } + } + + if (res !=null) + return res; - public boolean equals(Object obj) - { - if(this == obj) - return true; - if((obj == null) || (obj.getClass() != this.getClass())) - return false; - BooleanFilter test = (BooleanFilter)obj; - return (notFilters == test.notFilters|| - (notFilters!= null && notFilters.equals(test.notFilters))) - && - (mustFilters == test.mustFilters|| - (mustFilters!= null && mustFilters.equals(test.mustFilters))) - && - (shouldFilters == test.shouldFilters|| - (shouldFilters!= null && shouldFilters.equals(test.shouldFilters))); - } + if (emptyDocIdSet == null) { // test and assignment of static not thread safe + emptyDocIdSet = new DocIdSet() { + public DocIdSetIterator iterator() { + return new DocIdSetIterator() { + public boolean next() { return false;} + public boolean skipTo(int target) { return false;} + public int doc() { throw new UnsupportedOperationException();} + }; + } + }; + } + return emptyDocIdSet; + } - public int hashCode() - { - int hash=7; - hash = 31 * hash + (null == mustFilters ? 0 : mustFilters.hashCode()); - hash = 31 * hash + (null == notFilters ? 0 : notFilters.hashCode()); - hash = 31 * hash + (null == shouldFilters ? 0 : shouldFilters.hashCode()); - return hash; - } - - - /** Prints a user-readable version of this query. */ - public String toString() - { - StringBuffer buffer = new StringBuffer(); + private static DocIdSet emptyDocIdSet = null; - buffer.append("BooleanFilter("); + /** + * Adds a new FilterClause to the Boolean Filter container + * @param filterClause A FilterClause object containing a Filter and an Occur parameter + */ + + public void add(FilterClause filterClause) + { + if (filterClause.getOccur().equals(Occur.MUST)) { + if(mustFilters==null) { + mustFilters=new ArrayList(); + } + mustFilters.add(filterClause.getFilter()); + } + if (filterClause.getOccur().equals(Occur.SHOULD)) { + if(shouldFilters==null) { + shouldFilters=new ArrayList(); + } + shouldFilters.add(filterClause.getFilter()); + } + if (filterClause.getOccur().equals(Occur.MUST_NOT)) { + if(notFilters==null) { + notFilters=new ArrayList(); + } + notFilters.add(filterClause.getFilter()); + } + } - appendFilters(shouldFilters, null, buffer); - appendFilters(mustFilters, "+", buffer); - appendFilters(notFilters, "-", buffer); + private boolean equalFilters(ArrayList filters1, ArrayList filters2) + { + return (filters1 == filters2) || + ((filters1 != null) && filters1.equals(filters2)); + } + + public boolean equals(Object obj) + { + if(this == obj) + return true; - buffer.append(")"); + if((obj == null) || (obj.getClass() != this.getClass())) + return false; - return buffer.toString(); - } + BooleanFilter other = (BooleanFilter)obj; + return equalFilters(notFilters, other.notFilters) + && equalFilters(mustFilters, other.mustFilters) + && equalFilters(shouldFilters, other.shouldFilters); + } + + public int hashCode() + { + int hash=7; + hash = 31 * hash + (null == mustFilters ? 0 : mustFilters.hashCode()); + hash = 31 * hash + (null == notFilters ? 0 : notFilters.hashCode()); + hash = 31 * hash + (null == shouldFilters ? 0 : shouldFilters.hashCode()); + return hash; + } - private void appendFilters(ArrayList filters, String occurString, - StringBuffer buffer) - { - if (filters == null) - return; + /** Prints a user-readable version of this query. */ + public String toString() + { + StringBuffer buffer = new StringBuffer(); + buffer.append("BooleanFilter("); + appendFilters(shouldFilters, "", buffer); + appendFilters(mustFilters, "+", buffer); + appendFilters(notFilters, "-", buffer); + buffer.append(")"); + return buffer.toString(); + } + + private void appendFilters(ArrayList filters, String occurString, StringBuffer buffer) + { + if (filters != null) { + for (int i = 0; i < filters.size(); i++) { + buffer.append(' '); + buffer.append(occurString); + buffer.append(filters.get(i).toString()); + } + } + } - for (int i = 0; i < filters.size(); i++) - { - Filter filter = (Filter) filters.get(i); - if (occurString != null) - { - buffer.append(occurString); - } - - buffer.append(filter); - - if (i < filters.size() - 1) - { - buffer.append(' '); - } - } - } }