Index: src/java/org/apache/lucene/search/BooleanClause.java =================================================================== --- src/java/org/apache/lucene/search/BooleanClause.java (revision 679152) +++ src/java/org/apache/lucene/search/BooleanClause.java (working copy) @@ -19,7 +19,12 @@ * limitations under the License. */ -/** A clause in a BooleanQuery. */ +/** A clause in a BooleanQuery. + This is either a Query or a Filter. + A Query may occur as MUST, SHOULD or MUST_NOT. + A Query may occur as MUST or MUST_NOT. + */ +// FIXME: probably better to split off new class BooleanFilterClause from this. public class BooleanClause implements java.io.Serializable { /** Specifies how clauses are to occur in matching documents. */ @@ -52,9 +57,10 @@ } - /** The query whose matching documents are combined by the boolean query. + /** The query or filter whose matching documents are combined by the boolean query. */ private Query query; + private Filter filter; private Occur occur; @@ -62,28 +68,49 @@ /** Constructs a BooleanClause. */ public BooleanClause(Query query, Occur occur) { + assert query != null; this.query = query; this.occur = occur; - } + /** Constructs a BooleanClause. + */ + public BooleanClause(Filter filter, Occur occur) { + assert filter != null; + if (occur == Occur.SHOULD) { + throw new IllegalArgumentException("Filter can only occur as MUST or MUST_NOT"); + } + this.filter = filter; + this.occur = occur; + } + public Occur getOccur() { return occur; } public void setOccur(Occur occur) { + assert (filter == null) || (occur != Occur.SHOULD); this.occur = occur; - } public Query getQuery() { return query; } + public Filter getFilter() { + return filter; + } + public void setQuery(Query query) { + assert filter == null; this.query = query; } + public void setFilter(Filter filter) { + assert query == null; + this.filter = filter; + } + public boolean isProhibited() { return Occur.MUST_NOT.equals(occur); } @@ -99,17 +126,20 @@ if (!(o instanceof BooleanClause)) return false; BooleanClause other = (BooleanClause)o; - return this.query.equals(other.query) - && this.occur.equals(other.occur); + return this.occur.equals(other.occur) + && ((query != null) + ? this.query.equals(other.query) + : this.filter.equals(other.filter)); } /** Returns a hash code value for this object.*/ public int hashCode() { - return query.hashCode() ^ (Occur.MUST.equals(occur)?1:0) ^ (Occur.MUST_NOT.equals(occur)?2:0); + return ((query != null) ? query.hashCode() : filter.hashCode()) + ^ (Occur.MUST.equals(occur)?1:0) ^ (Occur.MUST_NOT.equals(occur)?2:0); } public String toString() { - return occur.toString() + query.toString(); + return occur.toString() + ((query != null) ? query.toString() : filter.toString()); } } Index: src/java/org/apache/lucene/search/BooleanQuery.java =================================================================== --- src/java/org/apache/lucene/search/BooleanQuery.java (revision 679152) +++ src/java/org/apache/lucene/search/BooleanQuery.java (working copy) @@ -141,13 +141,13 @@ /** * Gets the minimum number of the optional BooleanClauses - * which must be satisifed. + * which must be satisfied. */ public int getMinimumNumberShouldMatch() { return minNrShouldMatch; } - /** Adds a clause to a boolean query. + /** Adds a query clause to a boolean query. * * @throws TooManyClauses if the new number of clauses exceeds the maximum clause number * @see #getMaxClauseCount() @@ -156,6 +156,16 @@ add(new BooleanClause(query, occur)); } + /** Adds a filter clause to a boolean query. + * The filter can only occur as #Boolean.Occur.MUST or #BooleanClause.Occur.MUST_NOT . + * + * @throws TooManyClauses if the new number of clauses exceeds the maximum clause number + * @see #getMaxClauseCount() + */ + public void add(Filter filter, BooleanClause.Occur occur) { + add(new BooleanClause(filter, occur)); + } + /** Adds a clause to a boolean query. * @throws TooManyClauses if the new number of clauses exceeds the maximum clause number * @see #getMaxClauseCount() @@ -177,14 +187,16 @@ private class BooleanWeight implements Weight { protected Similarity similarity; - protected Vector weights = new Vector(); + protected Vector weights = new Vector(); // only for the query clauses, not for the filter clauses. public BooleanWeight(Searcher searcher) throws IOException { this.similarity = getSimilarity(searcher); for (int i = 0 ; i < clauses.size(); i++) { BooleanClause c = (BooleanClause)clauses.get(i); - weights.add(c.getQuery().createWeight(searcher)); + if (c.getQuery() != null) { // ignore filter clauses + weights.add(c.getQuery().createWeight(searcher)); + } } } @@ -193,14 +205,17 @@ public float sumOfSquaredWeights() throws IOException { float sum = 0.0f; + int wi = 0; for (int i = 0 ; i < weights.size(); i++) { BooleanClause c = (BooleanClause)clauses.get(i); - Weight w = (Weight)weights.elementAt(i); - // call sumOfSquaredWeights for all clauses in case of side effects - float s = w.sumOfSquaredWeights(); // sum sub weights - if (!c.isProhibited()) - // only add to sum for non-prohibited clauses - sum += s; + if (c.getQuery() != null) { // ignore filter clauses + Weight w = (Weight)weights.elementAt(wi++); + // call sumOfSquaredWeights for all clauses in case of side effects + float s = w.sumOfSquaredWeights(); // sum sub weights + if (!c.isProhibited()) { // only add to sum for non-prohibited query clauses + sum += s; + } + } } sum *= getBoost() * getBoost(); // boost each sub-weight @@ -213,7 +228,7 @@ norm *= getBoost(); // incorporate boost for (int i = 0 ; i < weights.size(); i++) { Weight w = (Weight)weights.elementAt(i); - // normalize all clauses, (even if prohibited in case of side affects) + // normalize all query clauses, (even if prohibited in case of side affects) w.normalize(norm); } } @@ -228,12 +243,17 @@ for (int i = 0 ; i < weights.size(); i++) { BooleanClause c = (BooleanClause)clauses.get(i); - Weight w = (Weight)weights.elementAt(i); - Scorer subScorer = w.scorer(reader); - if (subScorer != null) - result.add(subScorer, c.isRequired(), c.isProhibited()); - else if (c.isRequired()) - return null; + if (c.getQuery() != null) { // query clause + Weight w = (Weight)weights.elementAt(i); + Scorer subScorer = w.scorer(reader); + if (subScorer != null) + result.add(subScorer, c.isRequired(), c.isProhibited()); + else if (c.isRequired()) + return null; + } else { // filter clause + DocIdSetIterator disi = c.getFilter().getDocIdSet(reader).iterator(); + result.add(disi, c.isRequired()); + } } return result; @@ -250,30 +270,55 @@ float sum = 0.0f; boolean fail = false; int shouldMatchCount = 0; + int wi = 0; for (int i = 0 ; i < weights.size(); i++) { BooleanClause c = (BooleanClause)clauses.get(i); - Weight w = (Weight)weights.elementAt(i); - Explanation e = w.explain(reader, doc); - if (!c.isProhibited()) maxCoord++; - if (e.isMatch()) { + if (c.getQuery() != null) { // explain query clause + Weight w = (Weight)weights.elementAt(wi++); + Explanation e = w.explain(reader, doc); if (!c.isProhibited()) { - sumExpl.addDetail(e); - sum += e.getValue(); - coord++; - } else { - Explanation r = - new Explanation(0.0f, "match on prohibited clause (" + c.getQuery().toString() + ")"); + maxCoord++; + } + if (e.isMatch()) { + if (!c.isProhibited()) { + sumExpl.addDetail(e); + sum += e.getValue(); + coord++; + } else { + Explanation r = + new Explanation(0.0f, "match on prohibited clause (" + c.getQuery().toString() + ")"); + r.addDetail(e); + sumExpl.addDetail(r); + fail = true; + } + if (c.getOccur().equals(Occur.SHOULD)) + shouldMatchCount++; + } else if (c.isRequired()) { + Explanation r = new Explanation(0.0f, "no match on required clause (" + c.getQuery().toString() + ")"); r.addDetail(e); sumExpl.addDetail(r); fail = true; } - if (c.getOccur().equals(Occur.SHOULD)) - shouldMatchCount++; - } else if (c.isRequired()) { - Explanation r = new Explanation(0.0f, "no match on required clause (" + c.getQuery().toString() + ")"); - r.addDetail(e); + } else { // explain all cases for filter clause, filter clause does not count for coordination. + Filter f = c.getFilter(); + DocIdSetIterator disi = f.getDocIdSet(reader).iterator(); + boolean matches = disi.skipTo(doc) && (disi.doc() == doc); + Explanation r; + if (matches) { + if (c.isRequired()) { + r = new Explanation(0.0f, "match on required filter clause (" + f.toString() + ")"); + } else { + r = new Explanation(0.0f, "match on prohibited filter clause (" + f.toString() + ")"); + } + } else { // no match + if (c.isRequired()) { + r = new Explanation(0.0f, "no match on required filter clause (" + f.toString() + ")"); + } else { + r = new Explanation(0.0f, "no match on prohibited filter clause (" + f.toString() + ")"); + } + } + fail = (matches != c.isRequired()); sumExpl.addDetail(r); - fail = true; } } if (fail) { @@ -365,8 +410,7 @@ public Query rewrite(IndexReader reader) throws IOException { if (minNrShouldMatch == 0 && clauses.size() == 1) { // optimize 1-clause queries BooleanClause c = (BooleanClause)clauses.get(0); - if (!c.isProhibited()) { // just return clause - + if ((c.getQuery() != null) && !c.isProhibited()) { // just return clause Query query = c.getQuery().rewrite(reader); // rewrite first if (getBoost() != 1.0f) { // incorporate boost @@ -382,11 +426,14 @@ BooleanQuery clone = null; // recursively rewrite for (int i = 0 ; i < clauses.size(); i++) { BooleanClause c = (BooleanClause)clauses.get(i); - Query query = c.getQuery().rewrite(reader); - if (query != c.getQuery()) { // clause rewrote: must clone - if (clone == null) - clone = (BooleanQuery)this.clone(); - clone.clauses.set(i, new BooleanClause(query, c.getOccur())); + if (c.getQuery() != null) { + Query query = c.getQuery().rewrite(reader); + if (query != c.getQuery()) { // clause rewrote: must clone + if (clone == null) { + clone = (BooleanQuery)this.clone(); + } + clone.clauses.set(i, new BooleanClause(query, c.getOccur())); + } } } if (clone != null) { @@ -397,10 +444,12 @@ // inherit javadoc public void extractTerms(Set terms) { - for (Iterator i = clauses.iterator(); i.hasNext();) { - BooleanClause clause = (BooleanClause) i.next(); - clause.getQuery().extractTerms(terms); - } + for (Iterator i = clauses.iterator(); i.hasNext();) { + BooleanClause clause = (BooleanClause) i.next(); + if (clause.getQuery() != null) { + clause.getQuery().extractTerms(terms); + } + } } public Object clone() { @@ -424,13 +473,18 @@ else if (c.isRequired()) buffer.append("+"); - Query subQuery = c.getQuery(); - if (subQuery instanceof BooleanQuery) { // wrap sub-bools in parens - buffer.append("("); - buffer.append(c.getQuery().toString(field)); - buffer.append(")"); - } else - buffer.append(c.getQuery().toString(field)); + if (c.getQuery() != null) { + Query subQuery = c.getQuery(); + if (subQuery instanceof BooleanQuery) { // wrap sub-bools in parens + buffer.append("("); + buffer.append(c.getQuery().toString(field)); + buffer.append(")"); + } else { + buffer.append(c.getQuery().toString(field)); + } + } else { // filter clause + buffer.append(c.getQuery().toString()); + } if (i != clauses.size()-1) buffer.append(" "); Index: src/java/org/apache/lucene/search/BooleanScorer2.java =================================================================== --- src/java/org/apache/lucene/search/BooleanScorer2.java (revision 679152) +++ src/java/org/apache/lucene/search/BooleanScorer2.java (working copy) @@ -24,13 +24,16 @@ /** An alternative to BooleanScorer that also allows a minimum number * of optional scorers that should match. - *
Implements skipTo(), and has no limitations on the numbers of added scorers. + * Also supports required and prohibited DocIdSetIterators. + *
Implements skipTo(), and has no limitations on the numbers + * of added Scorers and DocIdSetIterators. *
Uses ConjunctionScorer, DisjunctionScorer, ReqOptScorer and ReqExclScorer. */ class BooleanScorer2 extends Scorer { private ArrayList requiredScorers = new ArrayList(); + private ArrayList requiredDisis = new ArrayList(); private ArrayList optionalScorers = new ArrayList(); - private ArrayList prohibitedScorers = new ArrayList(); + private ArrayList prohibitedDisis = new ArrayList(); private class Coordinator { int maxCoord = 0; // to be increased for each non prohibited scorer @@ -126,12 +129,20 @@ } requiredScorers.add(scorer); } else if (prohibited) { - prohibitedScorers.add(scorer); + prohibitedDisis.add(scorer); // Scorer subclasses DocIdSetIterator } else { optionalScorers.add(scorer); } } + public void add(final DocIdSetIterator disi, boolean required) { + if (required) { + requiredDisis.add(disi); + } else { + prohibitedDisis.add(disi); + } + } + /** Initialize the match counting scorer that sums all the * scores.

* When "counting" is used in a name it means counting the number @@ -149,7 +160,7 @@ private Scorer scorer; private int lastScoredDoc = -1; - SingleMatchScorer(Scorer scorer) { + SingleMatchScorer(Scorer scorer, List requiredDisis) { // FIXME: use 2nd arg. super(scorer.getSimilarity()); this.scorer = scorer; } @@ -175,7 +186,8 @@ } private Scorer countingDisjunctionSumScorer(final List scorers, - int minNrShouldMatch) + int minNrShouldMatch, + List requiredDisis) // FIXME: use this arg. // each scorer from the list counted as a single matcher { return new DisjunctionSumScorer(scorers, minNrShouldMatch) { @@ -192,10 +204,12 @@ private static Similarity defaultSimilarity = new DefaultSimilarity(); - private Scorer countingConjunctionSumScorer(List requiredScorers) throws IOException { + private Scorer countingConjunctionSumScorer(List requiredScorers, + List requiredDisis) // FIXME: use this arg. + throws IOException { // each scorer from the list counted as a single matcher final int requiredNrMatchers = requiredScorers.size(); - return new ConjunctionScorer(defaultSimilarity, requiredScorers) { + return new ConjunctionScorer(defaultSimilarity, requiredScorers, requiredDisis) { private int lastScoredDoc = -1; public float score() throws IOException { @@ -221,7 +235,7 @@ } /** Returns the scorer to be used for match counting and score summing. - * Uses requiredScorers, optionalScorers and prohibitedScorers. + * Uses requiredScorers, optionalScorers and prohibitedScorers. Disis??? */ private Scorer makeCountingSumScorer() throws IOException { // each scorer counted as a single matcher return (requiredScorers.size() == 0) @@ -240,12 +254,12 @@ } else { // optionalScorers.size() >= nrOptRequired, no required scorers Scorer requiredCountingSumScorer = (optionalScorers.size() > nrOptRequired) - ? countingDisjunctionSumScorer(optionalScorers, nrOptRequired) + ? countingDisjunctionSumScorer(optionalScorers, nrOptRequired, requiredDisis) : // optionalScorers.size() == nrOptRequired (all optional scorers are required), no required scorers (optionalScorers.size() == 1) - ? new SingleMatchScorer((Scorer) optionalScorers.get(0)) - : countingConjunctionSumScorer(optionalScorers); - return addProhibitedScorers(requiredCountingSumScorer); + ? new SingleMatchScorer((Scorer) optionalScorers.get(0), requiredDisis) + : countingConjunctionSumScorer(optionalScorers, requiredDisis); + return addProhibitedDisis(requiredCountingSumScorer); } } } @@ -254,43 +268,44 @@ if (optionalScorers.size() < minNrShouldMatch) { return new NonMatchingScorer(); // fewer optional clauses than minimum that should match } else if (optionalScorers.size() == minNrShouldMatch) { // all optional scorers also required. - ArrayList allReq = new ArrayList(requiredScorers); - allReq.addAll(optionalScorers); - return addProhibitedScorers(countingConjunctionSumScorer(allReq)); + ArrayList allReqScorers = new ArrayList(requiredScorers); + allReqScorers.addAll(optionalScorers); + return addProhibitedDisis(countingConjunctionSumScorer(allReqScorers, requiredDisis)); } else { // optionalScorers.size() > minNrShouldMatch, and at least one required scorer Scorer requiredCountingSumScorer = (requiredScorers.size() == 1) - ? new SingleMatchScorer((Scorer) requiredScorers.get(0)) - : countingConjunctionSumScorer(requiredScorers); + ? new SingleMatchScorer((Scorer) requiredScorers.get(0), requiredDisis) + : countingConjunctionSumScorer(requiredScorers, requiredDisis); if (minNrShouldMatch > 0) { // use a required disjunction scorer over the optional scorers - return addProhibitedScorers( + return addProhibitedDisis( dualConjunctionSumScorer( // non counting requiredCountingSumScorer, countingDisjunctionSumScorer( optionalScorers, - minNrShouldMatch))); + minNrShouldMatch, + requiredDisis))); } else { // minNrShouldMatch == 0 return new ReqOptSumScorer( - addProhibitedScorers(requiredCountingSumScorer), + addProhibitedDisis(requiredCountingSumScorer), ((optionalScorers.size() == 1) - ? new SingleMatchScorer((Scorer) optionalScorers.get(0)) - : countingDisjunctionSumScorer(optionalScorers, 1))); // require 1 in combined, optional scorer. + ? new SingleMatchScorer((Scorer) optionalScorers.get(0), requiredDisis) + : countingDisjunctionSumScorer(optionalScorers, 1, requiredDisis))); // require 1 in combined, optional scorer. } } } /** Returns the scorer to be used for match counting and score summing. - * Uses the given required scorer and the prohibitedScorers. + * Uses the given required scorer and the prohibitedDisis. * @param requiredCountingSumScorer A required scorer already built. */ - private Scorer addProhibitedScorers(Scorer requiredCountingSumScorer) + private Scorer addProhibitedDisis(Scorer requiredCountingSumScorer) { - return (prohibitedScorers.size() == 0) - ? requiredCountingSumScorer // no prohibited + return (prohibitedDisis.size() == 0) + ? requiredCountingSumScorer : new ReqExclScorer(requiredCountingSumScorer, - ((prohibitedScorers.size() == 1) - ? (Scorer) prohibitedScorers.get(0) - : new DisjunctionSumScorer(prohibitedScorers))); + ((prohibitedDisis.size() == 1) + ? (DocIdSetIterator) prohibitedDisis.get(0) + : new DisjunctionDISI(prohibitedDisis))); } /** Scores and collects all matching documents. @@ -300,14 +315,14 @@ */ public void score(HitCollector hc) throws IOException { if (allowDocsOutOfOrder && requiredScorers.size() == 0 - && prohibitedScorers.size() < 32) { + && prohibitedDisis.size() < 32) { // fall back to BooleanScorer, scores documents somewhat out of order BooleanScorer bs = new BooleanScorer(getSimilarity(), minNrShouldMatch); Iterator si = optionalScorers.iterator(); while (si.hasNext()) { bs.add((Scorer) si.next(), false /* required */, false /* prohibited */); } - si = prohibitedScorers.iterator(); + si = prohibitedDisis.iterator(); while (si.hasNext()) { bs.add((Scorer) si.next(), false /* required */, true /* prohibited */); } Index: src/java/org/apache/lucene/search/ConjunctionScorer.java =================================================================== --- src/java/org/apache/lucene/search/ConjunctionScorer.java (revision 679152) +++ src/java/org/apache/lucene/search/ConjunctionScorer.java (working copy) @@ -19,12 +19,14 @@ import java.io.IOException; import java.util.Collection; +import java.util.Iterator; import java.util.Arrays; import java.util.Comparator; /** Scorer for conjunctions, sets of queries, all of which are required. */ class ConjunctionScorer extends Scorer { private final Scorer[] scorers; + private final DocIdSetIterator[] disis; // also includes all scorers. private boolean firstTime=true; private boolean more; @@ -35,12 +37,29 @@ this(similarity, (Scorer[])scorers.toArray(new Scorer[scorers.size()])); } + public ConjunctionScorer(Similarity similarity, Collection scorers, Collection disis) throws IOException { + this(similarity, (Scorer[])scorers.toArray(new Scorer[scorers.size()]), disis); + } + public ConjunctionScorer(Similarity similarity, Scorer[] scorers) throws IOException { super(similarity); this.scorers = scorers; + this.disis = scorers; coord = getSimilarity().coord(this.scorers.length, this.scorers.length); } + public ConjunctionScorer(Similarity similarity, Scorer[] scorers, Collection disis) throws IOException { + this(similarity, scorers); + if (disis.size() > 0) { + this.disis = new DocIdSetIterator[scorers.length + disis.size()]; + System.arraycopy(this.scorers, 0, this.disis, 0, this.scorers.length); + Iterator dit = disis.iterator(); + for (int i = 0; dit.hasNext(); i++) { + this.disis[i + this.scorers.length] = (DocIdSetIterator) dit.next(); + } + } + } + public int doc() { return lastDoc; } public boolean next() throws IOException { @@ -53,12 +72,12 @@ private boolean doNext() throws IOException { int first=0; - Scorer lastScorer = scorers[scorers.length-1]; - Scorer firstScorer; - while (more && (firstScorer=scorers[first]).doc() < (lastDoc=lastScorer.doc())) { - more = firstScorer.skipTo(lastDoc); - lastScorer = firstScorer; - first = (first == (scorers.length-1)) ? 0 : first+1; + DocIdSetIterator lastDisi = disis[disis.length-1]; + DocIdSetIterator firstDisi; + while (more && (firstDisi=disis[first]).doc() < (lastDoc=lastDisi.doc())) { + more = firstDisi.skipTo(lastDoc); + lastDisi = firstDisi; + first = (first == (disis.length-1)) ? 0 : first+1; } return more; } @@ -67,7 +86,7 @@ if (firstTime) return init(target); else if (more) - more = scorers[(scorers.length-1)].skipTo(target); + more = disis[(disis.length-1)].skipTo(target); return doNext(); } @@ -75,9 +94,9 @@ // thus skipping a check for firstTime per call to next() and skipTo() private boolean init(int target) throws IOException { firstTime=false; - more = scorers.length>1; - for (int i=0; i1; + for (int i=0; i>1); i++) { - Scorer tmp = scorers[i]; - scorers[i] = scorers[end-i-1]; - scorers[end-i-1] = tmp; + DocIdSetIterator tmp = disis[i]; + disis[i] = disis[end-i-1]; + disis[end-i-1] = tmp; } return more; @@ -113,7 +132,7 @@ public float score() throws IOException { float sum = 0.0f; - for (int i = 0; i < scorers.length; i++) { + for (int i = 0; i < scorers.length; i++) { // use scorers here, not disis. sum += scorers[i].score(); } return sum * coord;