Index: lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java (revision 1363128) +++ lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java (working copy) @@ -59,6 +59,8 @@ return doc < scores.length ? doc : NO_MORE_DOCS; } + @Override public long estimateCost() { return scores.length; } + } private static final class ScoreCachingCollector extends Collector { Index: lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java (revision 1363128) +++ lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java (working copy) @@ -47,6 +47,8 @@ @Override public int advance(int target) throws IOException { return 0; } + @Override public long estimateCost() { return 0; } + } private static class NoOpCollector extends Collector { Index: lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java (revision 1363128) +++ lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java (working copy) @@ -50,6 +50,8 @@ idx = target; return idx < scores.length ? idx : NO_MORE_DOCS; } + + @Override public long estimateCost() { return 1; } } // The scores must have positive as well as negative values Index: lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java (revision 1363128) +++ lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java (working copy) @@ -92,6 +92,8 @@ return doc = target <= 3000 ? 3000 : NO_MORE_DOCS; } + @Override public long estimateCost() { return 1; } + }}; BooleanScorer bs = new BooleanScorer(weight, false, 1, Arrays.asList(scorers), null, scorers.length); Index: lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java (revision 1363128) +++ lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java (working copy) @@ -198,6 +198,11 @@ protected float phraseFreq() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } + + @Override + public long estimateCost() { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } } @@ -245,6 +250,12 @@ public int advance(int target) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } + + @Override + public long estimateCost() { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } + } static final class JustCompileSimilarity extends Similarity { Index: lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java (revision 1363128) +++ lucene/core/src/test/org/apache/lucene/search/TestSubScorerFreqs.java (working copy) @@ -174,14 +174,16 @@ boolean includeOptional = occur.contains("SHOULD"); for (int i = 0; i < maxDocs; i++) { Map doc0 = c.docCounts.get(i); - assertEquals(includeOptional ? 5 : 4, doc0.size()); + // Y doesnt exist in the index, so its not in the scorer tree + assertEquals(4, doc0.size()); assertEquals(1.0F, doc0.get(aQuery), FLOAT_TOLERANCE); assertEquals(4.0F, doc0.get(dQuery), FLOAT_TOLERANCE); if (includeOptional) assertEquals(3.0F, doc0.get(cQuery), FLOAT_TOLERANCE); Map doc1 = c.docCounts.get(++i); - assertEquals(includeOptional ? 5 : 4, doc1.size()); + // Y doesnt exist in the index, so its not in the scorer tree + assertEquals(4, doc1.size()); assertEquals(1.0F, doc1.get(aQuery), FLOAT_TOLERANCE); assertEquals(1.0F, doc1.get(dQuery), FLOAT_TOLERANCE); if (includeOptional) Index: lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java (revision 1363215) +++ lucene/core/src/java/org/apache/lucene/search/DisjunctionScorer.java (working copy) @@ -39,7 +39,7 @@ * Organize subScorers into a min heap with scorers generating the earliest document on top. */ protected final void heapify() { - for (int i = (numScorers >> 1) - 1; i >= 0; i--) { + for (int i = (numScorers >>> 1) - 1; i >= 0; i--) { heapAdjust(i); } } @@ -52,7 +52,7 @@ Scorer scorer = subScorers[root]; int doc = scorer.docID(); int i = root; - while (i <= (numScorers >> 1) - 1) { + while (i <= (numScorers >>> 1) - 1) { int lchild = (i << 1) + 1; Scorer lscorer = subScorers[lchild]; int ldoc = lscorer.docID(); @@ -105,4 +105,13 @@ } return children; } + + @Override + public long estimateCost() { + long cost = 0; + for (int i = 0; i < numScorers; i++) { + cost += subScorers[i].estimateCost(); + } + return cost; + } } Index: lucene/core/src/java/org/apache/lucene/search/Scorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/Scorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/Scorer.java (working copy) @@ -125,4 +125,10 @@ this.relationship = relationship; } } + + /** + * returns an estimate of how expensive this scorer is: + * approximately how many postings it will read. + */ + public abstract long estimateCost(); } Index: lucene/core/src/java/org/apache/lucene/search/TermScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/TermScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/TermScorer.java (working copy) @@ -92,18 +92,9 @@ /** Returns a string representation of this TermScorer. */ @Override public String toString() { return "scorer(" + weight + ")"; } - - // TODO: benchmark if the specialized conjunction really benefits - // from this, or if instead its from sorting by docFreq, or both - - DocsEnum getDocsEnum() { - return docsEnum; - } - // TODO: generalize something like this for scorers? - // even this is just an estimation... - - int getDocFreq() { + @Override + public long estimateCost() { return docFreq; } } Index: lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java (working copy) @@ -124,6 +124,7 @@ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, final Bits acceptDocs) throws IOException { final DocIdSetIterator disi; + final long cost; if (filter != null) { assert query == null; final DocIdSet dis = filter.getDocIdSet(context, acceptDocs); @@ -131,15 +132,17 @@ return null; } disi = dis.iterator(); + cost = context.reader().maxDoc(); } else { assert query != null && innerWeight != null; disi = innerWeight.scorer(context, scoreDocsInOrder, topScorer, acceptDocs); + cost = disi == null ? 0 : ((Scorer) disi).estimateCost(); } if (disi == null) { return null; } - return new ConstantScorer(disi, this, queryWeight); + return new ConstantScorer(disi, this, queryWeight, cost); } @Override @@ -171,10 +174,12 @@ protected class ConstantScorer extends Scorer { final DocIdSetIterator docIdSetIterator; final float theScore; + final long theCost; - public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w, float theScore) { + public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w, float theScore, long theCost) { super(w); this.theScore = theScore; + this.theCost = theCost; this.docIdSetIterator = docIdSetIterator; } @@ -194,6 +199,11 @@ } @Override + public long estimateCost() { + return theCost; + } + + @Override public float freq() throws IOException { return 1; } @@ -208,7 +218,7 @@ @Override public void setScorer(Scorer scorer) throws IOException { // we must wrap again here, but using the scorer passed in as parameter: - collector.setScorer(new ConstantScorer(scorer, ConstantScorer.this.weight, ConstantScorer.this.theScore)); + collector.setScorer(new ConstantScorer(scorer, ConstantScorer.this.weight, ConstantScorer.this.theScore, ConstantScorer.this.theCost)); } @Override Index: lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java (working copy) @@ -1,341 +0,0 @@ -package org.apache.lucene.search; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.BooleanQuery.BooleanWeight; -import org.apache.lucene.search.similarities.Similarity; - -/* See the description in BooleanScorer.java, comparing - * BooleanScorer & BooleanScorer2 */ - -/** An alternative to BooleanScorer that also allows a minimum number - * of optional scorers that should match. - *
Implements skipTo(), and has no limitations on the numbers of added scorers. - *
Uses ConjunctionScorer, DisjunctionScorer, ReqOptScorer and ReqExclScorer. - */ -class BooleanScorer2 extends Scorer { - - private final List requiredScorers; - private final List optionalScorers; - private final List prohibitedScorers; - - private class Coordinator { - float[] coordFactors = null; - int maxCoord = 0; // to be increased for each non prohibited scorer - int nrMatchers; // to be increased by score() of match counting scorers. - - void init(boolean disableCoord) { // use after all scorers have been added. - coordFactors = new float[optionalScorers.size() + requiredScorers.size() + 1]; - for (int i = 0; i < coordFactors.length; i++) { - coordFactors[i] = disableCoord ? 1.0f : ((BooleanWeight)weight).coord(i, maxCoord); - } - } - } - - private final Coordinator coordinator; - - /** The scorer to which all scoring will be delegated, - * except for computing and using the coordination factor. - */ - private final Scorer countingSumScorer; - - /** The number of optionalScorers that need to match (if there are any) */ - private final int minNrShouldMatch; - - private int doc = -1; - - /** - * Creates a {@link Scorer} with the given similarity and lists of required, - * prohibited and optional scorers. In no required scorers are added, at least - * one of the optional scorers will have to match during the search. - * - * @param weight - * The BooleanWeight to be used. - * @param disableCoord - * If this parameter is true, coordination level matching - * ({@link Similarity#coord(int, int)}) is not used. - * @param minNrShouldMatch - * The minimum number of optional added scorers that should match - * during the search. In case no required scorers are added, at least - * one of the optional scorers will have to match during the search. - * @param required - * the list of required scorers. - * @param prohibited - * the list of prohibited scorers. - * @param optional - * the list of optional scorers. - */ - public BooleanScorer2(BooleanWeight weight, boolean disableCoord, int minNrShouldMatch, - List required, List prohibited, List optional, int maxCoord) throws IOException { - super(weight); - if (minNrShouldMatch < 0) { - throw new IllegalArgumentException("Minimum number of optional scorers should not be negative"); - } - coordinator = new Coordinator(); - this.minNrShouldMatch = minNrShouldMatch; - coordinator.maxCoord = maxCoord; - - optionalScorers = optional; - requiredScorers = required; - prohibitedScorers = prohibited; - - coordinator.init(disableCoord); - countingSumScorer = makeCountingSumScorer(disableCoord); - } - - /** Count a scorer as a single match. */ - private class SingleMatchScorer extends Scorer { - private Scorer scorer; - private int lastScoredDoc = -1; - // Save the score of lastScoredDoc, so that we don't compute it more than - // once in score(). - private float lastDocScore = Float.NaN; - - SingleMatchScorer(Scorer scorer) { - super(scorer.weight); - this.scorer = scorer; - } - - @Override - public float score() throws IOException { - int doc = docID(); - if (doc >= lastScoredDoc) { - if (doc > lastScoredDoc) { - lastDocScore = scorer.score(); - lastScoredDoc = doc; - } - coordinator.nrMatchers++; - } - return lastDocScore; - } - - @Override - public float freq() throws IOException { - return 1; - } - - @Override - public int docID() { - return scorer.docID(); - } - - @Override - public int nextDoc() throws IOException { - return scorer.nextDoc(); - } - - @Override - public int advance(int target) throws IOException { - return scorer.advance(target); - } - } - - private Scorer countingDisjunctionSumScorer(final List scorers, - int minNrShouldMatch) throws IOException { - // each scorer from the list counted as a single matcher - return new DisjunctionSumScorer(weight, scorers, minNrShouldMatch) { - private int lastScoredDoc = -1; - // Save the score of lastScoredDoc, so that we don't compute it more than - // once in score(). - private float lastDocScore = Float.NaN; - @Override public float score() throws IOException { - int doc = docID(); - if (doc >= lastScoredDoc) { - if (doc > lastScoredDoc) { - lastDocScore = super.score(); - lastScoredDoc = doc; - } - coordinator.nrMatchers += super.nrMatchers; - } - return lastDocScore; - } - }; - } - - private Scorer countingConjunctionSumScorer(boolean disableCoord, - List requiredScorers) throws IOException { - // each scorer from the list counted as a single matcher - final int requiredNrMatchers = requiredScorers.size(); - return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(requiredScorers.size(), requiredScorers.size()), requiredScorers) { - private int lastScoredDoc = -1; - // Save the score of lastScoredDoc, so that we don't compute it more than - // once in score(). - private float lastDocScore = Float.NaN; - @Override public float score() throws IOException { - int doc = docID(); - if (doc >= lastScoredDoc) { - if (doc > lastScoredDoc) { - lastDocScore = super.score(); - lastScoredDoc = doc; - } - coordinator.nrMatchers += requiredNrMatchers; - } - // All scorers match, so defaultSimilarity super.score() always has 1 as - // the coordination factor. - // Therefore the sum of the scores of the requiredScorers - // is used as score. - return lastDocScore; - } - }; - } - - private Scorer dualConjunctionSumScorer(boolean disableCoord, - Scorer req1, Scorer req2) throws IOException { // non counting. - return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(2, 2), req1, req2); - // All scorers match, so defaultSimilarity always has 1 as - // the coordination factor. - // Therefore the sum of the scores of two scorers - // is used as score. - } - - /** Returns the scorer to be used for match counting and score summing. - * Uses requiredScorers, optionalScorers and prohibitedScorers. - */ - private Scorer makeCountingSumScorer(boolean disableCoord) throws IOException { // each scorer counted as a single matcher - return (requiredScorers.size() == 0) - ? makeCountingSumScorerNoReq(disableCoord) - : makeCountingSumScorerSomeReq(disableCoord); - } - - private Scorer makeCountingSumScorerNoReq(boolean disableCoord) throws IOException { // No required scorers - // minNrShouldMatch optional scorers are required, but at least 1 - int nrOptRequired = (minNrShouldMatch < 1) ? 1 : minNrShouldMatch; - Scorer requiredCountingSumScorer; - if (optionalScorers.size() > nrOptRequired) - requiredCountingSumScorer = countingDisjunctionSumScorer(optionalScorers, nrOptRequired); - else if (optionalScorers.size() == 1) - requiredCountingSumScorer = new SingleMatchScorer(optionalScorers.get(0)); - else { - requiredCountingSumScorer = countingConjunctionSumScorer(disableCoord, optionalScorers); - } - return addProhibitedScorers(requiredCountingSumScorer); - } - - private Scorer makeCountingSumScorerSomeReq(boolean disableCoord) throws IOException { // At least one required scorer. - if (optionalScorers.size() == minNrShouldMatch) { // all optional scorers also required. - ArrayList allReq = new ArrayList(requiredScorers); - allReq.addAll(optionalScorers); - return addProhibitedScorers(countingConjunctionSumScorer(disableCoord, allReq)); - } else { // optionalScorers.size() > minNrShouldMatch, and at least one required scorer - Scorer requiredCountingSumScorer = - requiredScorers.size() == 1 - ? new SingleMatchScorer(requiredScorers.get(0)) - : countingConjunctionSumScorer(disableCoord, requiredScorers); - if (minNrShouldMatch > 0) { // use a required disjunction scorer over the optional scorers - return addProhibitedScorers( - dualConjunctionSumScorer( // non counting - disableCoord, - requiredCountingSumScorer, - countingDisjunctionSumScorer( - optionalScorers, - minNrShouldMatch))); - } else { // minNrShouldMatch == 0 - return new ReqOptSumScorer( - addProhibitedScorers(requiredCountingSumScorer), - optionalScorers.size() == 1 - ? new SingleMatchScorer(optionalScorers.get(0)) - // require 1 in combined, optional scorer. - : countingDisjunctionSumScorer(optionalScorers, 1)); - } - } - } - - /** Returns the scorer to be used for match counting and score summing. - * Uses the given required scorer and the prohibitedScorers. - * @param requiredCountingSumScorer A required scorer already built. - */ - private Scorer addProhibitedScorers(Scorer requiredCountingSumScorer) throws IOException - { - return (prohibitedScorers.size() == 0) - ? requiredCountingSumScorer // no prohibited - : new ReqExclScorer(requiredCountingSumScorer, - ((prohibitedScorers.size() == 1) - ? prohibitedScorers.get(0) - : new DisjunctionSumScorer(weight, prohibitedScorers))); - } - - /** Scores and collects all matching documents. - * @param collector The collector to which all matching documents are passed through. - */ - @Override - public void score(Collector collector) throws IOException { - collector.setScorer(this); - while ((doc = countingSumScorer.nextDoc()) != NO_MORE_DOCS) { - collector.collect(doc); - } - } - - @Override - public boolean score(Collector collector, int max, int firstDocID) throws IOException { - doc = firstDocID; - collector.setScorer(this); - while (doc < max) { - collector.collect(doc); - doc = countingSumScorer.nextDoc(); - } - return doc != NO_MORE_DOCS; - } - - @Override - public int docID() { - return doc; - } - - @Override - public int nextDoc() throws IOException { - return doc = countingSumScorer.nextDoc(); - } - - @Override - public float score() throws IOException { - coordinator.nrMatchers = 0; - float sum = countingSumScorer.score(); - return sum * coordinator.coordFactors[coordinator.nrMatchers]; - } - - @Override - public float freq() throws IOException { - return countingSumScorer.freq(); - } - - @Override - public int advance(int target) throws IOException { - return doc = countingSumScorer.advance(target); - } - - @Override - public Collection getChildren() { - ArrayList children = new ArrayList(); - for (Scorer s : optionalScorers) { - children.add(new ChildScorer(s, "SHOULD")); - } - for (Scorer s : prohibitedScorers) { - children.add(new ChildScorer(s, "MUST_NOT")); - } - for (Scorer s : requiredScorers) { - children.add(new ChildScorer(s, "MUST")); - } - return children; - } -} Index: lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java (working copy) @@ -110,8 +110,13 @@ } @Override + public long estimateCost() { + return reqScorer.estimateCost(); + } + + @Override public Collection getChildren() { - return Collections.singleton(new ChildScorer(reqScorer, "FILTERED")); + return Collections.singleton(new ChildScorer(reqScorer, "MUST")); } @Override Index: lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java (revision 1363215) +++ lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java (working copy) @@ -35,6 +35,8 @@ private double score = Float.NaN; + private final float[] coords; + /** Construct a DisjunctionScorer. * @param weight The weight to be used. * @param subScorers A collection of at least two subscorers. @@ -46,7 +48,7 @@ *
When minimumNrMatchers equals the number of subScorers, * it more efficient to use ConjunctionScorer. */ - public DisjunctionSumScorer(Weight weight, List subScorers, int minimumNrMatchers) throws IOException { + public DisjunctionSumScorer(Weight weight, List subScorers, int minimumNrMatchers, float coords[]) throws IOException { super(weight, subScorers.toArray(new Scorer[subScorers.size()]), subScorers.size()); if (minimumNrMatchers <= 0) { @@ -55,16 +57,11 @@ if (numScorers <= 1) { throw new IllegalArgumentException("There must be at least 2 subScorers"); } + assert coords.length > numScorers; + this.coords = coords; this.minimumNrMatchers = minimumNrMatchers; } - - /** Construct a DisjunctionScorer, using one as the minimum number - * of matching subscorers. - */ - public DisjunctionSumScorer(Weight weight, List subScorers) throws IOException { - this(weight, subScorers, 1); - } @Override public int nextDoc() throws IOException { @@ -116,7 +113,7 @@ */ @Override public float score() throws IOException { - return (float)score; + return coords[nrMatchers] * (float)score; } @Override Index: lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java (working copy) @@ -43,12 +43,17 @@ private PhrasePositions[][] rptGroups; // in each group are PPs that repeats each other (i.e. same term), sorted by (query) offset private PhrasePositions[] rptStack; // temporary stack for switching colliding repeating pps + private final long cost; + SloppyPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings, int slop, Similarity.SloppySimScorer docScorer) { super(weight, postings, docScorer); this.slop = slop; this.numPostings = postings==null ? 0 : postings.length; pq = new PhraseQueue(postings.length); + // minimum docfreq * # of terms + // TODO: should we use totalTermFreq instead? + cost = postings[0].docFreq * (long)postings.length; } /** @@ -482,6 +487,11 @@ } return tg; } + + @Override + public long estimateCost() { + return cost; + } // private void printQueue(PrintStream ps, PhrasePositions ext, String title) { // //if (min.doc != ?) return; Index: lucene/core/src/java/org/apache/lucene/search/FilteredScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FilteredScorer.java (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/FilteredScorer.java (working copy) @@ -0,0 +1,74 @@ +package org.apache.lucene.search; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +/** A FilteredScorer contains another Scorer, which it + * uses as its basic source of data, possibly transforming the data along the + * way or providing additional functionality. The class + * FilteredScorer itself simply implements all abstract methods + * of Scorer with versions that pass all requests to the + * contained scorer. Subclasses of FilteredScorer may + * further override some of these methods and may also provide additional + * methods and fields. + */ +public abstract class FilteredScorer extends Scorer { + protected final Scorer in; + + public FilteredScorer(Scorer in) { + super(in.weight); + this.in = in; + } + + @Override + public float score() throws IOException { + return in.score(); + } + + @Override + public float freq() throws IOException { + return in.freq(); + } + + @Override + public Collection getChildren() { + return in.getChildren(); + } + + @Override + public long estimateCost() { + return in.estimateCost(); + } + + @Override + public int docID() { + return in.docID(); + } + + @Override + public int nextDoc() throws IOException { + return in.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + return in.advance(target); + } +} Property changes on: lucene/core/src/java/org/apache/lucene/search/FilteredScorer.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java (working copy) @@ -29,8 +29,8 @@ /** The scorers passed from the constructor. * These are set to null as soon as their next() or skipTo() returns false. */ - private Scorer reqScorer; - private Scorer optScorer; + protected Scorer reqScorer; + protected Scorer optScorer; /** Construct a ReqOptScorer. * @param reqScorer The required scorer. This must match. @@ -92,11 +92,76 @@ } @Override + public long estimateCost() { + return reqScorer.estimateCost(); + } + + @Override public Collection getChildren() { ArrayList children = new ArrayList(2); children.add(new ChildScorer(reqScorer, "MUST")); children.add(new ChildScorer(optScorer, "SHOULD")); return children; } + + static class ReqSingleOptScorer extends ReqOptSumScorer { + // coord factor if just the required part matches + private final float coordReq; + // coord factor if both required and optional part matches + private final float coordBoth; + + public ReqSingleOptScorer(Scorer reqScorer, Scorer optScorer, float coordReq, float coordBoth) { + super(reqScorer, optScorer); + this.coordReq = coordReq; + this.coordBoth = coordBoth; + } + + @Override + public float score() throws IOException { + int curDoc = reqScorer.docID(); + float reqScore = reqScorer.score(); + if (optScorer == null) { + return reqScore * coordReq; + } + + int optScorerDoc = optScorer.docID(); + if (optScorerDoc < curDoc && (optScorerDoc = optScorer.advance(curDoc)) == NO_MORE_DOCS) { + optScorer = null; + return reqScore * coordReq; + } + + return optScorerDoc == curDoc ? (reqScore + optScorer.score()) * coordBoth : reqScore * coordReq; + } + } + + static class ReqMultiOptScorer extends ReqOptSumScorer { + private final int requiredCount; + private final float coords[]; + private final DisjunctionSumScorer disjunction; + + public ReqMultiOptScorer(Scorer reqScorer, DisjunctionSumScorer optScorer, int requiredCount, float coords[]) { + super(reqScorer, optScorer); + this.requiredCount = requiredCount; + this.coords = coords; + this.disjunction = optScorer; + } + + @Override + public float score() throws IOException { + int curDoc = reqScorer.docID(); + float reqScore = reqScorer.score(); + if (optScorer == null) { + return reqScore * coords[requiredCount]; + } + + int optScorerDoc = optScorer.docID(); + if (optScorerDoc < curDoc && (optScorerDoc = optScorer.advance(curDoc)) == NO_MORE_DOCS) { + optScorer = null; + return reqScore * coords[requiredCount]; + } + + return optScorerDoc == curDoc ? (reqScore + optScorer.score()) * coords[requiredCount + disjunction.nrMatchers] : reqScore * coords[requiredCount]; + } + } } Index: lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (working copy) @@ -73,6 +73,11 @@ } @Override + public long estimateCost() { + return maxDoc; + } + + @Override public int advance(int target) throws IOException { doc = target-1; return nextDoc(); Index: lucene/core/src/java/org/apache/lucene/search/CachingCollector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/CachingCollector.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/CachingCollector.java (working copy) @@ -89,6 +89,9 @@ @Override public final int nextDoc() { throw new UnsupportedOperationException(); } + + @Override + public long estimateCost() { return 0; } } // A CachingCollector which caches scores Index: lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java (working copy) @@ -25,130 +25,112 @@ /** Scorer for conjunctions, sets of queries, all of which are required. */ class ConjunctionScorer extends Scorer { - - private final Scorer[] scorers; - private final float coord; - private int lastDoc = -1; + protected final float coord; + protected int lastDoc = -1; + protected final Scorer[] scorers; + private final Scorer lead; - public ConjunctionScorer(Weight weight, float coord, Collection scorers) throws IOException { - this(weight, coord, scorers.toArray(new Scorer[scorers.size()])); - } - - public ConjunctionScorer(Weight weight, float coord, Scorer... scorers) throws IOException { + ConjunctionScorer(Weight weight, float coord, Scorer[] scorers) { super(weight); + this.coord = coord; this.scorers = scorers; - this.coord = coord; - - for (int i = 0; i < scorers.length; i++) { - if (scorers[i].nextDoc() == NO_MORE_DOCS) { - // If even one of the sub-scorers does not have any documents, this - // scorer should not attempt to do any more work. - lastDoc = NO_MORE_DOCS; - return; - } - } - - // Sort the array the first time... - // We don't need to sort the array in any future calls because we know - // it will already start off sorted (all scorers on same doc). - - // Note that this comparator is not consistent with equals! - // Also we use mergeSort here to be stable (so order of Scoreres that - // match on first document keeps preserved): - ArrayUtil.mergeSort(scorers, new Comparator() { // sort the array + // Sort the array the first time to allow the least frequent Scorer to + // lead the matching. + ArrayUtil.mergeSort(scorers, new Comparator() { public int compare(Scorer o1, Scorer o2) { - return o1.docID() - o2.docID(); + return Long.signum(o1.estimateCost() - o2.estimateCost()); } }); - // NOTE: doNext() must be called before the re-sorting of the array later on. - // The reason is this: assume there are 5 scorers, whose first docs are 1, - // 2, 3, 5, 5 respectively. Sorting (above) leaves the array as is. Calling - // doNext() here advances all the first scorers to 5 (or a larger doc ID - // they all agree on). - // However, if we re-sort before doNext() is called, the order will be 5, 3, - // 2, 1, 5 and then doNext() will stop immediately, since the first scorer's - // docs equals the last one. So the invariant that after calling doNext() - // all scorers are on the same doc ID is broken. - if (doNext() == NO_MORE_DOCS) { - // The scorers did not agree on any document. - lastDoc = NO_MORE_DOCS; - return; - } + lead = scorers[0]; // least frequent Scorer leads the intersection + } - // If first-time skip distance is any predictor of - // scorer sparseness, then we should always try to skip first on - // those scorers. - // Keep last scorer in it's last place (it will be the first - // to be skipped on), but reverse all of the others so that - // they will be skipped on in order of original high skip. - int end = scorers.length - 1; - int max = end >> 1; - for (int i = 0; i < max; i++) { - Scorer tmp = scorers[i]; - int idx = end - i - 1; - scorers[i] = scorers[idx]; - scorers[idx] = tmp; - } + private int doNext(int doc) throws IOException { + do { + if (lead.docID() == DocIdSetIterator.NO_MORE_DOCS) { + return NO_MORE_DOCS; + } + advanceHead: do { + for (int i = 1; i < scorers.length; i++) { + int currentID = scorers[i].docID(); + if (currentID < doc) { + currentID = scorers[i].advance(doc); + } + if (currentID > doc) { + // DocsEnum beyond the current doc - break and advance lead + break advanceHead; + } + } + // success - all DocsEnums are on the same doc + return doc; + } while (true); + // advance head for next iteration + doc = lead.nextDoc(); + } while (true); } - private int doNext() throws IOException { - int first = 0; - int doc = scorers[scorers.length - 1].docID(); - Scorer firstScorer; - while ((firstScorer = scorers[first]).docID() < doc) { - doc = firstScorer.advance(doc); - first = first == scorers.length - 1 ? 0 : first + 1; - } - return doc; - } - @Override public int advance(int target) throws IOException { - if (lastDoc == NO_MORE_DOCS) { - return lastDoc; - } else if (scorers[(scorers.length - 1)].docID() < target) { - scorers[(scorers.length - 1)].advance(target); - } - return lastDoc = doNext(); + int doc = lead.advance(target); + return lastDoc = doNext(doc); } @Override public int docID() { return lastDoc; } - + @Override public int nextDoc() throws IOException { - if (lastDoc == NO_MORE_DOCS) { - return lastDoc; - } else if (lastDoc == -1) { - return lastDoc = scorers[scorers.length - 1].docID(); - } - scorers[(scorers.length - 1)].nextDoc(); - return lastDoc = doNext(); + int doc = lead.nextDoc(); + return lastDoc = doNext(doc); } - + @Override public float score() throws IOException { float sum = 0.0f; - for (int i = 0; i < scorers.length; i++) { - sum += scorers[i].score(); + for (Scorer s : scorers) { + sum += s.score(); } return sum * coord; } - + @Override - public float freq() throws IOException { + public float freq() { return scorers.length; } @Override + public long estimateCost() { + return lead.estimateCost() * scorers.length; + } + + @Override public Collection getChildren() { ArrayList children = new ArrayList(scorers.length); - for (Scorer scorer : scorers) { - children.add(new ChildScorer(scorer, "MUST")); + for (Scorer s : scorers) { + children.add(new ChildScorer(s, "MUST")); } return children; } + + static class CoordinatingConjunctionScorer extends ConjunctionScorer { + private final float coords[]; + private final int reqCount; + private final Scorer req; + private final DisjunctionSumScorer opt; + + CoordinatingConjunctionScorer(Weight weight, float coords[], Scorer req, int reqCount, DisjunctionSumScorer opt) { + super(weight, Float.NaN, new Scorer[] { req, opt }); + this.coords = coords; + this.req = req; + this.reqCount = reqCount; + this.opt = opt; + } + + @Override + public float score() throws IOException { + return (req.score() + opt.score()) * coords[reqCount + opt.nrMatchers]; + } + } } Index: lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java (working copy) @@ -228,6 +228,9 @@ public float freq() throws IOException { return scorer.freq(); } @Override + public long estimateCost() { return scorer.estimateCost(); } + + @Override public Collection getChildren() { return Collections.singleton(new ChildScorer(scorer, "FILTERED")); } Index: lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (working copy) @@ -1,134 +0,0 @@ -package org.apache.lucene.search; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; - -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.util.ArrayUtil; - -/** Scorer for conjunctions, sets of terms, all of which are required. */ -class ConjunctionTermScorer extends Scorer { - protected final float coord; - protected int lastDoc = -1; - protected final DocsAndFreqs[] docsAndFreqs; - private final DocsAndFreqs lead; - - ConjunctionTermScorer(Weight weight, float coord, - DocsAndFreqs[] docsAndFreqs) { - super(weight); - this.coord = coord; - this.docsAndFreqs = docsAndFreqs; - // Sort the array the first time to allow the least frequent DocsEnum to - // lead the matching. - ArrayUtil.mergeSort(docsAndFreqs, new Comparator() { - public int compare(DocsAndFreqs o1, DocsAndFreqs o2) { - return o1.docFreq - o2.docFreq; - } - }); - - lead = docsAndFreqs[0]; // least frequent DocsEnum leads the intersection - } - - private int doNext(int doc) throws IOException { - do { - if (lead.doc == DocIdSetIterator.NO_MORE_DOCS) { - return NO_MORE_DOCS; - } - advanceHead: do { - for (int i = 1; i < docsAndFreqs.length; i++) { - if (docsAndFreqs[i].doc < doc) { - docsAndFreqs[i].doc = docsAndFreqs[i].docs.advance(doc); - } - if (docsAndFreqs[i].doc > doc) { - // DocsEnum beyond the current doc - break and advance lead - break advanceHead; - } - } - // success - all DocsEnums are on the same doc - return doc; - } while (true); - // advance head for next iteration - doc = lead.doc = lead.docs.nextDoc(); - } while (true); - } - - @Override - public int advance(int target) throws IOException { - lead.doc = lead.docs.advance(target); - return lastDoc = doNext(lead.doc); - } - - @Override - public int docID() { - return lastDoc; - } - - @Override - public int nextDoc() throws IOException { - lead.doc = lead.docs.nextDoc(); - return lastDoc = doNext(lead.doc); - } - - @Override - public float score() throws IOException { - float sum = 0.0f; - for (DocsAndFreqs docs : docsAndFreqs) { - sum += docs.scorer.score(); - } - return sum * coord; - } - - @Override - public float freq() { - return docsAndFreqs.length; - } - - @Override - public Collection getChildren() { - ArrayList children = new ArrayList(docsAndFreqs.length); - for (DocsAndFreqs docs : docsAndFreqs) { - children.add(new ChildScorer(docs.scorer, "MUST")); - } - return children; - } - - static final class DocsAndFreqs { - final DocsEnum docs; - final int docFreq; - final Scorer scorer; - int doc = -1; - - DocsAndFreqs(TermScorer termScorer) { - this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq()); - } - - DocsAndFreqs(MatchOnlyTermScorer termScorer) { - this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq()); - } - - DocsAndFreqs(Scorer scorer, DocsEnum docs, int docFreq) { - this.docs = docs; - this.docFreq = docFreq; - this.scorer = scorer; - } - } -} Index: lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java (working copy) @@ -97,4 +97,9 @@ public float freq() throws IOException { return freq; } + + @Override + public long estimateCost() { + return Integer.MAX_VALUE; // :) + } } Index: lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java (working copy) @@ -96,17 +96,8 @@ @Override public String toString() { return "scorer(" + weight + ")"; } - // TODO: benchmark if the specialized conjunction really benefits - // from these, or if instead its from sorting by docFreq, or both - - DocsEnum getDocsEnum() { - return docsEnum; - } - - // TODO: generalize something like this for scorers? - // even this is just an estimation... - - int getDocFreq() { + @Override + public long estimateCost() { return docFreq; } } Index: lucene/core/src/java/org/apache/lucene/search/BoostedScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/BoostedScorer.java (revision 0) +++ lucene/core/src/java/org/apache/lucene/search/BoostedScorer.java (working copy) @@ -0,0 +1,34 @@ +package org.apache.lucene.search; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +final class BoostedScorer extends FilteredScorer { + private final float boost; + + BoostedScorer(Scorer in, float boost) { + super(in); + this.boost = boost; + } + + @Override + public float score() throws IOException { + return in.score() * boost; + } +} Property changes on: lucene/core/src/java/org/apache/lucene/search/BoostedScorer.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java (working copy) @@ -32,61 +32,39 @@ * several places, however all they have in hand is a {@link Scorer} object, and * might end up computing the score of a document more than once. */ -public class ScoreCachingWrappingScorer extends Scorer { - - private final Scorer scorer; +public class ScoreCachingWrappingScorer extends FilteredScorer { private int curDoc = -1; private float curScore; /** Creates a new instance by wrapping the given scorer. */ - public ScoreCachingWrappingScorer(Scorer scorer) { - super(scorer.weight); - this.scorer = scorer; + public ScoreCachingWrappingScorer(Scorer in) { + super(in); } @Override - public boolean score(Collector collector, int max, int firstDocID) throws IOException { - return scorer.score(collector, max, firstDocID); - } - - @Override public float score() throws IOException { - int doc = scorer.docID(); + int doc = in.docID(); if (doc != curDoc) { - curScore = scorer.score(); + curScore = in.score(); curDoc = doc; } return curScore; } - - @Override - public float freq() throws IOException { - return scorer.freq(); - } - - @Override - public int docID() { - return scorer.docID(); - } - - @Override - public int nextDoc() throws IOException { - return scorer.nextDoc(); - } + // we intentionally don't cache the bulk scoring methods @Override public void score(Collector collector) throws IOException { - scorer.score(collector); + in.score(collector); } - + @Override - public int advance(int target) throws IOException { - return scorer.advance(target); + public boolean score(Collector collector, int max, int firstDocID) throws IOException { + return in.score(collector, max, firstDocID); } @Override public Collection getChildren() { - return Collections.singleton(new ChildScorer(scorer, "CACHED")); + return Collections.singleton(new ChildScorer(in, "CACHED")); } } Index: lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java (working copy) @@ -136,6 +136,9 @@ @Override public float score() { return score; } + @Override + public long estimateCost() { return 1; } + } static final class Bucket { @@ -205,6 +208,8 @@ // Any time a prohibited clause matches we set bit 0: private static final int PROHIBITED_MASK = 1; + private long cost; + BooleanScorer(BooleanWeight weight, boolean disableCoord, int minNrShouldMatch, List optionalScorers, List prohibitedScorers, int maxCoord) throws IOException { super(weight); @@ -213,6 +218,7 @@ if (optionalScorers != null && optionalScorers.size() > 0) { for (Scorer scorer : optionalScorers) { if (scorer.nextDoc() != NO_MORE_DOCS) { + cost += scorer.estimateCost(); scorers = new SubScorer(scorer, false, false, bucketTable.newCollector(0), scorers); } } @@ -323,6 +329,11 @@ } @Override + public long estimateCost() { + return cost; + } + + @Override public void score(Collector collector) throws IOException { score(collector, Integer.MAX_VALUE, -1); } Index: lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (working copy) @@ -21,15 +21,10 @@ import java.util.*; import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.ConjunctionTermScorer.DocsAndFreqs; -import org.apache.lucene.search.TermQuery.TermWeight; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarities.Similarity.ExactSimScorer; import org.apache.lucene.util.Bits; import org.apache.lucene.util.ToStringUtils; @@ -173,24 +168,18 @@ protected ArrayList weights; protected int maxCoord; // num optional + num required private final boolean disableCoord; - private final boolean termConjunction; public BooleanWeight(IndexSearcher searcher, boolean disableCoord) throws IOException { this.similarity = searcher.getSimilarity(); this.disableCoord = disableCoord; weights = new ArrayList(clauses.size()); - boolean termConjunction = clauses.isEmpty() || minNrShouldMatch != 0 ? false : true; for (int i = 0 ; i < clauses.size(); i++) { BooleanClause c = clauses.get(i); Weight w = c.getQuery().createWeight(searcher); - if (!(c.isRequired() && (w instanceof TermWeight))) { - termConjunction = false; - } weights.add(w); if (!c.isProhibited()) maxCoord++; } - this.termConjunction = termConjunction; } @Override @@ -305,10 +294,10 @@ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, Bits acceptDocs) throws IOException { - if (termConjunction) { - // specialized scorer for term conjunctions - return createConjunctionTermScorer(context, acceptDocs); - } + // initially the user provided value, + // but if minNrShouldMatch == optional.size(), + // we will optimize and move these to required, making this 0 + int minShouldMatch = minNrShouldMatch; List required = new ArrayList(); List prohibited = new ArrayList(); List optional = new ArrayList(); @@ -329,63 +318,141 @@ } } - // Check if we can return a BooleanScorer - if (!scoreDocsInOrder && topScorer && required.size() == 0) { - return new BooleanScorer(this, disableCoord, minNrShouldMatch, optional, prohibited, maxCoord); - } + // scorer simplifications: - if (required.size() == 0 && optional.size() == 0) { + if (required.isEmpty() && optional.isEmpty()) { // no required and optional clauses. return null; - } else if (optional.size() < minNrShouldMatch) { + } else if (optional.size() < minShouldMatch) { // either >1 req scorer, or there are 0 req scorers and at least 1 // optional scorer. Therefore if there are not enough optional scorers // no documents will be matched by the query return null; + } else if (optional.size() == minShouldMatch) { + // either we have no optional clauses, + // or they are all required + required.addAll(optional); + optional.clear(); + minShouldMatch = 0; } - // Return a BooleanScorer2 - return new BooleanScorer2(this, disableCoord, minNrShouldMatch, required, prohibited, optional, maxCoord); - } + // check if we can return a ConjunctionScorer + if (optional.isEmpty()) { + return excl(req(required, disableCoord), prohibited); + } - private Scorer createConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs) - throws IOException { - - // TODO: fix scorer API to specify "needsScores" up - // front, so we can do match-only if caller doesn't - // needs scores - - final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()]; - for (int i = 0; i < docsAndFreqs.length; i++) { - final TermWeight weight = (TermWeight) weights.get(i); - final Scorer scorer = weight.scorer(context, true, false, acceptDocs); - if (scorer == null) { - return null; + // Check if we can return a BooleanScorer + if (!scoreDocsInOrder && topScorer && required.size() == 0) { + return new BooleanScorer(this, disableCoord, minShouldMatch, optional, prohibited, maxCoord); + } + + // pure disjunction + if (required.isEmpty()) { + return excl(opt(optional, minShouldMatch, disableCoord), prohibited); + } + + // disjunction-conjunction mix: clean this up + Scorer req = excl(req(required, true), prohibited); + Scorer opt = opt(optional, minShouldMatch, true); + if (disableCoord) { + if (minShouldMatch > 0) { + return new ConjunctionScorer(this, 1F, new Scorer[] { req, opt }); + } else { + return new ReqOptSumScorer(req, opt); } - if (scorer instanceof TermScorer) { - docsAndFreqs[i] = new DocsAndFreqs((TermScorer) scorer); + } else if (optional.size() == 1) { + if (minShouldMatch > 0) { + return new ConjunctionScorer(this, coord(required.size()+1, maxCoord), new Scorer[] { req, opt }); } else { - docsAndFreqs[i] = new DocsAndFreqs((MatchOnlyTermScorer) scorer); + float coordReq = coord(required.size(), maxCoord); + float coordBoth = coord(required.size() + 1, maxCoord); + return new ReqOptSumScorer.ReqSingleOptScorer(req, opt, coordReq, coordBoth); } + } else { + if (minShouldMatch > 0) { + return new ConjunctionScorer.CoordinatingConjunctionScorer(this, coords(), req, required.size(), (DisjunctionSumScorer)opt); + } else { + return new ReqOptSumScorer.ReqMultiOptScorer(req, (DisjunctionSumScorer)opt, required.size(), coords()); + } } - return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord( - docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs); } @Override public boolean scoresDocsOutOfOrder() { + int optionalCount = 0; for (BooleanClause c : clauses) { if (c.isRequired()) { return false; // BS2 (in-order) will be used by scorer() + } else if (!c.isProhibited()) { + optionalCount++; } } + if (optionalCount == minNrShouldMatch) { + return false; // BS2 (in-order) will be used, as this means we have mandatory clauses + } + // scorer() will return an out-of-order scorer if requested. return true; } + private Scorer req(List required, boolean disableCoord) { + Scorer req; + if (required.size() == 1) { + req = required.get(0); + if (!disableCoord && maxCoord > 1) { + req = new BoostedScorer(req, coord(required.size(), maxCoord)); + } + } else { + req = new ConjunctionScorer(this, disableCoord ? 1.0F : + coord(required.size(), maxCoord), required.toArray(new Scorer[required.size()])); + } + return req; + } + + private Scorer excl(Scorer main, List prohibited) throws IOException { + if (prohibited.isEmpty()) { + return main; + } else if (prohibited.size() == 1) { + return new ReqExclScorer(main, prohibited.get(0)); + } else { + // TODO: this scores the required clauses (which is stupid). but we always did this. + float coords[] = new float[prohibited.size()+1]; + Arrays.fill(coords, 1F); + return new ReqExclScorer(main, new DisjunctionSumScorer(this, prohibited, 1, coords)); + } + } + + private Scorer opt(List optional, int minShouldMatch, boolean disableCoord) throws IOException { + Scorer opt; + if (optional.size() == 1) { + opt = optional.get(0); + if (!disableCoord && maxCoord > 1) { + opt = new BoostedScorer(opt, coord(optional.size(), maxCoord)); + } + } else { + float coords[]; + if (disableCoord) { + coords = new float[optional.size()+1]; + Arrays.fill(coords, 1F); + } else { + coords = coords(); + } + opt = new DisjunctionSumScorer(this, optional, minShouldMatch == 0 ? 1 : minShouldMatch, coords); + } + return opt; + } + + private float[] coords() { + float[] coords = new float[maxCoord+1]; + coords[0] = 0F; + for (int i = 1; i < coords.length; i++) { + coords[i] = coord(i, maxCoord); + } + return coords; + } } - + @Override public Weight createWeight(IndexSearcher searcher) throws IOException { return new BooleanWeight(searcher, disableCoord); Index: lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (working copy) @@ -753,6 +753,11 @@ public float score() { return score; } + + @Override + public long estimateCost() { + return 0; + } } private final FakeScorer fakeScorer = new FakeScorer(); Index: lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java (working copy) @@ -57,6 +57,8 @@ private final Similarity.ExactSimScorer docScorer; + private final long cost; + ExactPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings, Similarity.ExactSimScorer docScorer) throws IOException { super(weight); @@ -65,7 +67,10 @@ chunkStates = new ChunkState[postings.length]; endMinus1 = postings.length-1; - + // minimum docfreq * # of terms + // TODO: should we use totalTermFreq instead? + // and what about the optimization below? + cost = postings[0].docFreq * (long)postings.length; for(int i=0;i getChildren() { return Collections.singleton(new ChildScorer(scorer, "CUSTOM")); } Index: lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java (revision 1363128) +++ lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java (working copy) @@ -346,6 +346,11 @@ } return doc; } + + @Override + public long estimateCost() { + return subQueryScorer.estimateCost(); + } } @Override Index: lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java =================================================================== --- lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java (revision 1363128) +++ lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java (working copy) @@ -117,6 +117,11 @@ public int nextDoc() { throw new UnsupportedOperationException(); } + + @Override + public long estimateCost() { + return 0; + } } private static final class OneGroup { Index: solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java =================================================================== --- solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java (revision 1363128) +++ solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java (working copy) @@ -153,6 +153,7 @@ final DocIdSetIterator docIdSetIterator; final float theScore; final Bits acceptDocs; + final long cost; int doc = -1; public ConstantScorer(AtomicReaderContext context, ConstantWeight w, float theScore, Bits acceptDocs) throws IOException { @@ -162,12 +163,15 @@ DocIdSet docIdSet = filter instanceof SolrFilter ? ((SolrFilter)filter).getDocIdSet(w.context, context, acceptDocs) : filter.getDocIdSet(context, acceptDocs); if (docIdSet == null) { docIdSetIterator = DocIdSet.EMPTY_DOCIDSET.iterator(); + cost = 0; } else { DocIdSetIterator iter = docIdSet.iterator(); if (iter == null) { docIdSetIterator = DocIdSet.EMPTY_DOCIDSET.iterator(); + cost = 0; } else { docIdSetIterator = iter; + cost = context.reader().maxDoc(); } } } @@ -196,6 +200,11 @@ public int advance(int target) throws IOException { return docIdSetIterator.advance(target); } + + @Override + public long estimateCost() { + return cost; + } } @Override Index: solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java =================================================================== --- solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (revision 1363128) +++ solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (working copy) @@ -249,7 +249,7 @@ // Although this set only includes live docs, other filters can be pushed down to queries. DocIdSet readerSet = filter.getDocIdSet(context, acceptDocs); if (readerSet == null) readerSet=DocIdSet.EMPTY_DOCIDSET; - return new JoinScorer(this, readerSet.iterator(), getBoost()); + return new JoinScorer(this, readerSet.iterator(), getBoost(), context.reader().maxDoc()); } @@ -510,11 +510,13 @@ protected static class JoinScorer extends Scorer { final DocIdSetIterator iter; final float score; + final long cost; int doc = -1; - public JoinScorer(Weight w, DocIdSetIterator iter, float score) throws IOException { + public JoinScorer(Weight w, DocIdSetIterator iter, float score, long cost) throws IOException { super(w); this.score = score; + this.cost = cost; this.iter = iter==null ? DocIdSet.EMPTY_DOCIDSET.iterator() : iter; } @@ -542,6 +544,12 @@ public int advance(int target) throws IOException { return iter.advance(target); } + + @Override + public long estimateCost() { + return cost; + } + } Index: solr/core/src/java/org/apache/solr/schema/LatLonType.java =================================================================== --- solr/core/src/java/org/apache/solr/schema/LatLonType.java (revision 1363128) +++ solr/core/src/java/org/apache/solr/schema/LatLonType.java (working copy) @@ -490,6 +490,11 @@ return 1; } + @Override + public long estimateCost() { + return maxDoc; + } + public Explanation explain(int doc) throws IOException { advance(doc); boolean matched = this.doc == doc;