Index: lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java =================================================================== --- lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java (revision 1363128) +++ lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinCollector.java (working copy) @@ -347,6 +347,11 @@ public int nextDoc() { throw new UnsupportedOperationException(); } + + @Override + public long estimateCost() { + return 1; + } } private OneGroup[] sortedGroups; Index: lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java =================================================================== --- lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java (revision 1363128) +++ lucene/join/src/java/org/apache/lucene/search/join/ToChildBlockJoinQuery.java (working copy) @@ -293,6 +293,11 @@ } return childDoc; } + + @Override + public long estimateCost() { + return parentScorer.estimateCost(); // TODO: can we do better? + } } @Override Index: lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java =================================================================== --- lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java (revision 1363128) +++ lucene/join/src/java/org/apache/lucene/search/join/TermsIncludingScoreQuery.java (working copy) @@ -127,10 +127,12 @@ } segmentTermsEnum = terms.iterator(segmentTermsEnum); + // what is the runtime...seems ok? + long cost = context.reader().maxDoc() * terms.size(); if (multipleValuesPerDocument) { - return new MVInnerScorer(this, acceptDocs, segmentTermsEnum, context.reader().maxDoc()); + return new MVInnerScorer(this, acceptDocs, segmentTermsEnum, context.reader().maxDoc(), cost); } else { - return new SVInnerScorer(this, acceptDocs, segmentTermsEnum); + return new SVInnerScorer(this, acceptDocs, segmentTermsEnum, cost); } } }; @@ -147,11 +149,14 @@ DocsEnum docsEnum; DocsEnum reuse; int scoreUpto; + + private final long cost; - SVInnerScorer(Weight weight, Bits acceptDocs, TermsEnum termsEnum) { + SVInnerScorer(Weight weight, Bits acceptDocs, TermsEnum termsEnum, long cost) { super(weight); this.acceptDocs = acceptDocs; this.termsEnum = termsEnum; + this.cost = cost; } public float score() throws IOException { @@ -213,6 +218,11 @@ public float freq() { return 1; } + + @Override + public long estimateCost() { + return cost; + } } // This impl that tracks whether a docid has already been emitted. This check makes sure that docs aren't emitted @@ -222,8 +232,8 @@ final FixedBitSet alreadyEmittedDocs; - MVInnerScorer(Weight weight, Bits acceptDocs, TermsEnum termsEnum, int maxDoc) { - super(weight, acceptDocs, termsEnum); + MVInnerScorer(Weight weight, Bits acceptDocs, TermsEnum termsEnum, int maxDoc, long cost) { + super(weight, acceptDocs, termsEnum, cost); alreadyEmittedDocs = new FixedBitSet(maxDoc); } Index: lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java =================================================================== --- lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java (revision 1363128) +++ lucene/join/src/java/org/apache/lucene/search/join/ToParentBlockJoinQuery.java (working copy) @@ -405,6 +405,11 @@ //System.out.println(" return nextParentDoc=" + nd); return nd; } + + @Override + public long estimateCost() { + return childScorer.estimateCost(); // TODO: can we do better? + } public Explanation explain(int docBase) throws IOException { int start = docBase + prevParentDoc + 1; // +1 b/c prevParentDoc is previous parent doc Index: lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java (revision 1363128) +++ lucene/queries/src/java/org/apache/lucene/queries/CustomScoreQuery.java (working copy) @@ -346,6 +346,11 @@ } return doc; } + + @Override + public long estimateCost() { + return subQueryScorer.estimateCost(); + } } @Override Index: lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java (revision 1363128) +++ lucene/queries/src/java/org/apache/lucene/queries/function/FunctionQuery.java (working copy) @@ -174,6 +174,11 @@ result.addDetail(new Explanation(weight.queryNorm,"queryNorm")); return result; } + + @Override + public long estimateCost() { + return maxDoc; + } } Index: lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java (revision 1363128) +++ lucene/queries/src/java/org/apache/lucene/queries/function/ValueSourceScorer.java (working copy) @@ -87,4 +87,9 @@ public float freq() throws IOException { return 1; } + + @Override + public long estimateCost() { + return maxDoc; + } } Index: lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java =================================================================== --- lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java (revision 1363128) +++ lucene/queries/src/java/org/apache/lucene/queries/function/BoostedQuery.java (working copy) @@ -172,6 +172,11 @@ } @Override + public long estimateCost() { + return scorer.estimateCost(); + } + + @Override public Collection getChildren() { return Collections.singleton(new ChildScorer(scorer, "CUSTOM")); } Index: lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java =================================================================== --- lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java (revision 1363128) +++ lucene/grouping/src/java/org/apache/lucene/search/grouping/BlockGroupingCollector.java (working copy) @@ -117,6 +117,11 @@ public int nextDoc() { throw new UnsupportedOperationException(); } + + @Override + public long estimateCost() { + return 0; + } } private static final class OneGroup { Index: lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java (working copy) @@ -228,6 +228,9 @@ public float freq() throws IOException { return scorer.freq(); } @Override + public long estimateCost() { return scorer.estimateCost(); } + + @Override public Collection getChildren() { return Collections.singleton(new ChildScorer(scorer, "FILTERED")); } Index: lucene/core/src/java/org/apache/lucene/search/TermScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/TermScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/TermScorer.java (working copy) @@ -92,18 +92,9 @@ /** Returns a string representation of this TermScorer. */ @Override public String toString() { return "scorer(" + weight + ")"; } - - // TODO: benchmark if the specialized conjunction really benefits - // from this, or if instead its from sorting by docFreq, or both - - DocsEnum getDocsEnum() { - return docsEnum; - } - // TODO: generalize something like this for scorers? - // even this is just an estimation... - - int getDocFreq() { + @Override + public long estimateCost() { return docFreq; } } Index: lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (working copy) @@ -1,134 +0,0 @@ -package org.apache.lucene.search; - -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Comparator; - -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.util.ArrayUtil; - -/** Scorer for conjunctions, sets of terms, all of which are required. */ -class ConjunctionTermScorer extends Scorer { - protected final float coord; - protected int lastDoc = -1; - protected final DocsAndFreqs[] docsAndFreqs; - private final DocsAndFreqs lead; - - ConjunctionTermScorer(Weight weight, float coord, - DocsAndFreqs[] docsAndFreqs) { - super(weight); - this.coord = coord; - this.docsAndFreqs = docsAndFreqs; - // Sort the array the first time to allow the least frequent DocsEnum to - // lead the matching. - ArrayUtil.mergeSort(docsAndFreqs, new Comparator() { - public int compare(DocsAndFreqs o1, DocsAndFreqs o2) { - return o1.docFreq - o2.docFreq; - } - }); - - lead = docsAndFreqs[0]; // least frequent DocsEnum leads the intersection - } - - private int doNext(int doc) throws IOException { - do { - if (lead.doc == DocIdSetIterator.NO_MORE_DOCS) { - return NO_MORE_DOCS; - } - advanceHead: do { - for (int i = 1; i < docsAndFreqs.length; i++) { - if (docsAndFreqs[i].doc < doc) { - docsAndFreqs[i].doc = docsAndFreqs[i].docs.advance(doc); - } - if (docsAndFreqs[i].doc > doc) { - // DocsEnum beyond the current doc - break and advance lead - break advanceHead; - } - } - // success - all DocsEnums are on the same doc - return doc; - } while (true); - // advance head for next iteration - doc = lead.doc = lead.docs.nextDoc(); - } while (true); - } - - @Override - public int advance(int target) throws IOException { - lead.doc = lead.docs.advance(target); - return lastDoc = doNext(lead.doc); - } - - @Override - public int docID() { - return lastDoc; - } - - @Override - public int nextDoc() throws IOException { - lead.doc = lead.docs.nextDoc(); - return lastDoc = doNext(lead.doc); - } - - @Override - public float score() throws IOException { - float sum = 0.0f; - for (DocsAndFreqs docs : docsAndFreqs) { - sum += docs.scorer.score(); - } - return sum * coord; - } - - @Override - public float freq() { - return docsAndFreqs.length; - } - - @Override - public Collection getChildren() { - ArrayList children = new ArrayList(docsAndFreqs.length); - for (DocsAndFreqs docs : docsAndFreqs) { - children.add(new ChildScorer(docs.scorer, "MUST")); - } - return children; - } - - static final class DocsAndFreqs { - final DocsEnum docs; - final int docFreq; - final Scorer scorer; - int doc = -1; - - DocsAndFreqs(TermScorer termScorer) { - this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq()); - } - - DocsAndFreqs(MatchOnlyTermScorer termScorer) { - this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq()); - } - - DocsAndFreqs(Scorer scorer, DocsEnum docs, int docFreq) { - this.docs = docs; - this.docFreq = docFreq; - this.scorer = scorer; - } - } -} Index: lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java (working copy) @@ -124,6 +124,7 @@ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, final Bits acceptDocs) throws IOException { final DocIdSetIterator disi; + final long cost; if (filter != null) { assert query == null; final DocIdSet dis = filter.getDocIdSet(context, acceptDocs); @@ -131,15 +132,17 @@ return null; } disi = dis.iterator(); + cost = context.reader().maxDoc(); } else { assert query != null && innerWeight != null; disi = innerWeight.scorer(context, scoreDocsInOrder, topScorer, acceptDocs); + cost = disi == null ? 0 : ((Scorer) disi).estimateCost(); } if (disi == null) { return null; } - return new ConstantScorer(disi, this, queryWeight); + return new ConstantScorer(disi, this, queryWeight, cost); } @Override @@ -171,10 +174,12 @@ protected class ConstantScorer extends Scorer { final DocIdSetIterator docIdSetIterator; final float theScore; + final long theCost; - public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w, float theScore) { + public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w, float theScore, long theCost) { super(w); this.theScore = theScore; + this.theCost = theCost; this.docIdSetIterator = docIdSetIterator; } @@ -194,6 +199,11 @@ } @Override + public long estimateCost() { + return theCost; + } + + @Override public float freq() throws IOException { return 1; } @@ -208,7 +218,7 @@ @Override public void setScorer(Scorer scorer) throws IOException { // we must wrap again here, but using the scorer passed in as parameter: - collector.setScorer(new ConstantScorer(scorer, ConstantScorer.this.weight, ConstantScorer.this.theScore)); + collector.setScorer(new ConstantScorer(scorer, ConstantScorer.this.weight, ConstantScorer.this.theScore, ConstantScorer.this.theCost)); } @Override Index: lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java (working copy) @@ -136,6 +136,9 @@ @Override public float score() { return score; } + @Override + public long estimateCost() { return 1; } + } static final class Bucket { @@ -205,6 +208,8 @@ // Any time a prohibited clause matches we set bit 0: private static final int PROHIBITED_MASK = 1; + private long cost; + BooleanScorer(BooleanWeight weight, boolean disableCoord, int minNrShouldMatch, List optionalScorers, List prohibitedScorers, int maxCoord) throws IOException { super(weight); @@ -213,6 +218,7 @@ if (optionalScorers != null && optionalScorers.size() > 0) { for (Scorer scorer : optionalScorers) { if (scorer.nextDoc() != NO_MORE_DOCS) { + cost += scorer.estimateCost(); scorers = new SubScorer(scorer, false, false, bucketTable.newCollector(0), scorers); } } @@ -323,6 +329,11 @@ } @Override + public long estimateCost() { + return cost; + } + + @Override public void score(Collector collector) throws IOException { score(collector, Integer.MAX_VALUE, -1); } Index: lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/SloppyPhraseScorer.java (working copy) @@ -43,12 +43,17 @@ private PhrasePositions[][] rptGroups; // in each group are PPs that repeats each other (i.e. same term), sorted by (query) offset private PhrasePositions[] rptStack; // temporary stack for switching colliding repeating pps + private final long cost; + SloppyPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings, int slop, Similarity.SloppySimScorer docScorer) { super(weight, postings, docScorer); this.slop = slop; this.numPostings = postings==null ? 0 : postings.length; pq = new PhraseQueue(postings.length); + // minimum docfreq * # of terms + // TODO: should we use totalTermFreq instead? + cost = postings[0].docFreq * (long)postings.length; } /** @@ -482,6 +487,11 @@ } return tg; } + + @Override + public long estimateCost() { + return cost; + } // private void printQueue(PrintStream ps, PhrasePositions ext, String title) { // //if (min.doc != ?) return; Index: lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java (working copy) @@ -753,6 +753,11 @@ public float score() { return score; } + + @Override + public long estimateCost() { + return 0; + } } private final FakeScorer fakeScorer = new FakeScorer(); Index: lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java (working copy) @@ -96,17 +96,8 @@ @Override public String toString() { return "scorer(" + weight + ")"; } - // TODO: benchmark if the specialized conjunction really benefits - // from these, or if instead its from sorting by docFreq, or both - - DocsEnum getDocsEnum() { - return docsEnum; - } - - // TODO: generalize something like this for scorers? - // even this is just an estimation... - - int getDocFreq() { + @Override + public long estimateCost() { return docFreq; } } Index: lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (working copy) @@ -21,15 +21,11 @@ import java.util.*; import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; -import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.BooleanClause.Occur; -import org.apache.lucene.search.ConjunctionTermScorer.DocsAndFreqs; import org.apache.lucene.search.TermQuery.TermWeight; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarities.Similarity.ExactSimScorer; import org.apache.lucene.util.Bits; import org.apache.lucene.util.ToStringUtils; @@ -173,24 +169,18 @@ protected ArrayList weights; protected int maxCoord; // num optional + num required private final boolean disableCoord; - private final boolean termConjunction; public BooleanWeight(IndexSearcher searcher, boolean disableCoord) throws IOException { this.similarity = searcher.getSimilarity(); this.disableCoord = disableCoord; weights = new ArrayList(clauses.size()); - boolean termConjunction = clauses.isEmpty() || minNrShouldMatch != 0 ? false : true; for (int i = 0 ; i < clauses.size(); i++) { BooleanClause c = clauses.get(i); Weight w = c.getQuery().createWeight(searcher); - if (!(c.isRequired() && (w instanceof TermWeight))) { - termConjunction = false; - } weights.add(w); if (!c.isProhibited()) maxCoord++; } - this.termConjunction = termConjunction; } @Override @@ -305,14 +295,11 @@ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder, boolean topScorer, Bits acceptDocs) throws IOException { - if (termConjunction) { - // specialized scorer for term conjunctions - return createConjunctionTermScorer(context, acceptDocs); - } List required = new ArrayList(); List prohibited = new ArrayList(); List optional = new ArrayList(); Iterator cIter = clauses.iterator(); + boolean conjunction = true; for (Weight w : weights) { BooleanClause c = cIter.next(); Scorer subScorer = w.scorer(context, true, false, acceptDocs); @@ -324,16 +311,17 @@ required.add(subScorer); } else if (c.isProhibited()) { prohibited.add(subScorer); + conjunction = false; // TODO: this is dumb, but lets generalize the whole opto first } else { optional.add(subScorer); } } - // Check if we can return a BooleanScorer - if (!scoreDocsInOrder && topScorer && required.size() == 0) { - return new BooleanScorer(this, disableCoord, minNrShouldMatch, optional, prohibited, maxCoord); - } - + // either we have no optional clauses, + // or they are all required because of minNrShouldMatch: + // making them part of the conjunction + conjunction &= optional.size() == minNrShouldMatch; + if (required.size() == 0 && optional.size() == 0) { // no required and optional clauses. return null; @@ -344,42 +332,41 @@ return null; } + // check if we can return a ConjunctionScorer + // this is possible when all mandatory clauses are terms, + // and all optional clauses are also terms and minNrShouldMatch == optional.size() + if (conjunction) { + List subs = new ArrayList(required.size() + optional.size()); + subs.addAll(required); + subs.addAll(optional); + return new ConjunctionScorer(this, disableCoord ? 1.0F : coord( + subs.size(), maxCoord), subs.toArray(new Scorer[subs.size()])); + } + + // Check if we can return a BooleanScorer + if (!scoreDocsInOrder && topScorer && required.size() == 0 && optional.size() != minNrShouldMatch) { + return new BooleanScorer(this, disableCoord, minNrShouldMatch, optional, prohibited, maxCoord); + } + // Return a BooleanScorer2 return new BooleanScorer2(this, disableCoord, minNrShouldMatch, required, prohibited, optional, maxCoord); } - - private Scorer createConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs) - throws IOException { - - // TODO: fix scorer API to specify "needsScores" up - // front, so we can do match-only if caller doesn't - // needs scores - - final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()]; - for (int i = 0; i < docsAndFreqs.length; i++) { - final TermWeight weight = (TermWeight) weights.get(i); - final Scorer scorer = weight.scorer(context, true, false, acceptDocs); - if (scorer == null) { - return null; - } - if (scorer instanceof TermScorer) { - docsAndFreqs[i] = new DocsAndFreqs((TermScorer) scorer); - } else { - docsAndFreqs[i] = new DocsAndFreqs((MatchOnlyTermScorer) scorer); - } - } - return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord( - docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs); - } @Override public boolean scoresDocsOutOfOrder() { + int optionalCount = 0; for (BooleanClause c : clauses) { if (c.isRequired()) { return false; // BS2 (in-order) will be used by scorer() + } else if (!c.isProhibited()) { + optionalCount++; } } + if (optionalCount == minNrShouldMatch) { + return false; // BS2 (in-order) will be used, as this means we have mandatory clauses + } + // scorer() will return an out-of-order scorer if requested. return true; } Index: lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (working copy) @@ -73,6 +73,11 @@ } @Override + public long estimateCost() { + return maxDoc; + } + + @Override public int advance(int target) throws IOException { doc = target-1; return nextDoc(); Index: lucene/core/src/java/org/apache/lucene/search/Scorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/Scorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/Scorer.java (working copy) @@ -125,4 +125,10 @@ this.relationship = relationship; } } + + /** + * returns an estimate of how expensive this scorer is: + * approximately how many postings it will read. + */ + public abstract long estimateCost(); } Index: lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java (working copy) @@ -25,129 +25,91 @@ /** Scorer for conjunctions, sets of queries, all of which are required. */ class ConjunctionScorer extends Scorer { - - private final Scorer[] scorers; - private final float coord; - private int lastDoc = -1; + protected final float coord; + protected int lastDoc = -1; + protected final Scorer[] scorers; + private final Scorer lead; - public ConjunctionScorer(Weight weight, float coord, Collection scorers) throws IOException { - this(weight, coord, scorers.toArray(new Scorer[scorers.size()])); - } - - public ConjunctionScorer(Weight weight, float coord, Scorer... scorers) throws IOException { + ConjunctionScorer(Weight weight, float coord, Scorer[] scorers) { super(weight); + this.coord = coord; this.scorers = scorers; - this.coord = coord; - - for (int i = 0; i < scorers.length; i++) { - if (scorers[i].nextDoc() == NO_MORE_DOCS) { - // If even one of the sub-scorers does not have any documents, this - // scorer should not attempt to do any more work. - lastDoc = NO_MORE_DOCS; - return; - } - } - - // Sort the array the first time... - // We don't need to sort the array in any future calls because we know - // it will already start off sorted (all scorers on same doc). - - // Note that this comparator is not consistent with equals! - // Also we use mergeSort here to be stable (so order of Scoreres that - // match on first document keeps preserved): - ArrayUtil.mergeSort(scorers, new Comparator() { // sort the array + // Sort the array the first time to allow the least frequent Scorer to + // lead the matching. + ArrayUtil.mergeSort(scorers, new Comparator() { public int compare(Scorer o1, Scorer o2) { - return o1.docID() - o2.docID(); + return Long.signum(o1.estimateCost() - o2.estimateCost()); } }); - // NOTE: doNext() must be called before the re-sorting of the array later on. - // The reason is this: assume there are 5 scorers, whose first docs are 1, - // 2, 3, 5, 5 respectively. Sorting (above) leaves the array as is. Calling - // doNext() here advances all the first scorers to 5 (or a larger doc ID - // they all agree on). - // However, if we re-sort before doNext() is called, the order will be 5, 3, - // 2, 1, 5 and then doNext() will stop immediately, since the first scorer's - // docs equals the last one. So the invariant that after calling doNext() - // all scorers are on the same doc ID is broken. - if (doNext() == NO_MORE_DOCS) { - // The scorers did not agree on any document. - lastDoc = NO_MORE_DOCS; - return; - } + lead = scorers[0]; // least frequent Scorer leads the intersection + } - // If first-time skip distance is any predictor of - // scorer sparseness, then we should always try to skip first on - // those scorers. - // Keep last scorer in it's last place (it will be the first - // to be skipped on), but reverse all of the others so that - // they will be skipped on in order of original high skip. - int end = scorers.length - 1; - int max = end >> 1; - for (int i = 0; i < max; i++) { - Scorer tmp = scorers[i]; - int idx = end - i - 1; - scorers[i] = scorers[idx]; - scorers[idx] = tmp; - } + private int doNext(int doc) throws IOException { + do { + if (lead.docID() == DocIdSetIterator.NO_MORE_DOCS) { + return NO_MORE_DOCS; + } + advanceHead: do { + for (int i = 1; i < scorers.length; i++) { + int currentID = scorers[i].docID(); + if (currentID < doc) { + currentID = scorers[i].advance(doc); + } + if (currentID > doc) { + // DocsEnum beyond the current doc - break and advance lead + break advanceHead; + } + } + // success - all DocsEnums are on the same doc + return doc; + } while (true); + // advance head for next iteration + doc = lead.nextDoc(); + } while (true); } - private int doNext() throws IOException { - int first = 0; - int doc = scorers[scorers.length - 1].docID(); - Scorer firstScorer; - while ((firstScorer = scorers[first]).docID() < doc) { - doc = firstScorer.advance(doc); - first = first == scorers.length - 1 ? 0 : first + 1; - } - return doc; - } - @Override public int advance(int target) throws IOException { - if (lastDoc == NO_MORE_DOCS) { - return lastDoc; - } else if (scorers[(scorers.length - 1)].docID() < target) { - scorers[(scorers.length - 1)].advance(target); - } - return lastDoc = doNext(); + int doc = lead.advance(target); + return lastDoc = doNext(doc); } @Override public int docID() { return lastDoc; } - + @Override public int nextDoc() throws IOException { - if (lastDoc == NO_MORE_DOCS) { - return lastDoc; - } else if (lastDoc == -1) { - return lastDoc = scorers[scorers.length - 1].docID(); - } - scorers[(scorers.length - 1)].nextDoc(); - return lastDoc = doNext(); + int doc = lead.nextDoc(); + return lastDoc = doNext(doc); } - + @Override public float score() throws IOException { float sum = 0.0f; - for (int i = 0; i < scorers.length; i++) { - sum += scorers[i].score(); + for (Scorer s : scorers) { + sum += s.score(); } return sum * coord; } - + @Override - public float freq() throws IOException { + public float freq() { return scorers.length; } @Override + public long estimateCost() { + return lead.estimateCost() * scorers.length; + } + + @Override public Collection getChildren() { ArrayList children = new ArrayList(scorers.length); - for (Scorer scorer : scorers) { - children.add(new ChildScorer(scorer, "MUST")); + for (Scorer s : scorers) { + children.add(new ChildScorer(s, "MUST")); } return children; } Index: lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java (working copy) @@ -86,6 +86,11 @@ } @Override + public long estimateCost() { + return scorer.estimateCost(); + } + + @Override public Collection getChildren() { return Collections.singleton(new ChildScorer(scorer, "CACHED")); } Index: lucene/core/src/java/org/apache/lucene/search/CachingCollector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/CachingCollector.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/CachingCollector.java (working copy) @@ -89,6 +89,9 @@ @Override public final int nextDoc() { throw new UnsupportedOperationException(); } + + @Override + public long estimateCost() { return 0; } } // A CachingCollector which caches scores Index: lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java (working copy) @@ -92,6 +92,11 @@ } @Override + public long estimateCost() { + return reqScorer.estimateCost(); + } + + @Override public Collection getChildren() { ArrayList children = new ArrayList(2); children.add(new ChildScorer(reqScorer, "MUST")); Index: lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java (working copy) @@ -149,6 +149,11 @@ public int advance(int target) throws IOException { return scorer.advance(target); } + + @Override + public long estimateCost() { + return scorer.estimateCost(); + } } private Scorer countingDisjunctionSumScorer(final List scorers, @@ -177,7 +182,7 @@ List requiredScorers) throws IOException { // each scorer from the list counted as a single matcher final int requiredNrMatchers = requiredScorers.size(); - return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(requiredScorers.size(), requiredScorers.size()), requiredScorers) { + return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(requiredScorers.size(), requiredScorers.size()), requiredScorers.toArray(new Scorer[requiredScorers.size()])) { private int lastScoredDoc = -1; // Save the score of lastScoredDoc, so that we don't compute it more than // once in score(). @@ -202,7 +207,7 @@ private Scorer dualConjunctionSumScorer(boolean disableCoord, Scorer req1, Scorer req2) throws IOException { // non counting. - return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(2, 2), req1, req2); + return new ConjunctionScorer(weight, disableCoord ? 1.0f : ((BooleanWeight)weight).coord(2, 2), new Scorer[] { req1, req2 }); // All scorers match, so defaultSimilarity always has 1 as // the coordination factor. // Therefore the sum of the scores of two scorers @@ -325,6 +330,11 @@ } @Override + public long estimateCost() { + return countingSumScorer.estimateCost(); + } + + @Override public Collection getChildren() { ArrayList children = new ArrayList(); for (Scorer s : optionalScorers) { Index: lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/spans/SpanScorer.java (working copy) @@ -97,4 +97,9 @@ public float freq() throws IOException { return freq; } + + @Override + public long estimateCost() { + return Integer.MAX_VALUE; // :) + } } Index: lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java (working copy) @@ -110,6 +110,11 @@ } @Override + public long estimateCost() { + return reqScorer.estimateCost(); + } + + @Override public Collection getChildren() { return Collections.singleton(new ChildScorer(reqScorer, "FILTERED")); } Index: lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java (revision 1363128) +++ lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java (working copy) @@ -57,6 +57,8 @@ private final Similarity.ExactSimScorer docScorer; + private final long cost; + ExactPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings, Similarity.ExactSimScorer docScorer) throws IOException { super(weight); @@ -65,7 +67,10 @@ chunkStates = new ChunkState[postings.length]; endMinus1 = postings.length-1; - + // minimum docfreq * # of terms + // TODO: should we use totalTermFreq instead? + // and what about the optimization below? + cost = postings[0].docFreq * (long)postings.length; for(int i=0;i> 1) - 1; i >= 0; i--) { + for (int i = (numScorers >>> 1) - 1; i >= 0; i--) { heapAdjust(i); } } @@ -52,7 +52,7 @@ Scorer scorer = subScorers[root]; int doc = scorer.docID(); int i = root; - while (i <= (numScorers >> 1) - 1) { + while (i <= (numScorers >>> 1) - 1) { int lchild = (i << 1) + 1; Scorer lscorer = subScorers[lchild]; int ldoc = lscorer.docID(); @@ -105,4 +105,13 @@ } return children; } + + @Override + public long estimateCost() { + long cost = 0; + for (int i = 0; i < numScorers; i++) { + cost += subScorers[i].estimateCost(); + } + return cost; + } } Index: lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java (revision 1363128) +++ lucene/core/src/test/org/apache/lucene/search/TestScoreCachingWrappingScorer.java (working copy) @@ -59,6 +59,8 @@ return doc < scores.length ? doc : NO_MORE_DOCS; } + @Override public long estimateCost() { return scores.length; } + } private static final class ScoreCachingCollector extends Collector { Index: lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java (revision 1363128) +++ lucene/core/src/test/org/apache/lucene/search/TestBooleanScorer.java (working copy) @@ -92,6 +92,8 @@ return doc = target <= 3000 ? 3000 : NO_MORE_DOCS; } + @Override public long estimateCost() { return 1; } + }}; BooleanScorer bs = new BooleanScorer(weight, false, 1, Arrays.asList(scorers), null, scorers.length); Index: lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java (revision 1363128) +++ lucene/core/src/test/org/apache/lucene/search/TestCachingCollector.java (working copy) @@ -47,6 +47,8 @@ @Override public int advance(int target) throws IOException { return 0; } + @Override public long estimateCost() { return 0; } + } private static class NoOpCollector extends Collector { Index: lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java (revision 1363128) +++ lucene/core/src/test/org/apache/lucene/search/TestPositiveScoresOnlyCollector.java (working copy) @@ -50,6 +50,8 @@ idx = target; return idx < scores.length ? idx : NO_MORE_DOCS; } + + @Override public long estimateCost() { return 1; } } // The scores must have positive as well as negative values Index: lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java (revision 1363128) +++ lucene/core/src/test/org/apache/lucene/search/JustCompileSearch.java (working copy) @@ -198,6 +198,11 @@ protected float phraseFreq() { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } + + @Override + public long estimateCost() { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } } @@ -245,6 +250,12 @@ public int advance(int target) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } + + @Override + public long estimateCost() { + throw new UnsupportedOperationException(UNSUPPORTED_MSG); + } + } static final class JustCompileSimilarity extends Similarity { Index: solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java =================================================================== --- solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (revision 1363128) +++ solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (working copy) @@ -249,7 +249,7 @@ // Although this set only includes live docs, other filters can be pushed down to queries. DocIdSet readerSet = filter.getDocIdSet(context, acceptDocs); if (readerSet == null) readerSet=DocIdSet.EMPTY_DOCIDSET; - return new JoinScorer(this, readerSet.iterator(), getBoost()); + return new JoinScorer(this, readerSet.iterator(), getBoost(), context.reader().maxDoc()); } @@ -510,11 +510,13 @@ protected static class JoinScorer extends Scorer { final DocIdSetIterator iter; final float score; + final long cost; int doc = -1; - public JoinScorer(Weight w, DocIdSetIterator iter, float score) throws IOException { + public JoinScorer(Weight w, DocIdSetIterator iter, float score, long cost) throws IOException { super(w); this.score = score; + this.cost = cost; this.iter = iter==null ? DocIdSet.EMPTY_DOCIDSET.iterator() : iter; } @@ -542,6 +544,12 @@ public int advance(int target) throws IOException { return iter.advance(target); } + + @Override + public long estimateCost() { + return cost; + } + } Index: solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java =================================================================== --- solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java (revision 1363128) +++ solr/core/src/java/org/apache/solr/search/SolrConstantScoreQuery.java (working copy) @@ -153,6 +153,7 @@ final DocIdSetIterator docIdSetIterator; final float theScore; final Bits acceptDocs; + final long cost; int doc = -1; public ConstantScorer(AtomicReaderContext context, ConstantWeight w, float theScore, Bits acceptDocs) throws IOException { @@ -162,12 +163,15 @@ DocIdSet docIdSet = filter instanceof SolrFilter ? ((SolrFilter)filter).getDocIdSet(w.context, context, acceptDocs) : filter.getDocIdSet(context, acceptDocs); if (docIdSet == null) { docIdSetIterator = DocIdSet.EMPTY_DOCIDSET.iterator(); + cost = 0; } else { DocIdSetIterator iter = docIdSet.iterator(); if (iter == null) { docIdSetIterator = DocIdSet.EMPTY_DOCIDSET.iterator(); + cost = 0; } else { docIdSetIterator = iter; + cost = context.reader().maxDoc(); } } } @@ -196,6 +200,11 @@ public int advance(int target) throws IOException { return docIdSetIterator.advance(target); } + + @Override + public long estimateCost() { + return cost; + } } @Override Index: solr/core/src/java/org/apache/solr/schema/LatLonType.java =================================================================== --- solr/core/src/java/org/apache/solr/schema/LatLonType.java (revision 1363128) +++ solr/core/src/java/org/apache/solr/schema/LatLonType.java (working copy) @@ -490,6 +490,11 @@ return 1; } + @Override + public long estimateCost() { + return maxDoc; + } + public Explanation explain(int doc) throws IOException { advance(doc); boolean matched = this.doc == doc;