diff --git lucene/src/java/org/apache/lucene/search/BooleanQuery.java lucene/src/java/org/apache/lucene/search/BooleanQuery.java index ecddb66..4fba167 100644 --- lucene/src/java/org/apache/lucene/search/BooleanQuery.java +++ lucene/src/java/org/apache/lucene/search/BooleanQuery.java @@ -206,11 +206,11 @@ public class BooleanQuery extends Query implements Iterable { } @Override - public void normalize(float norm) { - norm *= getBoost(); // incorporate boost + public void normalize(float norm, float topLevelBoost) { + topLevelBoost *= getBoost(); // incorporate boost for (Weight w : weights) { // normalize all clauses, (even if prohibited in case of side affects) - w.normalize(norm); + w.normalize(norm, topLevelBoost); } } diff --git lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java index d5f5f50..3e5ed02 100644 --- lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java +++ lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java @@ -123,11 +123,11 @@ public class ConstantScoreQuery extends Query { } @Override - public void normalize(float norm) { - this.queryNorm = norm; + public void normalize(float norm, float topLevelBoost) { + this.queryNorm = norm * topLevelBoost; queryWeight *= this.queryNorm; // we normalize the inner weight, but ignore it (just to initialize everything) - if (innerWeight != null) innerWeight.normalize(norm); + if (innerWeight != null) innerWeight.normalize(norm, topLevelBoost); } @Override diff --git lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java index 0434232..618930c 100644 --- lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java +++ lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java @@ -130,10 +130,10 @@ public class DisjunctionMaxQuery extends Query implements Iterable { /** Apply the computed normalization factor to our subqueries */ @Override - public void normalize(float norm) { - norm *= getBoost(); // Incorporate our boost + public void normalize(float norm, float topLevelBoost) { + topLevelBoost *= getBoost(); // Incorporate our boost for (Weight wt : weights) { - wt.normalize(norm); + wt.normalize(norm, topLevelBoost); } } diff --git lucene/src/java/org/apache/lucene/search/FilteredQuery.java lucene/src/java/org/apache/lucene/search/FilteredQuery.java index 1bcd845..648b9d0 100644 --- lucene/src/java/org/apache/lucene/search/FilteredQuery.java +++ lucene/src/java/org/apache/lucene/search/FilteredQuery.java @@ -75,8 +75,8 @@ extends Query { } @Override - public void normalize (float v) { - weight.normalize(v); + public void normalize (float norm, float topLevelBoost) { + weight.normalize(norm, topLevelBoost); value = weight.getValue() * getBoost(); } diff --git lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java index 40cdc18..95a9aff 100644 --- lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java +++ lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java @@ -123,8 +123,8 @@ public class MatchAllDocsQuery extends Query { } @Override - public void normalize(float queryNorm) { - this.queryNorm = queryNorm; + public void normalize(float queryNorm, float topLevelBoost) { + this.queryNorm = queryNorm * topLevelBoost; queryWeight *= this.queryNorm; } diff --git lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java index 296dfa0..52306ff 100644 --- lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java +++ lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java @@ -18,21 +18,28 @@ package org.apache.lucene.search; */ import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.ListIterator; +import java.util.Set; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexReader.ReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.search.Explanation.IDFExplanation; +import org.apache.lucene.search.Similarity.Stats; import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.PriorityQueue; import org.apache.lucene.util.TermContext; import org.apache.lucene.util.ToStringUtils; -import org.apache.lucene.util.PriorityQueue; -import org.apache.lucene.util.Bits; /** * MultiPhraseQuery is a generalized version of PhraseQuery, with an added @@ -132,12 +139,11 @@ public class MultiPhraseQuery extends Query { private class MultiPhraseWeight extends Weight { private Similarity similarity; - private float value; - private final IDFExplanation idfExp; - private float idf; private float queryNorm; private float queryWeight; + private final Stats stats; + // nocommit: MultiPhraseWeight must not know of idf. public MultiPhraseWeight(IndexSearcher searcher) throws IOException { this.similarity = searcher.getSimilarityProvider().get(field); @@ -150,27 +156,26 @@ public class MultiPhraseQuery extends Query { allTerms.add(TermContext.build(context, term, true)); } } - idfExp = similarity.computeWeight(searcher, field, allTerms.toArray(new TermContext[allTerms.size()])); - idf = idfExp.getIdf(); + this.stats = similarity.computeStats(searcher, field, + allTerms.toArray(new TermContext[allTerms.size()])); } @Override public Query getQuery() { return MultiPhraseQuery.this; } @Override - public float getValue() { return value; } + public float getValue() { return queryWeight; } @Override public float sumOfSquaredWeights() { - queryWeight = idf * getBoost(); // compute query weight - return queryWeight * queryWeight; // square it + queryWeight = stats.getWeight() * getBoost(); // compute query weight + return queryWeight * queryWeight; // square it } @Override - public void normalize(float queryNorm) { - this.queryNorm = queryNorm; - queryWeight *= queryNorm; // normalize query weight - value = queryWeight * idf; // idf for document + public void normalize(float queryNorm, float topLevelBoost) { + this.queryNorm = queryNorm * topLevelBoost; + queryWeight *= this.queryNorm; // normalize query weight } @Override @@ -248,7 +253,10 @@ public class MultiPhraseQuery extends Query { ComplexExplanation result = new ComplexExplanation(); result.setDescription("weight("+getQuery()+" in "+doc+"), product of:"); - Explanation idfExpl = new Explanation(idf, "idf(" + field + ":" + idfExp.explain() +")"); + // nocommit: once again, idf proves difficult to get rid of + Explanation simExpl = stats.explainWeight(); + Explanation idfExpl = new Explanation(simExpl.getValue(), + "idf(" + field + ":" + simExpl.getDescription() + ")"); // explain query weight Explanation queryExpl = new Explanation(); diff --git lucene/src/java/org/apache/lucene/search/PhraseQuery.java lucene/src/java/org/apache/lucene/search/PhraseQuery.java index 092300e..1c0cb2e 100644 --- lucene/src/java/org/apache/lucene/search/PhraseQuery.java +++ lucene/src/java/org/apache/lucene/search/PhraseQuery.java @@ -18,21 +18,21 @@ package org.apache.lucene.search; */ import java.io.IOException; -import java.util.Set; import java.util.ArrayList; +import java.util.Set; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.IndexReader.ReaderContext; -import org.apache.lucene.index.Term; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.Explanation.IDFExplanation; -import org.apache.lucene.util.TermContext; -import org.apache.lucene.util.ToStringUtils; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.IndexReader.ReaderContext; +import org.apache.lucene.search.Similarity.Stats; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.TermContext; +import org.apache.lucene.util.ToStringUtils; /** A Query that matches documents containing a particular sequence of terms. * A PhraseQuery is built by QueryParser for input like "new york". @@ -174,13 +174,12 @@ public class PhraseQuery extends Query { private class PhraseWeight extends Weight { private final Similarity similarity; - private float value; - private float idf; private float queryNorm; private float queryWeight; - private IDFExplanation idfExp; + private final Stats stats; private transient TermContext states[]; + // nocommit: PhraseWeight must not know of idf. public PhraseWeight(IndexSearcher searcher) throws IOException { this.similarity = searcher.getSimilarityProvider().get(field); @@ -188,8 +187,7 @@ public class PhraseQuery extends Query { states = new TermContext[terms.size()]; for (int i = 0; i < terms.size(); i++) states[i] = TermContext.build(context, terms.get(i), true); - idfExp = similarity.computeWeight(searcher, field, states); - idf = idfExp.getIdf(); + this.stats = similarity.computeStats(searcher, field, states); } @Override @@ -199,19 +197,18 @@ public class PhraseQuery extends Query { public Query getQuery() { return PhraseQuery.this; } @Override - public float getValue() { return value; } + public float getValue() { return queryWeight; } @Override public float sumOfSquaredWeights() { - queryWeight = idf * getBoost(); // compute query weight - return queryWeight * queryWeight; // square it + queryWeight = stats.getWeight() * getBoost(); // compute query weight + return queryWeight * queryWeight; // square it } @Override - public void normalize(float queryNorm) { - this.queryNorm = queryNorm; - queryWeight *= queryNorm; // normalize query weight - value = queryWeight * idf; // idf for document + public void normalize(float queryNorm, float topLevelBoost) { + this.queryNorm = queryNorm * topLevelBoost; + queryWeight *= this.queryNorm; // normalize query weight } @Override @@ -280,7 +277,7 @@ public class PhraseQuery extends Query { StringBuilder docFreqs = new StringBuilder(); StringBuilder query = new StringBuilder(); query.append('\"'); - docFreqs.append(idfExp.explain()); + docFreqs.append(stats.explainWeight().getDescription()); for (int i = 0; i < terms.size(); i++) { if (i != 0) { query.append(" "); @@ -292,8 +289,9 @@ public class PhraseQuery extends Query { } query.append('\"'); + // nocommit: this is definitely idf-related, so the name remains for now Explanation idfExpl = - new Explanation(idf, "idf(" + field + ":" + docFreqs + ")"); + new Explanation(stats.getWeight(), "idf(" + field + ":" + docFreqs + ")"); // explain query weight Explanation queryExpl = new Explanation(); diff --git lucene/src/java/org/apache/lucene/search/Query.java lucene/src/java/org/apache/lucene/search/Query.java index 40ec80d..f22d560 100644 --- lucene/src/java/org/apache/lucene/search/Query.java +++ lucene/src/java/org/apache/lucene/search/Query.java @@ -101,7 +101,7 @@ public abstract class Query implements Cloneable { float norm = searcher.getSimilarityProvider().queryNorm(sum); if (Float.isInfinite(norm) || Float.isNaN(norm)) norm = 1.0f; - weight.normalize(norm); + weight.normalize(norm, 1.0f); return weight; } diff --git lucene/src/java/org/apache/lucene/search/Similarity.java lucene/src/java/org/apache/lucene/search/Similarity.java index ce34f90..07fbd6b 100644 --- lucene/src/java/org/apache/lucene/search/Similarity.java +++ lucene/src/java/org/apache/lucene/search/Similarity.java @@ -22,8 +22,6 @@ import java.io.IOException; import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.search.Explanation.IDFExplanation; -import org.apache.lucene.search.spans.SpanQuery; // javadoc import org.apache.lucene.util.TermContext; @@ -115,17 +113,10 @@ public abstract class Similarity { return 1; } - public abstract IDFExplanation computeWeight(IndexSearcher searcher, String fieldName, TermContext... termStats) throws IOException; + public abstract Stats computeStats(IndexSearcher searcher, String fieldName, TermContext... termStats) throws IOException; - /** - * returns a new {@link Similarity.ExactDocScorer}. - */ - public abstract ExactDocScorer exactDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException; - - /** - * returns a new {@link Similarity.SloppyDocScorer}. - */ - public abstract SloppyDocScorer sloppyDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException; + public abstract ExactDocScorer exactDocScorer(Weight weight, Stats stats, String fieldName, AtomicReaderContext context) throws IOException; + public abstract SloppyDocScorer sloppyDocScorer(Weight weight, Stats stats, String fieldName, AtomicReaderContext context) throws IOException; /** * API for scoring exact queries such as {@link TermQuery} and @@ -158,4 +149,32 @@ public abstract class Similarity { */ public abstract float score(int doc, float freq); } + + /** Stores the statistics for the indexed collection. This abstract + * implementation is empty; descendants of {@code Similarity} should + * subclass {@code Stats} and define the statistics they require in the + * subclass. Examples include idf, average field length, etc. + */ + public static abstract class Stats { + /** The query boost factor. */ + protected float boost = 1.0f; + + /** The "weight" portion of the query; that is, the part that is taken into + * account for the query norm. */ + // nocommit: to Similarity? + public abstract float getWeight(); + + /** Returns the explanations for the different scoring factors. */ + // nocommit: to Similarity.explainWeight(Stats)? + // nocommit: to make much more general (what is "weight"?) + public abstract Explanation explainWeight(); + + /** Gets the boost for the enclosing query clause. Documents matching + * this clause will (in addition to the normal weightings) have their score + * multiplied by b. The boost is 1.0 by default. + */ + public float getBoost() { + return boost; + } + } } diff --git lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java index daf991b..b97d274 100644 --- lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java +++ lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java @@ -21,11 +21,11 @@ package org.apache.lucene.search; import java.io.IOException; import java.io.Serializable; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.Term; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.search.Explanation.IDFExplanation; -import org.apache.lucene.util.TermContext; import org.apache.lucene.util.SmallFloat; +import org.apache.lucene.util.TermContext; /** @@ -687,27 +687,29 @@ public abstract class TFIDFSimilarity extends Similarity implements Serializable } @Override - public final IDFExplanation computeWeight(IndexSearcher searcher, String fieldName, + public final IDFStats computeStats(IndexSearcher searcher, String fieldName, TermContext... termStats) throws IOException { - return termStats.length == 1 - ? idfExplain(termStats[0], searcher) - : idfExplain(termStats, searcher); + return new IDFStats(termStats.length == 1 + ? idfExplain(termStats[0], searcher) + : idfExplain(termStats, searcher)); } @Override - public final ExactDocScorer exactDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException { + public final ExactDocScorer exactDocScorer(Weight weight, Stats stats, String fieldName, AtomicReaderContext context) throws IOException { final byte norms[] = context.reader.norms(fieldName); + final float weightedIdf = ((IDFStats)stats).idf.getIdf() * weight.getValue(); return norms == null - ? new RawExactTFIDFDocScorer(weight.getValue()) - : new ExactTFIDFDocScorer(weight.getValue(), norms); + ? new RawExactTFIDFDocScorer(weightedIdf) + : new ExactTFIDFDocScorer(weightedIdf, norms); } @Override - public final SloppyDocScorer sloppyDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException { + public final SloppyDocScorer sloppyDocScorer(Weight weight, Stats stats, String fieldName, AtomicReaderContext context) throws IOException { final byte norms[] = context.reader.norms(fieldName); + final float weightedIdf = ((IDFStats)stats).idf.getIdf() * weight.getValue(); return norms == null - ? new RawSloppyTFIDFDocScorer(weight.getValue()) - : new SloppyTFIDFDocScorer(weight.getValue(), norms); + ? new RawSloppyTFIDFDocScorer(weightedIdf) + : new SloppyTFIDFDocScorer(weightedIdf, norms); } // nocommit: below are specialized classes, we should test if it really helps to avoid the 'if' for omitNorms, etc @@ -783,4 +785,26 @@ public abstract class TFIDFSimilarity extends Similarity implements Serializable return tf(freq)*weightValue; // compute tf(f)*weight } } + + /** Collection statistics for the TF-IDF model. The only statistic of interest + * to this model is idf. */ + public static class IDFStats extends Stats { + /** The idf and its explanation. */ + public final IDFExplanation idf; + + public IDFStats(IDFExplanation idf) { + // TODO: Validate? + this.idf = idf; + } + + @Override + public float getWeight() { + return idf.getIdf(); + } + + @Override + public Explanation explainWeight() { + return new Explanation(idf.getIdf(), idf.explain()); + } + } } diff --git lucene/src/java/org/apache/lucene/search/TermQuery.java lucene/src/java/org/apache/lucene/search/TermQuery.java index 7d7a305..2f64a91 100644 --- lucene/src/java/org/apache/lucene/search/TermQuery.java +++ lucene/src/java/org/apache/lucene/search/TermQuery.java @@ -22,15 +22,15 @@ import java.util.Set; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; import org.apache.lucene.index.TermState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexReader.ReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.Explanation.IDFExplanation; +import org.apache.lucene.search.Similarity.Stats; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.TermContext; import org.apache.lucene.util.ReaderUtil; +import org.apache.lucene.util.TermContext; import org.apache.lucene.util.ToStringUtils; /** A Query that matches documents containing a term. @@ -43,11 +43,9 @@ public class TermQuery extends Query { private class TermWeight extends Weight { private final Similarity similarity; - private float value; - private final float idf; private float queryNorm; private float queryWeight; - private final IDFExplanation idfExp; + private final Stats stats; private transient TermContext termStates; public TermWeight(IndexSearcher searcher, TermContext termStates) @@ -55,8 +53,7 @@ public class TermQuery extends Query { assert termStates != null : "TermContext must not be null"; this.termStates = termStates; this.similarity = searcher.getSimilarityProvider().get(term.field()); - idfExp = similarity.computeWeight(searcher, term.field(), termStates); - idf = idfExp.getIdf(); + this.stats = similarity.computeStats(searcher, term.field(), termStates); } @Override @@ -66,19 +63,18 @@ public class TermQuery extends Query { public Query getQuery() { return TermQuery.this; } @Override - public float getValue() { return value; } + public float getValue() { return queryWeight; } @Override public float sumOfSquaredWeights() { - queryWeight = idf * getBoost(); // compute query weight - return queryWeight * queryWeight; // square it + queryWeight = stats.getWeight() * getBoost(); // compute query weight + return queryWeight * queryWeight; // square it } @Override - public void normalize(float queryNorm) { - this.queryNorm = queryNorm; - queryWeight *= queryNorm; // normalize query weight - value = queryWeight * idf; // idf for document + public void normalize(float queryNorm, float topLevelBoost) { + this.queryNorm = queryNorm * topLevelBoost; + queryWeight *= this.queryNorm; // normalize query weight } @Override @@ -116,8 +112,6 @@ public class TermQuery extends Query { ComplexExplanation result = new ComplexExplanation(); result.setDescription("weight("+getQuery()+" in "+doc+"), product of:"); - Explanation expl = new Explanation(idf, idfExp.explain()); - // explain query weight Explanation queryExpl = new Explanation(); queryExpl.setDescription("queryWeight(" + getQuery() + "), product of:"); @@ -125,13 +119,15 @@ public class TermQuery extends Query { Explanation boostExpl = new Explanation(getBoost(), "boost"); if (getBoost() != 1.0f) queryExpl.addDetail(boostExpl); - queryExpl.addDetail(expl); + + Explanation simExpl = stats.explainWeight(); + queryExpl.addDetail(simExpl); Explanation queryNormExpl = new Explanation(queryNorm,"queryNorm"); queryExpl.addDetail(queryNormExpl); queryExpl.setValue(boostExpl.getValue() * - expl.getValue() * + simExpl.getValue() * queryNormExpl.getValue()); result.addDetail(queryExpl); @@ -157,7 +153,7 @@ public class TermQuery extends Query { tfExplanation.setDescription("no matching term"); } fieldExpl.addDetail(tfExplanation); - fieldExpl.addDetail(expl); + fieldExpl.addDetail(simExpl); Explanation fieldNormExpl = new Explanation(); final byte[] fieldNorms = reader.norms(field); @@ -169,7 +165,7 @@ public class TermQuery extends Query { fieldExpl.setMatch(Boolean.valueOf(tfExplanation.isMatch())); fieldExpl.setValue(tfExplanation.getValue() * - expl.getValue() * + simExpl.getValue() * fieldNormExpl.getValue()); result.addDetail(fieldExpl); diff --git lucene/src/java/org/apache/lucene/search/Weight.java lucene/src/java/org/apache/lucene/search/Weight.java index 3fb8927..0dac19d 100644 --- lucene/src/java/org/apache/lucene/search/Weight.java +++ lucene/src/java/org/apache/lucene/search/Weight.java @@ -72,7 +72,7 @@ public abstract class Weight { public abstract float getValue(); /** Assigns the query normalization factor to this. */ - public abstract void normalize(float norm); + public abstract void normalize(float norm, float topLevelBoost); /** * Returns a {@link Scorer} which scores documents in/out-of order according diff --git lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java index 8a5ba9a..f06f00d 100755 --- lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java +++ lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java @@ -224,14 +224,14 @@ public class CustomScoreQuery extends Query { /*(non-Javadoc) @see org.apache.lucene.search.Weight#normalize(float) */ @Override - public void normalize(float norm) { - norm *= getBoost(); // incorporate boost - subQueryWeight.normalize(norm); + public void normalize(float norm, float topLevelBoost) { + topLevelBoost *= getBoost(); // incorporate boost + subQueryWeight.normalize(norm, topLevelBoost); for(int i = 0; i < valSrcWeights.length; i++) { if (qStrict) { - valSrcWeights[i].normalize(1); // do not normalize the ValueSource part + valSrcWeights[i].normalize(1, 1); // do not normalize the ValueSource part } else { - valSrcWeights[i].normalize(norm); + valSrcWeights[i].normalize(norm, topLevelBoost); } } } diff --git lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java index 4f26ee0..af8ff19 100644 --- lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java +++ lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java @@ -91,8 +91,8 @@ public class ValueSourceQuery extends Query { /*(non-Javadoc) @see org.apache.lucene.search.Weight#normalize(float) */ @Override - public void normalize(float norm) { - this.queryNorm = norm; + public void normalize(float norm, float topLevelBoost) { + this.queryNorm = norm * topLevelBoost; queryWeight *= this.queryNorm; } diff --git lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java index f2b084a..a36a81b 100644 --- lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java +++ lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java @@ -17,31 +17,36 @@ package org.apache.lucene.search.spans; * limitations under the License. */ -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.IndexReader.ReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.search.*; -import org.apache.lucene.search.Explanation.IDFExplanation; -import org.apache.lucene.util.TermContext; - import java.io.IOException; import java.util.Set; import java.util.TreeSet; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.IndexReader.ReaderContext; +import org.apache.lucene.search.ComplexExplanation; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.Weight; +import org.apache.lucene.search.Similarity.Stats; +import org.apache.lucene.util.TermContext; + /** * Expert-only. Public for use by other weight implementations */ public class SpanWeight extends Weight { protected Similarity similarity; - protected float value; - protected float idf; protected float queryNorm; protected float queryWeight; + private final Stats stats; protected Set terms; protected SpanQuery query; - private IDFExplanation idfExp; + // nocommit: SpanWeight must not know of idf. public SpanWeight(SpanQuery query, IndexSearcher searcher) throws IOException { this.similarity = searcher.getSimilarityProvider().get(query.getField()); @@ -54,27 +59,25 @@ public class SpanWeight extends Weight { int i = 0; for (Term term : terms) states[i++] = TermContext.build(context, term, true); - idfExp = similarity.computeWeight(searcher, query.getField(), states); - idf = idfExp.getIdf(); + this.stats = similarity.computeStats(searcher, query.getField(), states); } @Override public Query getQuery() { return query; } @Override - public float getValue() { return value; } + public float getValue() { return queryWeight; } @Override public float sumOfSquaredWeights() throws IOException { - queryWeight = idf * query.getBoost(); // compute query weight - return queryWeight * queryWeight; // square it + queryWeight = stats.getWeight() * query.getBoost(); // compute query weight + return queryWeight * queryWeight; // square it } @Override - public void normalize(float queryNorm) { - this.queryNorm = queryNorm; - queryWeight *= queryNorm; // normalize query weight - value = queryWeight * idf; // idf for document + public void normalize(float queryNorm, float topLevelBoost) { + this.queryNorm = queryNorm * topLevelBoost; + queryWeight *= this.queryNorm; // normalize query weight } @Override @@ -90,8 +93,10 @@ public class SpanWeight extends Weight { result.setDescription("weight("+getQuery()+" in "+doc+"), product of:"); String field = ((SpanQuery)getQuery()).getField(); - Explanation idfExpl = - new Explanation(idf, "idf(" + field + ": " + idfExp.explain() + ")"); + // nocommit: once again, idf proves difficult to get rid of + Explanation simExpl = stats.explainWeight(); + Explanation idfExpl = new Explanation(simExpl.getValue(), + "idf(" + field + ":" + simExpl.getDescription() + ")"); // explain query weight Explanation queryExpl = new Explanation(); diff --git lucene/src/test/org/apache/lucene/search/JustCompileSearch.java lucene/src/test/org/apache/lucene/search/JustCompileSearch.java index efc69dd..34ef577 100644 --- lucene/src/test/org/apache/lucene/search/JustCompileSearch.java +++ lucene/src/test/org/apache/lucene/search/JustCompileSearch.java @@ -354,7 +354,7 @@ final class JustCompileSearch { } @Override - public void normalize(float norm) { + public void normalize(float norm, float topLevelBoost) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } diff --git lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java index 1e12cde..131222b 100644 --- lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java +++ lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java @@ -20,12 +20,12 @@ package org.apache.lucene.search; import java.io.IOException; import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.IndexReader.ReaderContext; import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.IndexReader.ReaderContext; import org.apache.lucene.search.Explanation.IDFExplanation; -import org.apache.lucene.util.ReaderUtil; -import org.apache.lucene.util.TermContext; +import org.apache.lucene.search.TFIDFSimilarity.IDFStats; import org.apache.lucene.util.SmallFloat; +import org.apache.lucene.util.TermContext; /** * BM25 Similarity. @@ -36,6 +36,10 @@ public class MockBM25Similarity extends Similarity { private static final float k1 = 2f; private static final float b = 0.75f; + /** The collection statistics. */ + // nocommit: I am pretty sure won't be good -- at least in the provider, we only have 1 object. + private BM25Stats stats; + /** * Our normalization is k1 * ((1 - b) + b * numTerms / avgNumTerms) * currently we put doclen into the boost byte (divided by boost) for simple quantization @@ -74,11 +78,27 @@ public class MockBM25Similarity extends Similarity { return 1.0f / (distance + 1); } + @Override + public BM25Stats computeStats(IndexSearcher searcher, String fieldName, TermContext... termStats) throws IOException { + stats = new BM25Stats(idfExplain(searcher, termStats), + avgFieldLength(searcher, fieldName)); + return stats; + } + + private float avgFieldLength(IndexSearcher searcher, String fieldName) throws IOException { +// context = ReaderUtil.getTopLevelContext(context); + ReaderContext context = searcher.getTopReaderContext(); + long normsum = context.reader.getSumOfNorms(fieldName); + long maxdoc = context.reader.maxDoc(); + int avgnorm = (int) (normsum / (double) maxdoc); + return decodeNormValue((byte)avgnorm); + } + + /** Computes the IDF. */ // weight for a term as log(1 + ((n - dfj + 0.5F)/(dfj + 0.5F))) // nocommit: nuke IDFExplanation! // nocommit: are we summing this in the right place for phrase estimation???? - @Override - public IDFExplanation computeWeight(IndexSearcher searcher, String fieldName, TermContext... termStats) throws IOException { + private IDFExplanation idfExplain(IndexSearcher searcher, TermContext... termStats) { float value = 0.0f; final StringBuilder exp = new StringBuilder(); @@ -101,32 +121,25 @@ public class MockBM25Similarity extends Similarity { public String explain() { return exp.toString(); } - }; + }; } - + @Override - public ExactDocScorer exactDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException { + public ExactDocScorer exactDocScorer(Weight weight, Stats stats, String fieldName, AtomicReaderContext context) throws IOException { byte[] norms = context.reader.norms(fieldName); - float avgdl = norms == null ? 0f : avgDocumentLength(fieldName, context); - return new ExactBM25DocScorer((float) Math.sqrt(weight.getValue()), norms, avgdl); + float weightedIdf = weight.getValue() * ((BM25Stats)stats).idf.getIdf(); + float avgdl = norms == null ? 0f : ((BM25Stats)stats).avgdl; + return new ExactBM25DocScorer(weightedIdf, norms, avgdl); } @Override - public SloppyDocScorer sloppyDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException { + public SloppyDocScorer sloppyDocScorer(Weight weight, Stats stats, String fieldName, AtomicReaderContext context) throws IOException { byte[] norms = context.reader.norms(fieldName); - float avgdl = norms == null ? 0f : avgDocumentLength(fieldName, context); - return new SloppyBM25DocScorer((float) Math.sqrt(weight.getValue()), norms, avgdl); + float weightedIdf = weight.getValue() * ((BM25Stats)stats).idf.getIdf(); + float avgdl = norms == null ? 0f : ((BM25Stats)stats).avgdl; + return new SloppyBM25DocScorer(weightedIdf, norms, avgdl); } - private float avgDocumentLength(String field, ReaderContext context) throws IOException { - // nocommit: crap that we calc this over and over redundantly for each segment (we should just do it once in the weight, once its generalized) - context = ReaderUtil.getTopLevelContext(context); - long normsum = context.reader.getSumOfNorms(field); - long maxdoc = context.reader.maxDoc(); - int avgnorm = (int) (normsum / (double) maxdoc); - return decodeNormValue((byte)avgnorm); - } - private class ExactBM25DocScorer extends ExactDocScorer { private final float weightValue; private final byte[] norms; @@ -164,4 +177,15 @@ public class MockBM25Similarity extends Similarity { return weightValue * (freq * (k1 + 1)) / (freq + norm); } } + + /** Collection statistics for the BM25 model. */ + public static class BM25Stats extends IDFStats { + /** The average document length. */ + public final float avgdl; + + public BM25Stats(IDFExplanation idf, float avgLength) { + super(idf); + this.avgdl = avgLength; + } + } } diff --git lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java index 99c0123..ea331ac 100644 --- lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java +++ lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.search.Explanation.IDFExplanation; +import org.apache.lucene.search.TFIDFSimilarity.IDFStats; import org.apache.lucene.util.TermContext; import org.apache.lucene.util.SmallFloat; @@ -88,11 +89,15 @@ public class MockLMSimilarity extends Similarity { return 1.0f / (distance + 1); } + @Override + public IDFStats computeStats(IndexSearcher searcher, String fieldName, TermContext... termStats) throws IOException { + return new IDFStats(idfExplain(searcher, fieldName, termStats)); + } + // weight for a term as 1 / (mu * (totalTermFrequency / sumOfTotalTermFrequency)) // nocommit: nuke IDFExplanation! // nocommit: evil how we shove this crap in weight and unsquare it.. need to generalize weight - @Override - public IDFExplanation computeWeight(IndexSearcher searcher, String fieldName, TermContext... termStats) throws IOException { + private IDFExplanation idfExplain(IndexSearcher searcher, String fieldName, TermContext... termStats) throws IOException { float value = 0.0f; final StringBuilder exp = new StringBuilder(); final long sumOfTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), fieldName).getSumTotalTermFreq(); @@ -114,21 +119,23 @@ public class MockLMSimilarity extends Similarity { public String explain() { return exp.toString(); } - }; + }; } @Override - public ExactDocScorer exactDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException { - float unsquaredWeight = (float) Math.sqrt(weight.getValue()); + public ExactDocScorer exactDocScorer(Weight weight, Stats stats, String fieldName, AtomicReaderContext context) throws IOException { + float weightedIdf = weight.getValue() * ((IDFStats)stats).idf.getIdf(); +// float unsquaredWeight = (float) Math.sqrt(weight.getValue()); final byte norms[] = context.reader.norms(fieldName); return norms == null - ? new RawExactMockLMDocScorer(unsquaredWeight) - : new ExactMockLMDocScorer(unsquaredWeight, norms); + ? new RawExactMockLMDocScorer(weightedIdf) + : new ExactMockLMDocScorer(weightedIdf, norms); } @Override - public SloppyDocScorer sloppyDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException { - return new SloppyMockLMDocScorer((float) Math.sqrt(weight.getValue()), context.reader.norms(fieldName)); + public SloppyDocScorer sloppyDocScorer(Weight weight, Stats stats, String fieldName, AtomicReaderContext context) throws IOException { + float weightedIdf = weight.getValue() * ((IDFStats)stats).idf.getIdf(); + return new SloppyMockLMDocScorer(weightedIdf, context.reader.norms(fieldName)); } /** diff --git solr/src/java/org/apache/solr/schema/LatLonType.java solr/src/java/org/apache/solr/schema/LatLonType.java index 75dbd65..2db7a50 100644 --- solr/src/java/org/apache/solr/schema/LatLonType.java +++ solr/src/java/org/apache/solr/schema/LatLonType.java @@ -365,8 +365,8 @@ class SpatialDistanceQuery extends Query { } @Override - public void normalize(float norm) { - this.queryNorm = norm; + public void normalize(float norm, float topLevelBoost) { + this.queryNorm = norm * topLevelBoost; queryWeight *= this.queryNorm; } diff --git solr/src/java/org/apache/solr/search/JoinQParserPlugin.java solr/src/java/org/apache/solr/search/JoinQParserPlugin.java index da42a25..2565d99 100644 --- solr/src/java/org/apache/solr/search/JoinQParserPlugin.java +++ solr/src/java/org/apache/solr/search/JoinQParserPlugin.java @@ -180,8 +180,8 @@ class JoinQuery extends Query { } @Override - public void normalize(float norm) { - this.queryNorm = norm; + public void normalize(float norm, float topLevelBoost) { + this.queryNorm = norm * topLevelBoost; queryWeight *= this.queryNorm; } diff --git solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java index 357ee66..7f16087 100755 --- solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java +++ solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java @@ -85,8 +85,8 @@ public class SolrConstantScoreQuery extends ConstantScoreQuery { } @Override - public void normalize(float norm) { - this.queryNorm = norm; + public void normalize(float norm, float topLevelBoost) { + this.queryNorm = norm * topLevelBoost; queryWeight *= this.queryNorm; } diff --git solr/src/java/org/apache/solr/search/function/BoostedQuery.java solr/src/java/org/apache/solr/search/function/BoostedQuery.java index c065aef..1ea7963 100755 --- solr/src/java/org/apache/solr/search/function/BoostedQuery.java +++ solr/src/java/org/apache/solr/search/function/BoostedQuery.java @@ -90,9 +90,9 @@ public class BoostedQuery extends Query { } @Override - public void normalize(float norm) { - norm *= getBoost(); - qWeight.normalize(norm); + public void normalize(float norm, float topLevelBoost) { + topLevelBoost *= getBoost(); + qWeight.normalize(norm, topLevelBoost); } @Override diff --git solr/src/java/org/apache/solr/search/function/FunctionQuery.java solr/src/java/org/apache/solr/search/function/FunctionQuery.java index e7d4be0..2bb0c0d 100644 --- solr/src/java/org/apache/solr/search/function/FunctionQuery.java +++ solr/src/java/org/apache/solr/search/function/FunctionQuery.java @@ -88,8 +88,8 @@ public class FunctionQuery extends Query { } @Override - public void normalize(float norm) { - this.queryNorm = norm; + public void normalize(float norm, float topLevelBoost) { + this.queryNorm = norm * topLevelBoost; queryWeight *= this.queryNorm; }