Index: solr/src/java/org/apache/solr/schema/LatLonType.java
===================================================================
--- solr/src/java/org/apache/solr/schema/LatLonType.java (revision 1136270)
+++ solr/src/java/org/apache/solr/schema/LatLonType.java (working copy)
@@ -354,12 +354,7 @@
}
@Override
- public float getValue() {
- return queryWeight;
- }
-
- @Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
queryWeight = getBoost();
return queryWeight * queryWeight;
}
@@ -372,7 +367,7 @@
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new SpatialScorer(context, this);
+ return new SpatialScorer(context, this, queryWeight);
}
@Override
@@ -405,10 +400,10 @@
int lastDistDoc;
double lastDist;
- public SpatialScorer(AtomicReaderContext readerContext, SpatialWeight w) throws IOException {
+ public SpatialScorer(AtomicReaderContext readerContext, SpatialWeight w, float qWeight) throws IOException {
super(w);
this.weight = w;
- this.qWeight = w.getValue();
+ this.qWeight = qWeight;
this.reader = readerContext.reader;
this.maxDoc = reader.maxDoc();
this.delDocs = reader.getDeletedDocs();
Index: solr/src/java/org/apache/solr/search/function/FunctionQuery.java
===================================================================
--- solr/src/java/org/apache/solr/search/function/FunctionQuery.java (revision 1136270)
+++ solr/src/java/org/apache/solr/search/function/FunctionQuery.java (working copy)
@@ -77,12 +77,7 @@
}
@Override
- public float getValue() {
- return queryWeight;
- }
-
- @Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
queryWeight = getBoost();
return queryWeight * queryWeight;
}
@@ -95,7 +90,7 @@
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new AllScorer(context, this);
+ return new AllScorer(context, this, queryWeight);
}
@Override
@@ -114,10 +109,10 @@
final boolean hasDeletions;
final Bits delDocs;
- public AllScorer(AtomicReaderContext context, FunctionWeight w) throws IOException {
+ public AllScorer(AtomicReaderContext context, FunctionWeight w, float qWeight) throws IOException {
super(w);
this.weight = w;
- this.qWeight = w.getValue();
+ this.qWeight = qWeight;
this.reader = context.reader;
this.maxDoc = reader.maxDoc();
this.hasDeletions = reader.hasDeletions();
Index: solr/src/java/org/apache/solr/search/function/BoostedQuery.java
===================================================================
--- solr/src/java/org/apache/solr/search/function/BoostedQuery.java (revision 1136270)
+++ solr/src/java/org/apache/solr/search/function/BoostedQuery.java (working copy)
@@ -78,13 +78,8 @@
}
@Override
- public float getValue() {
- return getBoost();
- }
-
- @Override
- public float sumOfSquaredWeights() throws IOException {
- float sum = qWeight.sumOfSquaredWeights();
+ public float getValueForNormalization() throws IOException {
+ float sum = qWeight.getValueForNormalization();
sum *= getBoost() * getBoost();
return sum ;
}
@@ -101,7 +96,7 @@
if(subQueryScorer == null) {
return null;
}
- return new BoostedQuery.CustomScorer(context, this, subQueryScorer, boostVal);
+ return new BoostedQuery.CustomScorer(context, this, getBoost(), subQueryScorer, boostVal);
}
@Override
@@ -128,11 +123,11 @@
private final DocValues vals;
private final AtomicReaderContext readerContext;
- private CustomScorer(AtomicReaderContext readerContext, BoostedQuery.BoostedWeight w,
+ private CustomScorer(AtomicReaderContext readerContext, BoostedQuery.BoostedWeight w, float qWeight,
Scorer scorer, ValueSource vs) throws IOException {
super(w);
this.weight = w;
- this.qWeight = w.getValue();
+ this.qWeight = qWeight;
this.scorer = scorer;
this.readerContext = readerContext;
this.vals = vs.getValues(weight.fcontext, readerContext);
Index: solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java
===================================================================
--- solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java (revision 1136270)
+++ solr/src/java/org/apache/solr/search/SolrConstantScoreQuery.java (working copy)
@@ -74,12 +74,7 @@
}
@Override
- public float getValue() {
- return queryWeight;
- }
-
- @Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
queryWeight = getBoost();
return queryWeight * queryWeight;
}
@@ -92,13 +87,13 @@
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new ConstantScorer(context, this);
+ return new ConstantScorer(context, this, queryWeight);
}
@Override
public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
- ConstantScorer cs = new ConstantScorer(context, this);
+ ConstantScorer cs = new ConstantScorer(context, this, queryWeight);
boolean exists = cs.docIdSetIterator.advance(doc) == doc;
ComplexExplanation result = new ComplexExplanation();
@@ -125,9 +120,9 @@
final float theScore;
int doc = -1;
- public ConstantScorer(AtomicReaderContext context, ConstantWeight w) throws IOException {
+ public ConstantScorer(AtomicReaderContext context, ConstantWeight w, float theScore) throws IOException {
super(w);
- theScore = w.getValue();
+ this.theScore = theScore;
DocIdSet docIdSet = filter instanceof SolrFilter ? ((SolrFilter)filter).getDocIdSet(w.context, context) : filter.getDocIdSet(context);
if (docIdSet == null) {
docIdSetIterator = DocIdSet.EMPTY_DOCIDSET.iterator();
Index: solr/src/java/org/apache/solr/search/JoinQParserPlugin.java
===================================================================
--- solr/src/java/org/apache/solr/search/JoinQParserPlugin.java (revision 1136270)
+++ solr/src/java/org/apache/solr/search/JoinQParserPlugin.java (working copy)
@@ -169,12 +169,8 @@
return JoinQuery.this;
}
- public float getValue() {
- return getBoost();
- }
-
@Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
queryWeight = getBoost();
return queryWeight * queryWeight;
}
@@ -224,7 +220,7 @@
DocIdSet readerSet = filter.getDocIdSet(context);
if (readerSet == null) readerSet=DocIdSet.EMPTY_DOCIDSET;
- return new JoinScorer(this, readerSet.iterator());
+ return new JoinScorer(this, readerSet.iterator(), getBoost());
}
@@ -515,9 +511,9 @@
final float score;
int doc = -1;
- public JoinScorer(Weight w, DocIdSetIterator iter) throws IOException {
+ public JoinScorer(Weight w, DocIdSetIterator iter, float score) throws IOException {
super(w);
- score = w.getValue();
+ this.score = score;
this.iter = iter==null ? DocIdSet.EMPTY_DOCIDSET.iterator() : iter;
}
Index: lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (revision 1136270)
+++ lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java (working copy)
@@ -135,8 +135,8 @@
static final class JustCompileSpanScorer extends SpanScorer {
protected JustCompileSpanScorer(Spans spans, Weight weight,
- Similarity similarity, String field, AtomicReaderContext context) throws IOException {
- super(spans, weight, similarity, field, context);
+ Similarity similarity, Similarity.SloppyDocScorer docScorer) throws IOException {
+ super(spans, weight, similarity, docScorer);
}
@Override
Index: lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (revision 1136270)
+++ lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (working copy)
@@ -187,8 +187,8 @@
static final class JustCompilePhraseScorer extends PhraseScorer {
JustCompilePhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
- Similarity similarity, String field, AtomicReaderContext context) throws IOException {
- super(weight, postings, similarity, field, context);
+ Similarity.SloppyDocScorer docScorer) throws IOException {
+ super(weight, postings, docScorer);
}
@Override
@@ -349,17 +349,12 @@
}
@Override
- public float getValue() {
- throw new UnsupportedOperationException(UNSUPPORTED_MSG);
- }
-
- @Override
public void normalize(float norm, float topLevelBoost) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
@Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
Index: lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java (revision 1136270)
+++ lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java (working copy)
@@ -20,10 +20,7 @@
import java.io.IOException;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.search.Explanation.IDFExplanation;
-import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.SmallFloat;
@@ -75,10 +72,9 @@
}
// weight for a term as log(1 + ((n - dfj + 0.5F)/(dfj + 0.5F)))
- // nocommit: nuke IDFExplanation!
- // nocommit: are we summing this in the right place for phrase estimation????
+ // TODO: are we summing this in the right place for phrase estimation????
@Override
- public IDFExplanation computeWeight(IndexSearcher searcher, String fieldName, TermContext... termStats) throws IOException {
+ public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException {
float value = 0.0f;
final StringBuilder exp = new StringBuilder();
@@ -91,40 +87,29 @@
exp.append(dfj);
}
- final float idfValue = value;
- return new IDFExplanation() {
- @Override
- public float getIdf() {
- return idfValue;
- }
- @Override
- public String explain() {
- return exp.toString();
- }
- };
+ return new BM25Stats(value, queryBoost, avgDocumentLength(searcher, fieldName));
}
@Override
- public ExactDocScorer exactDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException {
- byte[] norms = context.reader.norms(fieldName);
- float avgdl = norms == null ? 0f : avgDocumentLength(fieldName, context);
- return new ExactBM25DocScorer((float) Math.sqrt(weight.getValue()), norms, avgdl);
+ public ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new ExactBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName));
}
@Override
- public SloppyDocScorer sloppyDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException {
- byte[] norms = context.reader.norms(fieldName);
- float avgdl = norms == null ? 0f : avgDocumentLength(fieldName, context);
- return new SloppyBM25DocScorer((float) Math.sqrt(weight.getValue()), norms, avgdl);
+ public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new SloppyBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName));
}
- private float avgDocumentLength(String field, ReaderContext context) throws IOException {
- // nocommit: crap that we calc this over and over redundantly for each segment (we should just do it once in the weight, once its generalized)
- context = ReaderUtil.getTopLevelContext(context);
- long normsum = context.reader.getSumOfNorms(field);
- long maxdoc = context.reader.maxDoc();
- int avgnorm = (int) (normsum / (double) maxdoc);
- return decodeNormValue((byte)avgnorm);
+ /** return avg doc length across the field (zero if the field has no norms */
+ private float avgDocumentLength(IndexSearcher searcher, String field) throws IOException {
+ if (!searcher.reader.hasNorms(field)) {
+ return 0f;
+ } else {
+ long normsum = searcher.reader.getSumOfNorms(field);
+ long maxdoc = searcher.reader.maxDoc();
+ int avgnorm = (int) (normsum / (double) maxdoc);
+ return decodeNormValue((byte)avgnorm);
+ }
}
private class ExactBM25DocScorer extends ExactDocScorer {
@@ -132,10 +117,11 @@
private final byte[] norms;
private final float avgdl;
- ExactBM25DocScorer(float weightValue, byte norms[], float avgdl) {
- this.weightValue = weightValue;
+ ExactBM25DocScorer(BM25Stats stats, byte norms[]) {
+ // we incorporate boost here up front... maybe we should multiply by tf instead?
+ this.weightValue = stats.idf * stats.queryBoost * stats.topLevelBoost;
+ this.avgdl = stats.avgdl;
this.norms = norms;
- this.avgdl = avgdl;
}
// todo: optimize
@@ -151,10 +137,11 @@
private final byte[] norms;
private final float avgdl;
- SloppyBM25DocScorer(float weightValue, byte norms[], float avgdl) {
- this.weightValue = weightValue;
+ SloppyBM25DocScorer(BM25Stats stats, byte norms[]) {
+ // we incorporate boost here up front... maybe we should multiply by tf instead?
+ this.weightValue = stats.idf * stats.queryBoost * stats.topLevelBoost;
+ this.avgdl = stats.avgdl;
this.norms = norms;
- this.avgdl = avgdl;
}
// todo: optimize
@@ -164,4 +151,35 @@
return weightValue * (freq * (k1 + 1)) / (freq + norm);
}
}
+
+ /** Collection statistics for the BM25 model. */
+ public static class BM25Stats extends Stats {
+ /** BM25's idf */
+ private final float idf;
+ /** The average document length. */
+ private final float avgdl;
+ /** query's inner boost */
+ private final float queryBoost;
+ /** any outer query's boost */
+ private float topLevelBoost;
+
+ public BM25Stats(float idf, float queryBoost, float avgdl) {
+ this.idf = idf;
+ this.queryBoost = queryBoost;
+ this.avgdl = avgdl;
+ }
+
+ @Override
+ public float getValueForNormalization() {
+ // we return a TF-IDF like normalization to be nice, but we don't actually normalize ourselves.
+ final float queryWeight = idf * queryBoost;
+ return queryWeight * queryWeight;
+ }
+
+ @Override
+ public void normalize(float queryNorm, float topLevelBoost) {
+ // we don't normalize with queryNorm at all, we just capture the top-level boost
+ this.topLevelBoost = topLevelBoost;
+ }
+ }
}
Index: lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java (revision 1136270)
+++ lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java (working copy)
@@ -22,7 +22,6 @@
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.search.Explanation.IDFExplanation;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.SmallFloat;
@@ -89,10 +88,8 @@
}
// weight for a term as 1 / (mu * (totalTermFrequency / sumOfTotalTermFrequency))
- // nocommit: nuke IDFExplanation!
- // nocommit: evil how we shove this crap in weight and unsquare it.. need to generalize weight
@Override
- public IDFExplanation computeWeight(IndexSearcher searcher, String fieldName, TermContext... termStats) throws IOException {
+ public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException {
float value = 0.0f;
final StringBuilder exp = new StringBuilder();
final long sumOfTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), fieldName).getSumTotalTermFreq();
@@ -104,27 +101,17 @@
exp.append(totalTermFrequency);
}
- final float idfValue = value;
- return new IDFExplanation() {
- @Override
- public float getIdf() {
- return idfValue;
- }
- @Override
- public String explain() {
- return exp.toString();
- }
- };
+ return new LMStats(value, queryBoost);
}
@Override
- public ExactDocScorer exactDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException {
- return new ExactMockLMDocScorer((float) Math.sqrt(weight.getValue()), context.reader.norms(fieldName));
+ public ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new ExactMockLMDocScorer(((LMStats) stats).getValue(), context.reader.norms(fieldName));
}
@Override
- public SloppyDocScorer sloppyDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException {
- return new SloppyMockLMDocScorer((float) Math.sqrt(weight.getValue()), context.reader.norms(fieldName));
+ public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new SloppyMockLMDocScorer(((LMStats) stats).getValue(), context.reader.norms(fieldName));
}
/**
@@ -168,4 +155,33 @@
return norms == null ? raw : raw + decodeNormValue(norms[doc]);
}
}
+
+ public static class LMStats extends Stats {
+ // dunno if this idf-like thing has a real name, its part1 of the formula to me.
+ private final float part1;
+ private final float queryBoost;
+ private float value;
+
+ public LMStats(float part1, float queryBoost) {
+ this.part1 = part1;
+ this.queryBoost = queryBoost;
+ }
+
+ @Override
+ public float getValueForNormalization() {
+ // we return a TF-IDF like normalization to be nice, but we don't actually normalize ourselves.
+ final float queryWeight = part1 * queryBoost;
+ return queryWeight * queryWeight;
+ }
+
+ @Override
+ public void normalize(float queryNorm, float topLevelBoost) {
+ // set our value here
+ this.value = part1 * queryBoost * topLevelBoost;
+ }
+
+ public float getValue() {
+ return value;
+ }
+ }
}
Index: lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
===================================================================
--- lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java (revision 1136270)
+++ lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java (working copy)
@@ -338,7 +338,7 @@
query.add(new Term[] { new Term("body", "this"), new Term("body", "that") });
query.add(new Term("body", "is"));
Weight weight = query.createWeight(searcher);
- assertEquals(10f * 10f, weight.sumOfSquaredWeights(), 0.001f);
+ assertEquals(10f * 10f, weight.getValueForNormalization(), 0.001f);
writer.close();
searcher.close();
Index: lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (working copy)
@@ -110,14 +110,9 @@
}
@Override
- public float getValue() {
- return queryWeight;
- }
-
- @Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
// we calculate sumOfSquaredWeights of the inner weight, but ignore it (just to initialize everything)
- if (innerWeight != null) innerWeight.sumOfSquaredWeights();
+ if (innerWeight != null) innerWeight.getValueForNormalization();
queryWeight = getBoost();
return queryWeight * queryWeight;
}
@@ -146,7 +141,7 @@
}
if (disi == null)
return null;
- return new ConstantScorer(disi, this);
+ return new ConstantScorer(disi, this, queryWeight);
}
@Override
@@ -179,9 +174,9 @@
final DocIdSetIterator docIdSetIterator;
final float theScore;
- public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w) throws IOException {
+ public ConstantScorer(DocIdSetIterator docIdSetIterator, Weight w, float theScore) throws IOException {
super(w);
- theScore = w.getValue();
+ this.theScore = theScore;
this.docIdSetIterator = docIdSetIterator;
}
@@ -210,7 +205,7 @@
@Override
public void setScorer(Scorer scorer) throws IOException {
// we must wrap again here, but using the scorer passed in as parameter:
- collector.setScorer(new ConstantScorer(scorer, ConstantScorer.this.weight));
+ collector.setScorer(new ConstantScorer(scorer, ConstantScorer.this.weight, ConstantScorer.this.theScore));
}
@Override
Index: lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (working copy)
@@ -53,12 +53,12 @@
private final Bits delDocs;
private final Similarity similarity;
- MatchAllScorer(IndexReader reader, Similarity similarity, Weight w,
+ MatchAllScorer(IndexReader reader, Similarity similarity, Weight w, float score,
byte[] norms) throws IOException {
super(w);
this.similarity = similarity;
delDocs = reader.getDeletedDocs();
- score = w.getValue();
+ this.score = score;
maxDoc = reader.maxDoc();
this.norms = norms;
}
@@ -80,6 +80,8 @@
return doc;
}
+ // TODO: this is baked-in TF/IDF: should calling sim's docScorer.score() with a freq of 1 on the "norms field"
+ // but, by default it uses a "null field", which could pose a problem for someone's SimilarityProvider !!!!
@Override
public float score() {
return norms == null ? score : score * similarity.decodeNormValue(norms[docID()]);
@@ -112,12 +114,7 @@
}
@Override
- public float getValue() {
- return queryWeight;
- }
-
- @Override
- public float sumOfSquaredWeights() {
+ public float getValueForNormalization() {
queryWeight = getBoost();
return queryWeight * queryWeight;
}
@@ -130,7 +127,7 @@
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new MatchAllScorer(context.reader, similarity, this,
+ return new MatchAllScorer(context.reader, similarity, this, queryWeight,
normsField != null ? context.reader.norms(normsField) : null);
}
@@ -138,7 +135,7 @@
public Explanation explain(AtomicReaderContext context, int doc) {
// explain query weight
Explanation queryExpl = new ComplexExplanation
- (true, getValue(), "MatchAllDocsQuery, product of:");
+ (true, queryWeight, "MatchAllDocsQuery, product of:");
if (getBoost() != 1.0f) {
queryExpl.addDetail(new Explanation(getBoost(),"boost"));
}
Index: lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (working copy)
@@ -110,16 +110,12 @@
@Override
public Query getQuery() { return DisjunctionMaxQuery.this; }
- /** Return our boost */
- @Override
- public float getValue() { return getBoost(); }
-
/** Compute the sub of squared weights of us applied to our subqueries. Used for normalization. */
@Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
float max = 0.0f, sum = 0.0f;
for (Weight currentWeight : weights) {
- float sub = currentWeight.sumOfSquaredWeights();
+ float sub = currentWeight.getValueForNormalization();
sum += sub;
max = Math.max(max, sub);
Index: lucene/src/java/org/apache/lucene/search/Similarity.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/Similarity.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/Similarity.java (working copy)
@@ -22,7 +22,6 @@
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.search.Explanation.IDFExplanation;
import org.apache.lucene.search.spans.SpanQuery; // javadoc
import org.apache.lucene.util.TermContext;
@@ -115,17 +114,20 @@
return 1;
}
- public abstract IDFExplanation computeWeight(IndexSearcher searcher, String fieldName, TermContext... termStats) throws IOException;
+ /**
+ * Compute any collection-level stats (e.g. IDF, average document length, etc) needed for scoring a query.
+ */
+ public abstract Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException;
/**
* returns a new {@link Similarity.ExactDocScorer}.
*/
- public abstract ExactDocScorer exactDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException;
+ public abstract ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException;
/**
* returns a new {@link Similarity.SloppyDocScorer}.
*/
- public abstract SloppyDocScorer sloppyDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException;
+ public abstract SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException;
/**
* API for scoring exact queries such as {@link TermQuery} and
@@ -158,4 +160,34 @@
*/
public abstract float score(int doc, float freq);
}
+
+ /** Stores the statistics for the indexed collection. This abstract
+ * implementation is empty; descendants of {@code Similarity} should
+ * subclass {@code Stats} and define the statistics they require in the
+ * subclass. Examples include idf, average field length, etc.
+ */
+ public static abstract class Stats {
+
+ /** The value for normalization of contained query clauses (e.g. sum of squared weights).
+ *
+ * NOTE: a Similarity implementation might not use any query normalization at all,
+ * its not required. However, if it wants to participate in query normalization,
+ * it can return a value here.
+ */
+ public abstract float getValueForNormalization();
+
+ /** Assigns the query normalization factor and boost from parent queries to this.
+ *
+ * NOTE: a Similarity implementation might not use this normalized value at all,
+ * its not required. However, its usually a good idea to at least incorporate
+ * the topLevelBoost (e.g. from an outer BooleanQuery) into its score.
+ */
+ public abstract void normalize(float queryNorm, float topLevelBoost);
+
+ // NOTE: I think we should just leave explaining to the docScorer,
+ // if an impl's docScorer decides that stats are part of that
+ // and wants to delegate part of the explanation to its stats class,
+ // thats cool, but not required
+ //public abstract Explanation explain();
+ }
}
Index: lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (working copy)
@@ -145,7 +145,7 @@
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
return new PayloadNearSpanScorer(query.getSpans(context), this,
- similarity, query.getField(), context);
+ similarity, similarity.sloppyDocScorer(stats, query.getField(), context));
}
}
@@ -155,8 +155,8 @@
private int payloadsSeen;
protected PayloadNearSpanScorer(Spans spans, Weight weight,
- Similarity similarity, String field, AtomicReaderContext context) throws IOException {
- super(spans, weight, similarity, field, context);
+ Similarity similarity, Similarity.SloppyDocScorer docScorer) throws IOException {
+ super(spans, weight, similarity, docScorer);
this.spans = spans;
}
Index: lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (working copy)
@@ -76,7 +76,7 @@
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
return new PayloadTermSpanScorer((TermSpans) query.getSpans(context),
- this, similarity, query.getField(), context);
+ this, similarity, similarity.sloppyDocScorer(stats, query.getField(), context));
}
protected class PayloadTermSpanScorer extends SpanScorer {
@@ -86,8 +86,8 @@
private final TermSpans termSpans;
public PayloadTermSpanScorer(TermSpans spans, Weight weight,
- Similarity similarity, String field, AtomicReaderContext context) throws IOException {
- super(spans, weight, similarity, field, context);
+ Similarity similarity, Similarity.SloppyDocScorer docScorer) throws IOException {
+ super(spans, weight, similarity, docScorer);
termSpans = spans;
}
Index: lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java (working copy)
@@ -21,7 +21,6 @@
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
-import org.apache.lucene.search.Explanation.IDFExplanation;
import org.apache.lucene.util.TermContext;
import java.io.IOException;
@@ -33,14 +32,9 @@
*/
public class SpanWeight extends Weight {
protected Similarity similarity;
- protected float value;
- protected float idf;
- protected float queryNorm;
- protected float queryWeight;
-
protected Set terms;
protected SpanQuery query;
- private IDFExplanation idfExp;
+ protected Similarity.Stats stats;
public SpanWeight(SpanQuery query, IndexSearcher searcher)
throws IOException {
@@ -54,44 +48,42 @@
int i = 0;
for (Term term : terms)
states[i++] = TermContext.build(context, term, true);
- idfExp = similarity.computeWeight(searcher, query.getField(), states);
- idf = idfExp.getIdf();
+ stats = similarity.computeStats(searcher, query.getField(), query.getBoost(), states);
}
@Override
public Query getQuery() { return query; }
@Override
- public float getValue() { return value; }
-
- @Override
- public float sumOfSquaredWeights() throws IOException {
- queryWeight = idf * query.getBoost(); // compute query weight
- return queryWeight * queryWeight; // square it
+ public float getValueForNormalization() throws IOException {
+ return stats.getValueForNormalization();
}
@Override
public void normalize(float queryNorm, float topLevelBoost) {
- this.queryNorm = queryNorm * topLevelBoost;
- queryWeight *= this.queryNorm; // normalize query weight
- value = queryWeight * idf; // idf for document
+ stats.normalize(queryNorm, topLevelBoost);
}
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new SpanScorer(query.getSpans(context), this, similarity, query.getField(), context);
+ return new SpanScorer(query.getSpans(context), this, similarity, similarity.sloppyDocScorer(stats, query.getField(), context));
}
@Override
public Explanation explain(AtomicReaderContext context, int doc)
throws IOException {
+ //nocommit: fix explains
+ if (!(similarity instanceof TFIDFSimilarity))
+ return new ComplexExplanation();
+ final TFIDFSimilarity similarity = (TFIDFSimilarity) this.similarity;
+ final TFIDFSimilarity.IDFStats stats = (TFIDFSimilarity.IDFStats) this.stats;
ComplexExplanation result = new ComplexExplanation();
result.setDescription("weight("+getQuery()+" in "+doc+"), product of:");
String field = ((SpanQuery)getQuery()).getField();
Explanation idfExpl =
- new Explanation(idf, "idf(" + field + ": " + idfExp.explain() + ")");
+ new Explanation(stats.idf.getIdf(), "idf(" + field + ": " + stats.idf.explain() + ")");
// explain query weight
Explanation queryExpl = new Explanation();
@@ -102,7 +94,7 @@
queryExpl.addDetail(boostExpl);
queryExpl.addDetail(idfExpl);
- Explanation queryNormExpl = new Explanation(queryNorm,"queryNorm");
+ Explanation queryNormExpl = new Explanation(stats.queryNorm,"queryNorm");
queryExpl.addDetail(queryNormExpl);
queryExpl.setValue(boostExpl.getValue() *
Index: lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java (working copy)
@@ -19,7 +19,6 @@
import java.io.IOException;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.TFIDFSimilarity;
import org.apache.lucene.search.Weight;
@@ -39,11 +38,11 @@
protected final Similarity similarity;
protected final Similarity.SloppyDocScorer docScorer;
- protected SpanScorer(Spans spans, Weight weight, Similarity similarity, String field, AtomicReaderContext context)
+ protected SpanScorer(Spans spans, Weight weight, Similarity similarity, Similarity.SloppyDocScorer docScorer)
throws IOException {
super(weight);
this.similarity = similarity;
- this.docScorer = similarity.sloppyDocScorer(weight, field, context);
+ this.docScorer = docScorer;
this.spans = spans;
if (this.spans.next()) {
Index: lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java (working copy)
@@ -20,8 +20,6 @@
import java.io.IOException;
import java.util.HashSet;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-
final class SloppyPhraseScorer extends PhraseScorer {
private int slop;
private PhrasePositions repeats[];
@@ -30,8 +28,8 @@
private final Similarity similarity;
SloppyPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings, Similarity similarity,
- int slop, String field, AtomicReaderContext context) throws IOException {
- super(weight, postings, similarity, field, context);
+ int slop, Similarity.SloppyDocScorer docScorer) throws IOException {
+ super(weight, postings, docScorer);
this.slop = slop;
this.similarity = similarity;
}
Index: lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (working copy)
@@ -131,12 +131,8 @@
private class MultiPhraseWeight extends Weight {
- private Similarity similarity;
- private float value;
- private final IDFExplanation idfExp;
- private float idf;
- private float queryNorm;
- private float queryWeight;
+ private final Similarity similarity;
+ private final Similarity.Stats stats;
public MultiPhraseWeight(IndexSearcher searcher)
throws IOException {
@@ -150,27 +146,20 @@
allTerms.add(TermContext.build(context, term, true));
}
}
- idfExp = similarity.computeWeight(searcher, field, allTerms.toArray(new TermContext[allTerms.size()]));
- idf = idfExp.getIdf();
+ stats = similarity.computeStats(searcher, field, getBoost(), allTerms.toArray(new TermContext[allTerms.size()]));
}
@Override
public Query getQuery() { return MultiPhraseQuery.this; }
@Override
- public float getValue() { return value; }
-
- @Override
- public float sumOfSquaredWeights() {
- queryWeight = idf * getBoost(); // compute query weight
- return queryWeight * queryWeight; // square it
+ public float getValueForNormalization() {
+ return stats.getValueForNormalization();
}
@Override
public void normalize(float queryNorm, float topLevelBoost) {
- this.queryNorm = queryNorm * topLevelBoost;
- queryWeight *= this.queryNorm; // normalize query weight
- value = queryWeight * idf; // idf for document
+ stats.normalize(queryNorm, topLevelBoost);
}
@Override
@@ -225,7 +214,7 @@
}
if (slop == 0) {
- ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity, field, context);
+ ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity.exactDocScorer(stats, field, context));
if (s.noDocs) {
return null;
} else {
@@ -233,7 +222,7 @@
}
} else {
return new SloppyPhraseScorer(this, postingsFreqs, similarity,
- slop, field, context);
+ slop, similarity.sloppyDocScorer(stats, field, context));
}
}
@@ -244,11 +233,12 @@
if (!(similarity instanceof TFIDFSimilarity))
return new ComplexExplanation();
final TFIDFSimilarity similarity = (TFIDFSimilarity) this.similarity;
-
+ final TFIDFSimilarity.IDFStats stats = (TFIDFSimilarity.IDFStats) this.stats;
+
ComplexExplanation result = new ComplexExplanation();
result.setDescription("weight("+getQuery()+" in "+doc+"), product of:");
- Explanation idfExpl = new Explanation(idf, "idf(" + field + ":" + idfExp.explain() +")");
+ Explanation idfExpl = new Explanation(stats.idf.getIdf(), "idf(" + field + ":" + stats.idf.explain() +")");
// explain query weight
Explanation queryExpl = new Explanation();
@@ -260,7 +250,7 @@
queryExpl.addDetail(idfExpl);
- Explanation queryNormExpl = new Explanation(queryNorm,"queryNorm");
+ Explanation queryNormExpl = new Explanation(stats.queryNorm,"queryNorm");
queryExpl.addDetail(queryNormExpl);
queryExpl.setValue(boostExpl.getValue() *
Index: lucene/src/java/org/apache/lucene/search/FilteredQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/FilteredQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/FilteredQuery.java (working copy)
@@ -63,21 +63,15 @@
public Weight createWeight(final IndexSearcher searcher) throws IOException {
final Weight weight = query.createWeight (searcher);
return new Weight() {
- private float value;
-
- // pass these methods through to enclosed query's weight
- @Override
- public float getValue() { return value; }
@Override
- public float sumOfSquaredWeights() throws IOException {
- return weight.sumOfSquaredWeights() * getBoost() * getBoost();
+ public float getValueForNormalization() throws IOException {
+ return weight.getValueForNormalization() * getBoost() * getBoost();
}
@Override
public void normalize (float norm, float topLevelBoost) {
weight.normalize(norm, topLevelBoost);
- value = weight.getValue() * getBoost();
}
@Override
Index: lucene/src/java/org/apache/lucene/search/PhraseScorer.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/PhraseScorer.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/PhraseScorer.java (working copy)
@@ -19,8 +19,6 @@
import java.io.IOException;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-
/** Expert: Scoring functionality for phrase queries.
*
A document is considered matching if it contains the phrase-query terms
* at "valid" positions. What "valid positions" are
@@ -32,9 +30,6 @@
* means a match.
*/
abstract class PhraseScorer extends Scorer {
- protected byte[] norms;
- protected float value;
-
private boolean firstTime = true;
private boolean more = true;
protected PhraseQueue pq;
@@ -45,9 +40,9 @@
protected final Similarity.SloppyDocScorer docScorer;
PhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
- Similarity similarity, String field, AtomicReaderContext context) throws IOException {
+ Similarity.SloppyDocScorer docScorer) throws IOException {
super(weight);
- docScorer = similarity.sloppyDocScorer(weight, field, context);
+ this.docScorer = docScorer;
// convert tps to a list of phrase positions.
// note: phrase-position differs from term-position in that its position
Index: lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java (working copy)
@@ -76,15 +76,9 @@
return ValueSourceQuery.this;
}
- /*(non-Javadoc) @see org.apache.lucene.search.Weight#getValue() */
- @Override
- public float getValue() {
- return queryWeight;
- }
-
/*(non-Javadoc) @see org.apache.lucene.search.Weight#sumOfSquaredWeights() */
@Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
queryWeight = getBoost();
return queryWeight * queryWeight;
}
@@ -98,7 +92,7 @@
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new ValueSourceScorer(context, this);
+ return new ValueSourceScorer(context, this, queryWeight);
}
/*(non-Javadoc) @see org.apache.lucene.search.Weight#explain(org.apache.lucene.index.IndexReader, int) */
@@ -131,10 +125,10 @@
private int doc = -1;
// constructor
- private ValueSourceScorer(AtomicReaderContext context, ValueSourceWeight w) throws IOException {
+ private ValueSourceScorer(AtomicReaderContext context, ValueSourceWeight w, float qWeight) throws IOException {
super(w);
final IndexReader reader = context.reader;
- qWeight = w.getValue();
+ this.qWeight = qWeight;
// this is when/where the values are first created.
vals = valSrc.getValues(context);
delDocs = reader.getDeletedDocs();
Index: lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java (working copy)
@@ -202,20 +202,19 @@
}
/*(non-Javadoc) @see org.apache.lucene.search.Weight#getValue() */
- @Override
public float getValue() {
return getBoost();
}
/*(non-Javadoc) @see org.apache.lucene.search.Weight#sumOfSquaredWeights() */
@Override
- public float sumOfSquaredWeights() throws IOException {
- float sum = subQueryWeight.sumOfSquaredWeights();
+ public float getValueForNormalization() throws IOException {
+ float sum = subQueryWeight.getValueForNormalization();
for(int i = 0; i < valSrcWeights.length; i++) {
if (qStrict) {
- valSrcWeights[i].sumOfSquaredWeights(); // do not include ValueSource part in the query normalization
+ valSrcWeights[i].getValueForNormalization(); // do not include ValueSource part in the query normalization
} else {
- sum += valSrcWeights[i].sumOfSquaredWeights();
+ sum += valSrcWeights[i].getValueForNormalization();
}
}
sum *= getBoost() * getBoost(); // boost each sub-weight
Index: lucene/src/java/org/apache/lucene/search/BooleanQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/BooleanQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/BooleanQuery.java (working copy)
@@ -183,14 +183,11 @@
public Query getQuery() { return BooleanQuery.this; }
@Override
- public float getValue() { return getBoost(); }
-
- @Override
- public float sumOfSquaredWeights() throws IOException {
+ public float getValueForNormalization() throws IOException {
float sum = 0.0f;
for (int i = 0 ; i < weights.size(); i++) {
// call sumOfSquaredWeights for all clauses in case of side effects
- float s = weights.get(i).sumOfSquaredWeights(); // sum sub weights
+ float s = weights.get(i).getValueForNormalization(); // sum sub weights
if (!clauses.get(i).isProhibited())
// only add to sum for non-prohibited clauses
sum += s;
Index: lucene/src/java/org/apache/lucene/search/Query.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/Query.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/Query.java (working copy)
@@ -97,8 +97,8 @@
public Weight weight(IndexSearcher searcher) throws IOException {
Query query = searcher.rewrite(this);
Weight weight = query.createWeight(searcher);
- float sum = weight.sumOfSquaredWeights();
- float norm = searcher.getSimilarityProvider().queryNorm(sum);
+ float v = weight.getValueForNormalization();
+ float norm = searcher.getSimilarityProvider().queryNorm(v);
if (Float.isInfinite(norm) || Float.isNaN(norm))
norm = 1.0f;
weight.normalize(norm, 1.0f);
Index: lucene/src/java/org/apache/lucene/search/PhraseQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/PhraseQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/PhraseQuery.java (working copy)
@@ -28,7 +28,6 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.Explanation.IDFExplanation;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.util.ArrayUtil;
@@ -174,11 +173,7 @@
private class PhraseWeight extends Weight {
private final Similarity similarity;
- private float value;
- private float idf;
- private float queryNorm;
- private float queryWeight;
- private IDFExplanation idfExp;
+ private final Similarity.Stats stats;
private transient TermContext states[];
public PhraseWeight(IndexSearcher searcher)
@@ -188,8 +183,7 @@
states = new TermContext[terms.size()];
for (int i = 0; i < terms.size(); i++)
states[i] = TermContext.build(context, terms.get(i), true);
- idfExp = similarity.computeWeight(searcher, field, states);
- idf = idfExp.getIdf();
+ stats = similarity.computeStats(searcher, field, getBoost(), states);
}
@Override
@@ -199,19 +193,13 @@
public Query getQuery() { return PhraseQuery.this; }
@Override
- public float getValue() { return value; }
-
- @Override
- public float sumOfSquaredWeights() {
- queryWeight = idf * getBoost(); // compute query weight
- return queryWeight * queryWeight; // square it
+ public float getValueForNormalization() {
+ return stats.getValueForNormalization();
}
@Override
public void normalize(float queryNorm, float topLevelBoost) {
- this.queryNorm = queryNorm * topLevelBoost;
- queryWeight *= this.queryNorm; // normalize query weight
- value = queryWeight * idf; // idf for document
+ stats.normalize(queryNorm, topLevelBoost);
}
@Override
@@ -254,7 +242,7 @@
}
if (slop == 0) { // optimize exact case
- ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity, field, context);
+ ExactPhraseScorer s = new ExactPhraseScorer(this, postingsFreqs, similarity.exactDocScorer(stats, field, context));
if (s.noDocs) {
return null;
} else {
@@ -262,7 +250,7 @@
}
} else {
return
- new SloppyPhraseScorer(this, postingsFreqs, similarity, slop, field, context);
+ new SloppyPhraseScorer(this, postingsFreqs, similarity, slop, similarity.sloppyDocScorer(stats, field, context));
}
}
@@ -273,6 +261,7 @@
if (!(similarity instanceof TFIDFSimilarity))
return new ComplexExplanation();
final TFIDFSimilarity similarity = (TFIDFSimilarity) this.similarity;
+ final TFIDFSimilarity.IDFStats stats = (TFIDFSimilarity.IDFStats) this.stats;
ComplexExplanation result = new ComplexExplanation();
result.setDescription("weight("+getQuery()+" in "+doc+"), product of:");
@@ -280,7 +269,7 @@
StringBuilder docFreqs = new StringBuilder();
StringBuilder query = new StringBuilder();
query.append('\"');
- docFreqs.append(idfExp.explain());
+ docFreqs.append(stats.idf.explain());
for (int i = 0; i < terms.size(); i++) {
if (i != 0) {
query.append(" ");
@@ -293,7 +282,7 @@
query.append('\"');
Explanation idfExpl =
- new Explanation(idf, "idf(" + field + ":" + docFreqs + ")");
+ new Explanation(stats.idf.getIdf(), "idf(" + field + ":" + docFreqs + ")");
// explain query weight
Explanation queryExpl = new Explanation();
@@ -304,7 +293,7 @@
queryExpl.addDetail(boostExpl);
queryExpl.addDetail(idfExpl);
- Explanation queryNormExpl = new Explanation(queryNorm,"queryNorm");
+ Explanation queryNormExpl = new Explanation(stats.queryNorm,"queryNorm");
queryExpl.addDetail(queryNormExpl);
queryExpl.setValue(boostExpl.getValue() *
Index: lucene/src/java/org/apache/lucene/search/TermQuery.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/TermQuery.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/TermQuery.java (working copy)
@@ -27,7 +27,6 @@
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Explanation.IDFExplanation;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ReaderUtil;
@@ -43,11 +42,7 @@
private class TermWeight extends Weight {
private final Similarity similarity;
- private float value;
- private final float idf;
- private float queryNorm;
- private float queryWeight;
- private final IDFExplanation idfExp;
+ private final Similarity.Stats stats;
private transient TermContext termStates;
public TermWeight(IndexSearcher searcher, TermContext termStates)
@@ -55,8 +50,7 @@
assert termStates != null : "TermContext must not be null";
this.termStates = termStates;
this.similarity = searcher.getSimilarityProvider().get(term.field());
- idfExp = similarity.computeWeight(searcher, term.field(), termStates);
- idf = idfExp.getIdf();
+ this.stats = similarity.computeStats(searcher, term.field(), getBoost(), termStates);
}
@Override
@@ -66,19 +60,13 @@
public Query getQuery() { return TermQuery.this; }
@Override
- public float getValue() { return value; }
-
- @Override
- public float sumOfSquaredWeights() {
- queryWeight = idf * getBoost(); // compute query weight
- return queryWeight * queryWeight; // square it
+ public float getValueForNormalization() {
+ return stats.getValueForNormalization();
}
@Override
public void normalize(float queryNorm, float topLevelBoost) {
- this.queryNorm = queryNorm * topLevelBoost;
- queryWeight *= this.queryNorm; // normalize query weight
- value = queryWeight * idf; // idf for document
+ stats.normalize(queryNorm, topLevelBoost);
}
@Override
@@ -94,7 +82,7 @@
}
final DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), field, term.bytes(), state);
assert docs != null;
- return new TermScorer(this, docs, similarity, field, context);
+ return new TermScorer(this, docs, similarity.exactDocScorer(stats, field, context));
}
private boolean termNotInReader(IndexReader reader, String field, BytesRef bytes) throws IOException {
@@ -110,13 +98,14 @@
if (!(similarity instanceof TFIDFSimilarity))
return new ComplexExplanation();
final TFIDFSimilarity similarity = (TFIDFSimilarity) this.similarity;
+ final TFIDFSimilarity.IDFStats stats = (TFIDFSimilarity.IDFStats) this.stats;
final IndexReader reader = context.reader;
ComplexExplanation result = new ComplexExplanation();
result.setDescription("weight("+getQuery()+" in "+doc+"), product of:");
- Explanation expl = new Explanation(idf, idfExp.explain());
+ Explanation expl = new Explanation(stats.idf.getIdf(), stats.idf.explain());
// explain query weight
Explanation queryExpl = new Explanation();
@@ -127,7 +116,7 @@
queryExpl.addDetail(boostExpl);
queryExpl.addDetail(expl);
- Explanation queryNormExpl = new Explanation(queryNorm,"queryNorm");
+ Explanation queryNormExpl = new Explanation(stats.queryNorm,"queryNorm");
queryExpl.addDetail(queryNormExpl);
queryExpl.setValue(boostExpl.getValue() *
Index: lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java (working copy)
@@ -687,21 +687,22 @@
}
@Override
- public final IDFExplanation computeWeight(IndexSearcher searcher, String fieldName,
- TermContext... termStats) throws IOException {
- return termStats.length == 1
- ? idfExplain(termStats[0], searcher)
- : idfExplain(termStats, searcher);
+ public final Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost,
+ TermContext... termContexts) throws IOException {
+ final IDFExplanation idf = termContexts.length == 1
+ ? idfExplain(termContexts[0], searcher)
+ : idfExplain(termContexts, searcher);
+ return new IDFStats(idf, queryBoost);
}
@Override
- public final ExactDocScorer exactDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException {
- return new ExactTFIDFDocScorer(weight.getValue(), context.reader.norms(fieldName));
+ public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new ExactTFIDFDocScorer(((IDFStats)stats).getValue(), context.reader.norms(fieldName));
}
@Override
- public final SloppyDocScorer sloppyDocScorer(Weight weight, String fieldName, AtomicReaderContext context) throws IOException {
- return new SloppyTFIDFDocScorer(weight.getValue(), context.reader.norms(fieldName));
+ public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new SloppyTFIDFDocScorer(((IDFStats)stats).getValue(), context.reader.norms(fieldName));
}
// TODO: we can specialize these for omitNorms up front, but we should test that it doesn't confuse stupid hotspot.
@@ -746,4 +747,38 @@
return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field
}
}
+
+ /** Collection statistics for the TF-IDF model. The only statistic of interest
+ * to this model is idf. */
+ public static class IDFStats extends Stats {
+ /** The idf and its explanation: public for now until we fix explains. */
+ public final IDFExplanation idf;
+ // again public for now until we fix explains
+ public float queryNorm;
+ private float queryWeight;
+ private float value;
+
+ public IDFStats(IDFExplanation idf, float queryBoost) {
+ // TODO: Validate?
+ this.idf = idf;
+ this.queryWeight = idf.getIdf() * queryBoost; // compute query weight
+ }
+
+ @Override
+ public float getValueForNormalization() {
+ // TODO: (sorta LUCENE-1907) make non-static class and expose this squaring via a nice method to subclasses?
+ return queryWeight * queryWeight; // sum of squared weights
+ }
+
+ @Override
+ public void normalize(float queryNorm, float topLevelBoost) {
+ this.queryNorm = queryNorm * topLevelBoost;
+ queryWeight *= this.queryNorm; // normalize query weight
+ value = queryWeight * idf.getIdf(); // idf for document
+ }
+
+ public float getValue() {
+ return value;
+ }
+ }
}
Index: lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java (working copy)
@@ -21,7 +21,6 @@
import java.util.Arrays;
import org.apache.lucene.index.*;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
final class ExactPhraseScorer extends Scorer {
private final int endMinus1;
@@ -58,9 +57,9 @@
private final Similarity.ExactDocScorer docScorer;
ExactPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
- Similarity similarity, String field, AtomicReaderContext context) throws IOException {
+ Similarity.ExactDocScorer docScorer) throws IOException {
super(weight);
- this.docScorer = similarity.exactDocScorer(weight, field, context);
+ this.docScorer = docScorer;
chunkStates = new ChunkState[postings.length];
Index: lucene/src/java/org/apache/lucene/search/TermScorer.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/TermScorer.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/TermScorer.java (working copy)
@@ -44,15 +44,13 @@
* The weight of the Term in the query.
* @param td
* An iterator over the documents matching the Term.
- * @param similarity
- * The Similarity implementation to be used for score
- * computations.
- * @param norms
- * The field norms of the document fields for the Term.
+ * @param docScorer
+ * The Similarity.ExactDocScorer implementation
+ * to be used for score computations.
*/
- TermScorer(Weight weight, DocsEnum td, Similarity similarity, String fieldName, AtomicReaderContext context) throws IOException {
+ TermScorer(Weight weight, DocsEnum td, Similarity.ExactDocScorer docScorer) throws IOException {
super(weight);
- this.docScorer = similarity.exactDocScorer(weight, fieldName, context);
+ this.docScorer = docScorer;
this.docsEnum = td;
bulkResult = td.getBulkResult();
}
Index: lucene/src/java/org/apache/lucene/search/Weight.java
===================================================================
--- lucene/src/java/org/apache/lucene/search/Weight.java (revision 1136270)
+++ lucene/src/java/org/apache/lucene/search/Weight.java (working copy)
@@ -67,11 +67,11 @@
/** The query that this concerns. */
public abstract Query getQuery();
+
+ /** The value for normalization of contained query clauses (e.g. sum of squared weights). */
+ public abstract float getValueForNormalization() throws IOException;
- /** The weight for this query. */
- public abstract float getValue();
-
- /** Assigns the query normalization factor to this. */
+ /** Assigns the query normalization factor and boost from parent queries to this. */
public abstract void normalize(float norm, float topLevelBoost);
/**
@@ -93,9 +93,6 @@
* @throws IOException
*/
public abstract Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException;
-
- /** The sum of squared weights of contained query clauses. */
- public abstract float sumOfSquaredWeights() throws IOException;
/**
* Returns true iff this implementation scores docs only out of order. This