diff --git lucene/src/java/org/apache/lucene/search/similarities/AfterEffect.java lucene/src/java/org/apache/lucene/search/similarities/AfterEffect.java index faee912..4610f2b 100644 --- lucene/src/java/org/apache/lucene/search/similarities/AfterEffect.java +++ lucene/src/java/org/apache/lucene/search/similarities/AfterEffect.java @@ -31,20 +31,20 @@ import org.apache.lucene.search.Explanation; */ public abstract class AfterEffect { /** Returns the aftereffect score. */ - public abstract float score(EasyStats stats, float tfn); + public abstract float score(BasicStats stats, float tfn); /** Returns an explanation for the score. */ - public abstract Explanation explain(EasyStats stats, float tfn); + public abstract Explanation explain(BasicStats stats, float tfn); /** Implementation used when there is no aftereffect. */ public static final class NoAfterEffect extends AfterEffect { @Override - public final float score(EasyStats stats, float tfn) { + public final float score(BasicStats stats, float tfn) { return 1f; } @Override - public final Explanation explain(EasyStats stats, float tfn) { + public final Explanation explain(BasicStats stats, float tfn) { return new Explanation(1, "no aftereffect"); } diff --git lucene/src/java/org/apache/lucene/search/similarities/AfterEffectB.java lucene/src/java/org/apache/lucene/search/similarities/AfterEffectB.java index 90079b2..df34030 100644 --- lucene/src/java/org/apache/lucene/search/similarities/AfterEffectB.java +++ lucene/src/java/org/apache/lucene/search/similarities/AfterEffectB.java @@ -25,14 +25,14 @@ import org.apache.lucene.search.Explanation; */ public class AfterEffectB extends AfterEffect { @Override - public final float score(EasyStats stats, float tfn) { + public final float score(BasicStats stats, float tfn) { long F = stats.getTotalTermFreq(); int n = stats.getDocFreq(); return (F + 1) / (n * (tfn + 1)); } @Override - public final Explanation explain(EasyStats stats, float tfn) { + public final Explanation explain(BasicStats stats, float tfn) { Explanation result = new Explanation(); result.setDescription(getClass().getSimpleName() + ", computed from: "); result.setValue(score(stats, tfn)); diff --git lucene/src/java/org/apache/lucene/search/similarities/AfterEffectL.java lucene/src/java/org/apache/lucene/search/similarities/AfterEffectL.java index 3a1f223..5479830 100644 --- lucene/src/java/org/apache/lucene/search/similarities/AfterEffectL.java +++ lucene/src/java/org/apache/lucene/search/similarities/AfterEffectL.java @@ -25,12 +25,12 @@ import org.apache.lucene.search.Explanation; */ public class AfterEffectL extends AfterEffect { @Override - public final float score(EasyStats stats, float tfn) { + public final float score(BasicStats stats, float tfn) { return 1 / (tfn + 1); } @Override - public final Explanation explain(EasyStats stats, float tfn) { + public final Explanation explain(BasicStats stats, float tfn) { Explanation result = new Explanation(); result.setDescription(getClass().getSimpleName() + ", computed from: "); result.setValue(score(stats, tfn)); diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModel.java lucene/src/java/org/apache/lucene/search/similarities/BasicModel.java index e811869..ff4d12d 100644 --- lucene/src/java/org/apache/lucene/search/similarities/BasicModel.java +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModel.java @@ -30,7 +30,7 @@ import org.apache.lucene.search.Explanation; */ public abstract class BasicModel { /** Returns the informative content score. */ - public abstract float score(EasyStats stats, float tfn); + public abstract float score(BasicStats stats, float tfn); /** * Returns an explanation for the score. @@ -39,7 +39,7 @@ public abstract class BasicModel { * explanation for such models. Subclasses that use other statistics must * override this method.

*/ - public Explanation explain(EasyStats stats, float tfn) { + public Explanation explain(BasicStats stats, float tfn) { Explanation result = new Explanation(); result.setDescription(getClass().getSimpleName() + ", computed from: "); result.setValue(score(stats, tfn)); diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelBE.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelBE.java index 46216d6..4553afc 100644 --- lucene/src/java/org/apache/lucene/search/similarities/BasicModelBE.java +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelBE.java @@ -17,7 +17,7 @@ package org.apache.lucene.search.similarities; * limitations under the License. */ -import static org.apache.lucene.search.similarities.EasySimilarity.log2; +import static org.apache.lucene.search.similarities.SimilarityBase.log2; /** * Limiting form of the Bose-Einstein model. The formula used in Lucene differs @@ -28,7 +28,7 @@ import static org.apache.lucene.search.similarities.EasySimilarity.log2; */ public class BasicModelBE extends BasicModel { @Override - public final float score(EasyStats stats, float tfn) { + public final float score(BasicStats stats, float tfn) { long N = stats.getNumberOfDocuments() + 1; // long F = stats.getTotalTermFreq() + 1; long F = Math.max(stats.getTotalTermFreq(), (long)(tfn + 0.5) + 1); diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelD.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelD.java index 7c253d9..0472330 100644 --- lucene/src/java/org/apache/lucene/search/similarities/BasicModelD.java +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelD.java @@ -17,7 +17,7 @@ package org.apache.lucene.search.similarities; * limitations under the License. */ -import static org.apache.lucene.search.similarities.EasySimilarity.log2; +import static org.apache.lucene.search.similarities.SimilarityBase.log2; /** * Implements the approximation of the binomial model with the divergence @@ -29,7 +29,7 @@ import static org.apache.lucene.search.similarities.EasySimilarity.log2; */ public class BasicModelD extends BasicModel { @Override - public final float score(EasyStats stats, float tfn) { + public final float score(BasicStats stats, float tfn) { long F = Math.max(stats.getTotalTermFreq(), (long)(tfn + 0.5) + 1); // long F = stats.getTotalTermFreq() + 1; double phi = (double)tfn / F; diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelG.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelG.java index 4a8e75a..86f5dea 100644 --- lucene/src/java/org/apache/lucene/search/similarities/BasicModelG.java +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelG.java @@ -17,7 +17,7 @@ package org.apache.lucene.search.similarities; * limitations under the License. */ -import static org.apache.lucene.search.similarities.EasySimilarity.log2; +import static org.apache.lucene.search.similarities.SimilarityBase.log2; /** * Geometric as limiting form of the Bose-Einstein model. @@ -25,7 +25,7 @@ import static org.apache.lucene.search.similarities.EasySimilarity.log2; */ public class BasicModelG extends BasicModel { @Override - public final float score(EasyStats stats, float tfn) { + public final float score(BasicStats stats, float tfn) { double lambda = stats.getTotalTermFreq() / (double) stats.getNumberOfDocuments(); // -log(1 / (lambda + 1)) -> log(lambda + 1) return (float)(log2(lambda + 1) + tfn * log2((1 + lambda) / lambda)); diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelIF.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelIF.java index c646bb8..3cef323 100644 --- lucene/src/java/org/apache/lucene/search/similarities/BasicModelIF.java +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelIF.java @@ -17,7 +17,7 @@ package org.apache.lucene.search.similarities; * limitations under the License. */ -import static org.apache.lucene.search.similarities.EasySimilarity.log2; +import static org.apache.lucene.search.similarities.SimilarityBase.log2; /** * An approximation of the I(ne) model. @@ -25,7 +25,7 @@ import static org.apache.lucene.search.similarities.EasySimilarity.log2; */ public class BasicModelIF extends BasicModel { @Override - public final float score(EasyStats stats, float tfn) { + public final float score(BasicStats stats, float tfn) { int N = stats.getNumberOfDocuments(); long F = stats.getTotalTermFreq(); return tfn * (float)(log2(1 + (N + 1) / (F + 0.5))); diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelIn.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelIn.java index 3d7bf96..a61222e 100644 --- lucene/src/java/org/apache/lucene/search/similarities/BasicModelIn.java +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelIn.java @@ -18,7 +18,7 @@ package org.apache.lucene.search.similarities; */ import org.apache.lucene.search.Explanation; -import static org.apache.lucene.search.similarities.EasySimilarity.log2; +import static org.apache.lucene.search.similarities.SimilarityBase.log2; /** * The basic tf-idf model of randomness. @@ -26,14 +26,14 @@ import static org.apache.lucene.search.similarities.EasySimilarity.log2; */ public class BasicModelIn extends BasicModel { @Override - public final float score(EasyStats stats, float tfn) { + public final float score(BasicStats stats, float tfn) { int N = stats.getNumberOfDocuments(); int n = stats.getDocFreq(); return tfn * (float)(log2((N + 1) / (n + 0.5))); } @Override - public final Explanation explain(EasyStats stats, float tfn) { + public final Explanation explain(BasicStats stats, float tfn) { Explanation result = new Explanation(); result.setDescription(getClass().getSimpleName() + ", computed from: "); result.setValue(score(stats, tfn)); diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelIne.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelIne.java index e9b51ac..cdbdeb4 100644 --- lucene/src/java/org/apache/lucene/search/similarities/BasicModelIne.java +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelIne.java @@ -17,7 +17,7 @@ package org.apache.lucene.search.similarities; * limitations under the License. */ -import static org.apache.lucene.search.similarities.EasySimilarity.log2; +import static org.apache.lucene.search.similarities.SimilarityBase.log2; /** * Tf-idf model of randomness, based on a mixture of Poisson and inverse @@ -26,7 +26,7 @@ import static org.apache.lucene.search.similarities.EasySimilarity.log2; */ public class BasicModelIne extends BasicModel { @Override - public final float score(EasyStats stats, float tfn) { + public final float score(BasicStats stats, float tfn) { int N = stats.getNumberOfDocuments(); long F = stats.getTotalTermFreq(); double ne = N * (1 - Math.pow((N - 1) / (double)N, F)); diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelP.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelP.java index 94ff145..c9575dc 100644 --- lucene/src/java/org/apache/lucene/search/similarities/BasicModelP.java +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelP.java @@ -17,7 +17,7 @@ package org.apache.lucene.search.similarities; * limitations under the License. */ -import static org.apache.lucene.search.similarities.EasySimilarity.log2; +import static org.apache.lucene.search.similarities.SimilarityBase.log2; /** * Implements the Poisson approximation for the binomial model for DFR. @@ -28,7 +28,7 @@ public class BasicModelP extends BasicModel { protected static double LOG2_E = log2(Math.E); @Override - public final float score(EasyStats stats, float tfn) { + public final float score(BasicStats stats, float tfn) { float lambda = (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); // System.out.printf("tfn=%f, lambda=%f, log1=%f, log2=%f%n", tfn, lambda, // tfn / lambda, 2 * Math.PI * tfn); diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicStats.java lucene/src/java/org/apache/lucene/search/similarities/BasicStats.java new file mode 100644 index 0000000..a96e7a0 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/BasicStats.java @@ -0,0 +1,144 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.Terms; + +/** + * Stores all statistics commonly used ranking methods. + * @lucene.experimental + */ +public class BasicStats extends Similarity.Stats { + /** The number of documents. */ + protected int numberOfDocuments; + /** The total number of tokens in the field. */ + protected long numberOfFieldTokens; + /** The average field length. */ + protected float avgFieldLength; + /** The document frequency. */ + protected int docFreq; + /** The total number of occurrences of this term across all documents. */ + protected long totalTermFreq; + + // -------------------------- Boost-related stuff -------------------------- + + /** Query's inner boost. */ + protected final float queryBoost; + /** Any outer query's boost. */ + protected float topLevelBoost; + /** For most Similarities, the immediate and the top level query boosts are + * not handled differently. Hence, this field is just the product of the + * other two. */ + protected float totalBoost; + + /** Constructor. Sets the query boost. */ + public BasicStats(float queryBoost) { + this.queryBoost = queryBoost; + this.totalBoost = queryBoost; + } + + // ------------------------- Getter/setter methods ------------------------- + + /** Returns the number of documents. */ + public int getNumberOfDocuments() { + return numberOfDocuments; + } + + /** Sets the number of documents. */ + public void setNumberOfDocuments(int numberOfDocuments) { + this.numberOfDocuments = numberOfDocuments; + } + + /** + * Returns the total number of tokens in the field. + * @see Terms#getSumTotalTermFreq() + */ + public long getNumberOfFieldTokens() { + return numberOfFieldTokens; + } + + /** + * Sets the total number of tokens in the field. + * @see Terms#getSumTotalTermFreq() + */ + public void setNumberOfFieldTokens(long numberOfFieldTokens) { + this.numberOfFieldTokens = numberOfFieldTokens; + } + + /** Returns the average field length. */ + public float getAvgFieldLength() { + return avgFieldLength; + } + + /** Sets the average field length. */ + public void setAvgFieldLength(float avgFieldLength) { + this.avgFieldLength = avgFieldLength; + } + + /** Returns the document frequency. */ + public int getDocFreq() { + return docFreq; + } + + /** Sets the document frequency. */ + public void setDocFreq(int docFreq) { + this.docFreq = docFreq; + } + + /** Returns the total number of occurrences of this term across all documents. */ + public long getTotalTermFreq() { + return totalTermFreq; + } + + /** Sets the total number of occurrences of this term across all documents. */ + public void setTotalTermFreq(long totalTermFreq) { + this.totalTermFreq = totalTermFreq; + } + + // -------------------------- Boost-related stuff -------------------------- + + /** The square of the raw normalization value. + * @see #rawNormalizationValue() */ + @Override + public float getValueForNormalization() { + float rawValue = rawNormalizationValue(); + return rawValue * rawValue; + } + + /** Computes the raw normalization value. This basic implementation returns + * the query boost. Subclasses may override this method to include other + * factors (such as idf), or to save the value for inclusion in + * {@link #normalize(float, float)}, etc. + */ + protected float rawNormalizationValue() { + return queryBoost; + } + + /** No normalization is done. {@code topLevelBoost} is saved in the object, + * however. */ + @Override + public void normalize(float queryNorm, float topLevelBoost) { + this.topLevelBoost = topLevelBoost; + totalBoost = queryBoost * topLevelBoost; + } + + /** Returns the total boost. */ + public float getTotalBoost() { + return totalBoost; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java index 69dffc3..3bbca1c 100644 --- lucene/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java +++ lucene/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java @@ -39,7 +39,7 @@ import org.apache.lucene.search.Explanation; * @see Normalization * @lucene.experimental */ -public class DFRSimilarity extends EasySimilarity { +public class DFRSimilarity extends SimilarityBase { /** The basic model for information content. */ protected final BasicModel basicModel; /** The first normalization of the information content. */ @@ -78,7 +78,7 @@ public class DFRSimilarity extends EasySimilarity { } @Override - protected float score(EasyStats stats, float freq, float docLen) { + protected float score(BasicStats stats, float freq, float docLen) { float tfn = normalization.tfn(stats, freq, docLen); return stats.getTotalBoost() * basicModel.score(stats, tfn) * afterEffect.score(stats, tfn); @@ -86,7 +86,7 @@ public class DFRSimilarity extends EasySimilarity { @Override protected void explain(Explanation expl, - EasyStats stats, int doc, float freq, float docLen) { + BasicStats stats, int doc, float freq, float docLen) { if (stats.getTotalBoost() != 1.0f) { expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); } diff --git lucene/src/java/org/apache/lucene/search/similarities/Distribution.java lucene/src/java/org/apache/lucene/search/similarities/Distribution.java index aa2702a..c31164e 100644 --- lucene/src/java/org/apache/lucene/search/similarities/Distribution.java +++ lucene/src/java/org/apache/lucene/search/similarities/Distribution.java @@ -27,11 +27,11 @@ import org.apache.lucene.search.Explanation; */ public abstract class Distribution { /** Computes the score. */ - public abstract float score(EasyStats stats, float tfn, float lambda); + public abstract float score(BasicStats stats, float tfn, float lambda); /** Explains the score. Returns the name of the model only, since * both {@code tfn} and {@code lambda} are explained elsewhere. */ - public Explanation explain(EasyStats stats, float tfn, float lambda) { + public Explanation explain(BasicStats stats, float tfn, float lambda) { return new Explanation( score(stats, tfn, lambda), getClass().getSimpleName()); } diff --git lucene/src/java/org/apache/lucene/search/similarities/DistributionLL.java lucene/src/java/org/apache/lucene/search/similarities/DistributionLL.java index f7a4a4a..b9e0913 100644 --- lucene/src/java/org/apache/lucene/search/similarities/DistributionLL.java +++ lucene/src/java/org/apache/lucene/search/similarities/DistributionLL.java @@ -26,7 +26,7 @@ package org.apache.lucene.search.similarities; */ public class DistributionLL extends Distribution { @Override - public final float score(EasyStats stats, float tfn, float lambda) { + public final float score(BasicStats stats, float tfn, float lambda) { return (float)-Math.log(lambda / (tfn + lambda)); } diff --git lucene/src/java/org/apache/lucene/search/similarities/DistributionSPL.java lucene/src/java/org/apache/lucene/search/similarities/DistributionSPL.java index 1027543..13baf52 100644 --- lucene/src/java/org/apache/lucene/search/similarities/DistributionSPL.java +++ lucene/src/java/org/apache/lucene/search/similarities/DistributionSPL.java @@ -27,7 +27,7 @@ package org.apache.lucene.search.similarities; */ public class DistributionSPL extends Distribution { @Override - public final float score(EasyStats stats, float tfn, float lambda) { + public final float score(BasicStats stats, float tfn, float lambda) { if (lambda == 1f) { lambda = 0.99f; } diff --git lucene/src/java/org/apache/lucene/search/similarities/EasySimilarity.java lucene/src/java/org/apache/lucene/search/similarities/EasySimilarity.java deleted file mode 100644 index 6cc13ea..0000000 --- lucene/src/java/org/apache/lucene/search/similarities/EasySimilarity.java +++ /dev/null @@ -1,302 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.MultiFields; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.SmallFloat; -import org.apache.lucene.util.TermContext; - -/** - * A subclass of {@code Similarity} that provides a simplified API for its - * descendants. Subclasses are only required to implement the {@link #score} - * and {@link #toString()} methods. Implementing - * {@link #explain(Explanation, EasyStats, int, float, float)} is optional, - * inasmuch as EasySimilarity already provides a basic explanation of the score - * and the term frequency. However, implementers of a subclass are encouraged to - * include as much detail about the scoring method as possible. - * @lucene.experimental - */ -public abstract class EasySimilarity extends Similarity { - /** For {@link #log2(double)}. Precomputed for efficiency reasons. */ - private static final double LOG_2 = Math.log(2); - - /** @see #setDiscountOverlaps */ - protected boolean discountOverlaps = true; - - /** Determines whether overlap tokens (Tokens with - * 0 position increment) are ignored when computing - * norm. By default this is true, meaning overlap - * tokens do not count when computing norms. - * - * @lucene.experimental - * - * @see #computeNorm - */ - public void setDiscountOverlaps(boolean v) { - discountOverlaps = v; - } - - /** @see #setDiscountOverlaps */ - public boolean getDiscountOverlaps() { - return discountOverlaps; - } - - /** - * Calls {@link #fillEasyStats(EasyStats, IndexSearcher, String, TermContext...)}. - * Subclasses that override this method may invoke {@code fillStats} with any - * subclass of {@code EasyStats}. - */ - @Override - public EasyStats computeStats(IndexSearcher searcher, String fieldName, - float queryBoost, TermContext... termContexts) throws IOException { - EasyStats stats = new EasyStats(queryBoost); - fillEasyStats(stats, searcher, fieldName, termContexts); - return stats; - } - - /** Fills all member fields defined in {@code EasyStats} in {@code stats}. */ - protected final void fillEasyStats(EasyStats stats, IndexSearcher searcher, - String fieldName, TermContext... termContexts) throws IOException { - IndexReader reader = searcher.getIndexReader(); - int numberOfDocuments = reader.maxDoc(); - long numberOfFieldTokens = MultiFields.getTerms(searcher.getIndexReader(), - fieldName).getSumTotalTermFreq(); - float avgFieldLength = (float)numberOfFieldTokens / numberOfDocuments; - - // nocommit Take the minimum of term frequencies for phrases. This is not - // correct though, we'll need something like a scorePhrase(MultiStats ...) - int docFreq = Integer.MAX_VALUE; - long totalTermFreq = Integer.MAX_VALUE; - for (final TermContext context : termContexts) { - docFreq = Math.min(docFreq, context.docFreq()); - totalTermFreq = Math.min(totalTermFreq, context.totalTermFreq()); - } - - // We have to provide something if codec doesnt supply these measures, - // or if someone omitted frequencies for the field... negative values cause - // NaN/Inf for some scorers. - if (numberOfFieldTokens == -1) { - numberOfFieldTokens = docFreq; - avgFieldLength = 1; - } - if (totalTermFreq == -1) { - totalTermFreq = docFreq; - } - - stats.setNumberOfDocuments(numberOfDocuments); - stats.setNumberOfFieldTokens(numberOfFieldTokens); - stats.setAvgFieldLength(avgFieldLength); - stats.setDocFreq(docFreq); - stats.setTotalTermFreq(totalTermFreq); - } - - /** - * Scores the document {@code doc}. - *

Subclasses must apply their scoring formula in this class.

- * @param stats the corpus level statistics. - * @param freq the term frequency. - * @param docLen the document length. - * @return the score. - */ - protected abstract float score(EasyStats stats, float freq, float docLen); - - /** - * Subclasses should implement this method to explain the score. {@code expl} - * already contains the score, the name of the class and the doc id, as well - * as the term frequency and its explanation; subclasses can add additional - * clauses to explain details of their scoring formulae. - *

The default implementation does nothing.

- * - * @param expl the explanation to extend with details. - * @param stats the corpus level statistics. - * @param doc the document id. - * @param freq the term frequency. - * @param docLen the document length. - */ - protected void explain( - Explanation expl, EasyStats stats, int doc, float freq, float docLen) {} - - /** - * Explains the score. The implementation here provides a basic explanation - * in the format score(name-of-similarity, doc=doc-id, - * freq=term-frequency), computed from:, and - * attaches the score (computed via the {@link #score(EasyStats, float, float)} - * method) and the explanation for the term frequency. Subclasses content with - * this format may add additional details in - * {@link #explain(Explanation, EasyStats, int, float, float)}. - * - * @param stats the corpus level statistics. - * @param doc the document id. - * @param freq the term frequency and its explanation. - * @param docLen the document length. - * @return the explanation. - */ - protected Explanation explain( - EasyStats stats, int doc, Explanation freq, float docLen) { - Explanation result = new Explanation(); - result.setValue(score(stats, freq.getValue(), docLen)); - result.setDescription("score(" + getClass().getSimpleName() + - ", doc=" + doc + ", freq=" + freq.getValue() +"), computed from:"); - result.addDetail(freq); - - explain(result, stats, doc, freq.getValue(), docLen); - - return result; - } - - @Override - public ExactDocScorer exactDocScorer(Stats stats, String fieldName, - AtomicReaderContext context) throws IOException { - return new EasyExactDocScorer((EasyStats) stats, - context.reader.norms(fieldName)); - } - - @Override - public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, - AtomicReaderContext context) throws IOException { - return new EasySloppyDocScorer((EasyStats) stats, - context.reader.norms(fieldName)); - } - - /** - * Subclasses must override this method to return the name of the Similarity - * and preferably the values of parameters (if any) as well. - */ - @Override - public abstract String toString(); // nocommit: to Similarity? - - // ------------------------------ Norm handling ------------------------------ - - /** Norm -> document length map. */ - private static final float[] NORM_TABLE = new float[256]; - - static { - for (int i = 0; i < 256; i++) { - float floatNorm = SmallFloat.byte315ToFloat((byte)i); - NORM_TABLE[i] = 1.0f / (floatNorm * floatNorm); - } - } - - /** Encodes the document length in the same way as {@link TFIDFSimilarity}. */ - @Override - public byte computeNorm(FieldInvertState state) { - final float numTerms; - if (discountOverlaps) - numTerms = state.getLength() - state.getNumOverlap(); - else - numTerms = state.getLength() / state.getBoost(); - return encodeNormValue(numTerms); - } - - /** Decodes a normalization factor (document length) stored in an index. - * @see #encodeNormValue(float) - */ - protected float decodeNormValue(byte norm) { - return NORM_TABLE[norm & 0xFF]; // & 0xFF maps negative bytes to positive above 127 - } - - /** Encodes the length to a byte via SmallFloat. */ - protected byte encodeNormValue(float length) { - return SmallFloat.floatToByte315((float)(1.0 / Math.sqrt(length))); - } - - // ----------------------------- Static methods ------------------------------ - - /** Returns the base two logarithm of {@code x}. */ - public static double log2(double x) { - // Put this to a 'util' class if we need more of these. - return Math.log(x) / LOG_2; - } - - // --------------------------------- Classes --------------------------------- - - /** Delegates the {@link #score(int, int)} and - * {@link #explain(int, Explanation)} methods to - * {@link EasySimilarity#score(EasyStats, float, int)} and - * {@link EasySimilarity#explain(EasyStats, int, Explanation, int)}, - * respectively. - */ - private class EasyExactDocScorer extends ExactDocScorer { - private final EasyStats stats; - private final byte[] norms; - - EasyExactDocScorer(EasyStats stats, byte norms[]) { - this.stats = stats; - this.norms = norms; - } - - @Override - public float score(int doc, int freq) { - // We have to supply something in case norms are omitted - return EasySimilarity.this.score(stats, freq, - norms == null ? freq : decodeNormValue(norms[doc])); - } - - @Override - public Explanation explain(int doc, Explanation freq) { - return EasySimilarity.this.explain(stats, doc, freq, - norms == null ? freq.getValue() : decodeNormValue(norms[doc])); - } - } - - /** Delegates the {@link #score(int, int)} and - * {@link #explain(int, Explanation)} methods to - * {@link EasySimilarity#score(EasyStats, float, int)} and - * {@link EasySimilarity#explain(EasyStats, int, Explanation, int)}, - * respectively. - */ - private class EasySloppyDocScorer extends SloppyDocScorer { - private final EasyStats stats; - private final byte[] norms; - - EasySloppyDocScorer(EasyStats stats, byte norms[]) { - this.stats = stats; - this.norms = norms; - } - - @Override - public float score(int doc, float freq) { - // We have to supply something in case norms are omitted - return EasySimilarity.this.score(stats, freq, - norms == null ? freq : decodeNormValue(norms[doc])); - } - @Override - public Explanation explain(int doc, Explanation freq) { - return EasySimilarity.this.explain(stats, doc, freq, - norms == null ? freq.getValue() : decodeNormValue(norms[doc])); - } - - @Override - public float computeSlopFactor(int distance) { - return 1.0f / (distance + 1); - } - - @Override - public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { - return 1f; - } - } -} diff --git lucene/src/java/org/apache/lucene/search/similarities/EasyStats.java lucene/src/java/org/apache/lucene/search/similarities/EasyStats.java deleted file mode 100644 index fbf3cc3..0000000 --- lucene/src/java/org/apache/lucene/search/similarities/EasyStats.java +++ /dev/null @@ -1,144 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.index.Terms; - -/** - * Stores all statistics commonly used ranking methods. - * @lucene.experimental - */ -public class EasyStats extends Similarity.Stats { - /** The number of documents. */ - protected int numberOfDocuments; - /** The total number of tokens in the field. */ - protected long numberOfFieldTokens; - /** The average field length. */ - protected float avgFieldLength; - /** The document frequency. */ - protected int docFreq; - /** The total number of occurrences of this term across all documents. */ - protected long totalTermFreq; - - // -------------------------- Boost-related stuff -------------------------- - - /** Query's inner boost. */ - protected final float queryBoost; - /** Any outer query's boost. */ - protected float topLevelBoost; - /** For most Similarities, the immediate and the top level query boosts are - * not handled differently. Hence, this field is just the product of the - * other two. */ - protected float totalBoost; - - /** Constructor. Sets the query boost. */ - public EasyStats(float queryBoost) { - this.queryBoost = queryBoost; - this.totalBoost = queryBoost; - } - - // ------------------------- Getter/setter methods ------------------------- - - /** Returns the number of documents. */ - public int getNumberOfDocuments() { - return numberOfDocuments; - } - - /** Sets the number of documents. */ - public void setNumberOfDocuments(int numberOfDocuments) { - this.numberOfDocuments = numberOfDocuments; - } - - /** - * Returns the total number of tokens in the field. - * @see Terms#getSumTotalTermFreq() - */ - public long getNumberOfFieldTokens() { - return numberOfFieldTokens; - } - - /** - * Sets the total number of tokens in the field. - * @see Terms#getSumTotalTermFreq() - */ - public void setNumberOfFieldTokens(long numberOfFieldTokens) { - this.numberOfFieldTokens = numberOfFieldTokens; - } - - /** Returns the average field length. */ - public float getAvgFieldLength() { - return avgFieldLength; - } - - /** Sets the average field length. */ - public void setAvgFieldLength(float avgFieldLength) { - this.avgFieldLength = avgFieldLength; - } - - /** Returns the document frequency. */ - public int getDocFreq() { - return docFreq; - } - - /** Sets the document frequency. */ - public void setDocFreq(int docFreq) { - this.docFreq = docFreq; - } - - /** Returns the total number of occurrences of this term across all documents. */ - public long getTotalTermFreq() { - return totalTermFreq; - } - - /** Sets the total number of occurrences of this term across all documents. */ - public void setTotalTermFreq(long totalTermFreq) { - this.totalTermFreq = totalTermFreq; - } - - // -------------------------- Boost-related stuff -------------------------- - - /** The square of the raw normalization value. - * @see #rawNormalizationValue() */ - @Override - public float getValueForNormalization() { - float rawValue = rawNormalizationValue(); - return rawValue * rawValue; - } - - /** Computes the raw normalization value. This basic implementation returns - * the query boost. Subclasses may override this method to include other - * factors (such as idf), or to save the value for inclusion in - * {@link #normalize(float, float)}, etc. - */ - protected float rawNormalizationValue() { - return queryBoost; - } - - /** No normalization is done. {@code topLevelBoost} is saved in the object, - * however. */ - @Override - public void normalize(float queryNorm, float topLevelBoost) { - this.topLevelBoost = topLevelBoost; - totalBoost = queryBoost * topLevelBoost; - } - - /** Returns the total boost. */ - public float getTotalBoost() { - return totalBoost; - } -} diff --git lucene/src/java/org/apache/lucene/search/similarities/IBSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/IBSimilarity.java index 07d40e5..3676bae 100644 --- lucene/src/java/org/apache/lucene/search/similarities/IBSimilarity.java +++ lucene/src/java/org/apache/lucene/search/similarities/IBSimilarity.java @@ -41,7 +41,7 @@ import org.apache.lucene.search.Explanation; * Similarities will be merged at one point.

* @lucene.experimental */ -public class IBSimilarity extends EasySimilarity { +public class IBSimilarity extends SimilarityBase { /** The probabilistic distribution used to model term occurrence. */ protected final Distribution distribution; /** The lambda (λw) parameter. */ @@ -63,7 +63,7 @@ public class IBSimilarity extends EasySimilarity { } @Override - protected float score(EasyStats stats, float freq, float docLen) { + protected float score(BasicStats stats, float freq, float docLen) { return stats.getTotalBoost() * distribution.score( stats, @@ -73,7 +73,7 @@ public class IBSimilarity extends EasySimilarity { @Override protected void explain( - Explanation expl, EasyStats stats, int doc, float freq, float docLen) { + Explanation expl, BasicStats stats, int doc, float freq, float docLen) { if (stats.getTotalBoost() != 1.0f) { expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); } diff --git lucene/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java index 6c8f5f4..ad7e309 100644 --- lucene/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java +++ lucene/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java @@ -60,7 +60,7 @@ public class LMDirichletSimilarity extends LMSimilarity { } @Override - protected float score(EasyStats stats, float freq, float docLen) { + protected float score(BasicStats stats, float freq, float docLen) { float score = stats.getTotalBoost() * (float)(Math.log(1 + freq / (mu * ((LMStats)stats).getCollectionProbability())) + Math.log(mu / (docLen + mu))); @@ -68,7 +68,7 @@ public class LMDirichletSimilarity extends LMSimilarity { } @Override - protected void explain(Explanation expl, EasyStats stats, int doc, + protected void explain(Explanation expl, BasicStats stats, int doc, float freq, float docLen) { if (stats.getTotalBoost() != 1.0f) { expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); diff --git lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java index f002caf..910e769 100644 --- lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java +++ lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java @@ -48,7 +48,7 @@ public class LMJelinekMercerSimilarity extends LMSimilarity { } @Override - protected float score(EasyStats stats, float freq, float docLen) { + protected float score(BasicStats stats, float freq, float docLen) { return stats.getTotalBoost() * (float)Math.log(1 + ((1 - lambda) * freq / docLen) / @@ -56,7 +56,7 @@ public class LMJelinekMercerSimilarity extends LMSimilarity { } @Override - protected void explain(Explanation expl, EasyStats stats, int doc, + protected void explain(Explanation expl, BasicStats stats, int doc, float freq, float docLen) { if (stats.getTotalBoost() != 1.0f) { expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); diff --git lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java index 2aa2e04..00e1cfe 100644 --- lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java +++ lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java @@ -38,7 +38,7 @@ import org.apache.lucene.util.TermContext; * * @lucene.experimental */ -public abstract class LMSimilarity extends EasySimilarity { +public abstract class LMSimilarity extends SimilarityBase { /** The collection model. */ protected final CollectionModel collectionModel; @@ -57,16 +57,16 @@ public abstract class LMSimilarity extends EasySimilarity { * usual statistics. */ @Override - public EasyStats computeStats(IndexSearcher searcher, String fieldName, + public BasicStats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException { LMStats stats = new LMStats(queryBoost); - fillEasyStats(stats, searcher, fieldName, termContexts); + fillBasicStats(stats, searcher, fieldName, termContexts); stats.setCollectionProbability(collectionModel.computeProbability(stats)); return stats; } @Override - protected void explain(Explanation expl, EasyStats stats, int doc, + protected void explain(Explanation expl, BasicStats stats, int doc, float freq, float docLen) { expl.addDetail(new Explanation(collectionModel.computeProbability(stats), "collection probability")); @@ -97,7 +97,7 @@ public abstract class LMSimilarity extends EasySimilarity { } /** Stores the collection distribution of the current term. */ - public static class LMStats extends EasyStats { + public static class LMStats extends BasicStats { /** The probability that the current term is generated by the collection. */ private float collectionProbability; @@ -128,7 +128,7 @@ public abstract class LMSimilarity extends EasySimilarity { * Computes the probability {@code p(w|C)} according to the language model * strategy for the current term. */ - public float computeProbability(EasyStats stats); + public float computeProbability(BasicStats stats); /** The name of the collection model strategy. */ public String getName(); @@ -140,7 +140,7 @@ public abstract class LMSimilarity extends EasySimilarity { */ public static class DefaultCollectionModel implements CollectionModel { @Override - public float computeProbability(EasyStats stats) { + public float computeProbability(BasicStats stats) { return (float)stats.getTotalTermFreq() / (stats.getNumberOfFieldTokens() +1); } diff --git lucene/src/java/org/apache/lucene/search/similarities/Lambda.java lucene/src/java/org/apache/lucene/search/similarities/Lambda.java index 5df1b4a..64b8c34 100644 --- lucene/src/java/org/apache/lucene/search/similarities/Lambda.java +++ lucene/src/java/org/apache/lucene/search/similarities/Lambda.java @@ -27,9 +27,9 @@ import org.apache.lucene.search.Explanation; */ public abstract class Lambda { /** Computes the lambda parameter. */ - public abstract float lambda(EasyStats stats); + public abstract float lambda(BasicStats stats); /** Explains the lambda parameter. */ - public abstract Explanation explain(EasyStats stats); + public abstract Explanation explain(BasicStats stats); /** * Subclasses must override this method to return the code of the lambda diff --git lucene/src/java/org/apache/lucene/search/similarities/LambdaDF.java lucene/src/java/org/apache/lucene/search/similarities/LambdaDF.java index 4786ec6..7e4a824 100644 --- lucene/src/java/org/apache/lucene/search/similarities/LambdaDF.java +++ lucene/src/java/org/apache/lucene/search/similarities/LambdaDF.java @@ -25,12 +25,12 @@ import org.apache.lucene.search.Explanation; */ public class LambdaDF extends Lambda { @Override - public final float lambda(EasyStats stats) { + public final float lambda(BasicStats stats) { return (float)stats.getDocFreq() / stats.getNumberOfDocuments(); } @Override - public final Explanation explain(EasyStats stats) { + public final Explanation explain(BasicStats stats) { Explanation result = new Explanation(); result.setDescription(getClass().getSimpleName() + ", computed from: "); result.setValue(lambda(stats)); diff --git lucene/src/java/org/apache/lucene/search/similarities/LambdaTTF.java lucene/src/java/org/apache/lucene/search/similarities/LambdaTTF.java index f07f146..25c55bd 100644 --- lucene/src/java/org/apache/lucene/search/similarities/LambdaTTF.java +++ lucene/src/java/org/apache/lucene/search/similarities/LambdaTTF.java @@ -25,12 +25,12 @@ import org.apache.lucene.search.Explanation; */ public class LambdaTTF extends Lambda { @Override - public final float lambda(EasyStats stats) { + public final float lambda(BasicStats stats) { return (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); } @Override - public final Explanation explain(EasyStats stats) { + public final Explanation explain(BasicStats stats) { Explanation result = new Explanation(); result.setDescription(getClass().getSimpleName() + ", computed from: "); result.setValue(lambda(stats)); diff --git lucene/src/java/org/apache/lucene/search/similarities/Normalization.java lucene/src/java/org/apache/lucene/search/similarities/Normalization.java index 6a2afea..f635baa 100644 --- lucene/src/java/org/apache/lucene/search/similarities/Normalization.java +++ lucene/src/java/org/apache/lucene/search/similarities/Normalization.java @@ -29,7 +29,7 @@ import org.apache.lucene.search.Explanation; public abstract class Normalization { /** Returns the normalized term frequency. * @param len the field length. */ - public abstract float tfn(EasyStats stats, float tf, float len); + public abstract float tfn(BasicStats stats, float tf, float len); /** Returns an explanation for the normalized term frequency. *

The default normalization methods use the field length of the document @@ -37,7 +37,7 @@ public abstract class Normalization { * This method provides a generic explanation for such methods. * Subclasses that use other statistics must override this method.

*/ - public Explanation explain(EasyStats stats, float tf, float len) { + public Explanation explain(BasicStats stats, float tf, float len) { Explanation result = new Explanation(); result.setDescription(getClass().getSimpleName() + ", computed from: "); result.setValue(tfn(stats, tf, len)); @@ -51,12 +51,12 @@ public abstract class Normalization { /** Implementation used when there is no normalization. */ public static final class NoNormalization extends Normalization { @Override - public final float tfn(EasyStats stats, float tf, float len) { + public final float tfn(BasicStats stats, float tf, float len) { return tf; } @Override - public final Explanation explain(EasyStats stats, float tf, float len) { + public final Explanation explain(BasicStats stats, float tf, float len) { return new Explanation(1, "no normalization"); } diff --git lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java index d289f61..98ed604 100644 --- lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java +++ lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java @@ -22,7 +22,7 @@ package org.apache.lucene.search.similarities; */ public class NormalizationH1 extends Normalization { @Override - public final float tfn(EasyStats stats, float tf, float len) { + public final float tfn(BasicStats stats, float tf, float len) { return tf * stats.getAvgFieldLength() / len; } diff --git lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java index c49eaf4..ad7f965 100644 --- lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java +++ lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java @@ -17,7 +17,7 @@ package org.apache.lucene.search.similarities; * limitations under the License. */ -import static org.apache.lucene.search.similarities.EasySimilarity.log2; +import static org.apache.lucene.search.similarities.SimilarityBase.log2; /** * Normalization model in which the term frequency is inversely related to the @@ -25,7 +25,7 @@ import static org.apache.lucene.search.similarities.EasySimilarity.log2; */ public class NormalizationH2 extends Normalization { @Override - public final float tfn(EasyStats stats, float tf, float len) { + public final float tfn(BasicStats stats, float tf, float len) { return (float)(tf * log2(1 + stats.getAvgFieldLength() / len)); } diff --git lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java new file mode 100644 index 0000000..e61c51c --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java @@ -0,0 +1,302 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SmallFloat; +import org.apache.lucene.util.TermContext; + +/** + * A subclass of {@code Similarity} that provides a simplified API for its + * descendants. Subclasses are only required to implement the {@link #score} + * and {@link #toString()} methods. Implementing + * {@link #explain(Explanation, BasicStats, int, float, float)} is optional, + * inasmuch as SimilarityBase already provides a basic explanation of the score + * and the term frequency. However, implementers of a subclass are encouraged to + * include as much detail about the scoring method as possible. + * @lucene.experimental + */ +public abstract class SimilarityBase extends Similarity { + /** For {@link #log2(double)}. Precomputed for efficiency reasons. */ + private static final double LOG_2 = Math.log(2); + + /** @see #setDiscountOverlaps */ + protected boolean discountOverlaps = true; + + /** Determines whether overlap tokens (Tokens with + * 0 position increment) are ignored when computing + * norm. By default this is true, meaning overlap + * tokens do not count when computing norms. + * + * @lucene.experimental + * + * @see #computeNorm + */ + public void setDiscountOverlaps(boolean v) { + discountOverlaps = v; + } + + /** @see #setDiscountOverlaps */ + public boolean getDiscountOverlaps() { + return discountOverlaps; + } + + /** + * Calls {@link #fillBasicStats(BasicStats, IndexSearcher, String, TermContext...)}. + * Subclasses that override this method may invoke {@code fillStats} with any + * subclass of {@code BasicStats}. + */ + @Override + public BasicStats computeStats(IndexSearcher searcher, String fieldName, + float queryBoost, TermContext... termContexts) throws IOException { + BasicStats stats = new BasicStats(queryBoost); + fillBasicStats(stats, searcher, fieldName, termContexts); + return stats; + } + + /** Fills all member fields defined in {@code BasicStats} in {@code stats}. */ + protected final void fillBasicStats(BasicStats stats, IndexSearcher searcher, + String fieldName, TermContext... termContexts) throws IOException { + IndexReader reader = searcher.getIndexReader(); + int numberOfDocuments = reader.maxDoc(); + long numberOfFieldTokens = MultiFields.getTerms(searcher.getIndexReader(), + fieldName).getSumTotalTermFreq(); + float avgFieldLength = (float)numberOfFieldTokens / numberOfDocuments; + + // nocommit Take the minimum of term frequencies for phrases. This is not + // correct though, we'll need something like a scorePhrase(MultiStats ...) + int docFreq = Integer.MAX_VALUE; + long totalTermFreq = Integer.MAX_VALUE; + for (final TermContext context : termContexts) { + docFreq = Math.min(docFreq, context.docFreq()); + totalTermFreq = Math.min(totalTermFreq, context.totalTermFreq()); + } + + // We have to provide something if codec doesnt supply these measures, + // or if someone omitted frequencies for the field... negative values cause + // NaN/Inf for some scorers. + if (numberOfFieldTokens == -1) { + numberOfFieldTokens = docFreq; + avgFieldLength = 1; + } + if (totalTermFreq == -1) { + totalTermFreq = docFreq; + } + + stats.setNumberOfDocuments(numberOfDocuments); + stats.setNumberOfFieldTokens(numberOfFieldTokens); + stats.setAvgFieldLength(avgFieldLength); + stats.setDocFreq(docFreq); + stats.setTotalTermFreq(totalTermFreq); + } + + /** + * Scores the document {@code doc}. + *

Subclasses must apply their scoring formula in this class.

+ * @param stats the corpus level statistics. + * @param freq the term frequency. + * @param docLen the document length. + * @return the score. + */ + protected abstract float score(BasicStats stats, float freq, float docLen); + + /** + * Subclasses should implement this method to explain the score. {@code expl} + * already contains the score, the name of the class and the doc id, as well + * as the term frequency and its explanation; subclasses can add additional + * clauses to explain details of their scoring formulae. + *

The default implementation does nothing.

+ * + * @param expl the explanation to extend with details. + * @param stats the corpus level statistics. + * @param doc the document id. + * @param freq the term frequency. + * @param docLen the document length. + */ + protected void explain( + Explanation expl, BasicStats stats, int doc, float freq, float docLen) {} + + /** + * Explains the score. The implementation here provides a basic explanation + * in the format score(name-of-similarity, doc=doc-id, + * freq=term-frequency), computed from:, and + * attaches the score (computed via the {@link #score(BasicStats, float, float)} + * method) and the explanation for the term frequency. Subclasses content with + * this format may add additional details in + * {@link #explain(Explanation, BasicStats, int, float, float)}. + * + * @param stats the corpus level statistics. + * @param doc the document id. + * @param freq the term frequency and its explanation. + * @param docLen the document length. + * @return the explanation. + */ + protected Explanation explain( + BasicStats stats, int doc, Explanation freq, float docLen) { + Explanation result = new Explanation(); + result.setValue(score(stats, freq.getValue(), docLen)); + result.setDescription("score(" + getClass().getSimpleName() + + ", doc=" + doc + ", freq=" + freq.getValue() +"), computed from:"); + result.addDetail(freq); + + explain(result, stats, doc, freq.getValue(), docLen); + + return result; + } + + @Override + public ExactDocScorer exactDocScorer(Stats stats, String fieldName, + AtomicReaderContext context) throws IOException { + return new BasicExactDocScorer((BasicStats) stats, + context.reader.norms(fieldName)); + } + + @Override + public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, + AtomicReaderContext context) throws IOException { + return new BasicSloppyDocScorer((BasicStats) stats, + context.reader.norms(fieldName)); + } + + /** + * Subclasses must override this method to return the name of the Similarity + * and preferably the values of parameters (if any) as well. + */ + @Override + public abstract String toString(); // nocommit: to Similarity? + + // ------------------------------ Norm handling ------------------------------ + + /** Norm -> document length map. */ + private static final float[] NORM_TABLE = new float[256]; + + static { + for (int i = 0; i < 256; i++) { + float floatNorm = SmallFloat.byte315ToFloat((byte)i); + NORM_TABLE[i] = 1.0f / (floatNorm * floatNorm); + } + } + + /** Encodes the document length in the same way as {@link TFIDFSimilarity}. */ + @Override + public byte computeNorm(FieldInvertState state) { + final float numTerms; + if (discountOverlaps) + numTerms = state.getLength() - state.getNumOverlap(); + else + numTerms = state.getLength() / state.getBoost(); + return encodeNormValue(numTerms); + } + + /** Decodes a normalization factor (document length) stored in an index. + * @see #encodeNormValue(float) + */ + protected float decodeNormValue(byte norm) { + return NORM_TABLE[norm & 0xFF]; // & 0xFF maps negative bytes to positive above 127 + } + + /** Encodes the length to a byte via SmallFloat. */ + protected byte encodeNormValue(float length) { + return SmallFloat.floatToByte315((float)(1.0 / Math.sqrt(length))); + } + + // ----------------------------- Static methods ------------------------------ + + /** Returns the base two logarithm of {@code x}. */ + public static double log2(double x) { + // Put this to a 'util' class if we need more of these. + return Math.log(x) / LOG_2; + } + + // --------------------------------- Classes --------------------------------- + + /** Delegates the {@link #score(int, int)} and + * {@link #explain(int, Explanation)} methods to + * {@link SimilarityBase#score(BasicStats, float, int)} and + * {@link SimilarityBase#explain(BasicStats, int, Explanation, int)}, + * respectively. + */ + private class BasicExactDocScorer extends ExactDocScorer { + private final BasicStats stats; + private final byte[] norms; + + BasicExactDocScorer(BasicStats stats, byte norms[]) { + this.stats = stats; + this.norms = norms; + } + + @Override + public float score(int doc, int freq) { + // We have to supply something in case norms are omitted + return SimilarityBase.this.score(stats, freq, + norms == null ? freq : decodeNormValue(norms[doc])); + } + + @Override + public Explanation explain(int doc, Explanation freq) { + return SimilarityBase.this.explain(stats, doc, freq, + norms == null ? freq.getValue() : decodeNormValue(norms[doc])); + } + } + + /** Delegates the {@link #score(int, int)} and + * {@link #explain(int, Explanation)} methods to + * {@link SimilarityBase#score(BasicStats, float, int)} and + * {@link SimilarityBase#explain(BasicStats, int, Explanation, int)}, + * respectively. + */ + private class BasicSloppyDocScorer extends SloppyDocScorer { + private final BasicStats stats; + private final byte[] norms; + + BasicSloppyDocScorer(BasicStats stats, byte norms[]) { + this.stats = stats; + this.norms = norms; + } + + @Override + public float score(int doc, float freq) { + // We have to supply something in case norms are omitted + return SimilarityBase.this.score(stats, freq, + norms == null ? freq : decodeNormValue(norms[doc])); + } + @Override + public Explanation explain(int doc, Explanation freq) { + return SimilarityBase.this.explain(stats, doc, freq, + norms == null ? freq.getValue() : decodeNormValue(norms[doc])); + } + + @Override + public float computeSlopFactor(int distance) { + return 1.0f / (distance + 1); + } + + @Override + public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { + return 1f; + } + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java index bac07dd..22137c8 100644 --- lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java +++ lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java @@ -22,16 +22,9 @@ import java.io.IOException; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.Term; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.Weight; -import org.apache.lucene.search.similarities.Similarity.ExactDocScorer; -import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer; -import org.apache.lucene.search.similarities.Similarity.Stats; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.TermContext; import org.apache.lucene.util.SmallFloat; diff --git lucene/src/java/org/apache/lucene/search/similarities/package.html lucene/src/java/org/apache/lucene/search/similarities/package.html index b07091f..488d39d 100644 --- lucene/src/java/org/apache/lucene/search/similarities/package.html +++ lucene/src/java/org/apache/lucene/search/similarities/package.html @@ -46,10 +46,11 @@ information, see {@link org.apache.lucene.search.similarities.TFIDFSimilarity}.<

{@link org.apache.lucene.search.similarities.BM25Similarity} is an optimized implementation of the successful Okapi BM25 model.

-

{@link org.apache.lucene.search.similarities.EasySimilarity} provides a basic +

{@link org.apache.lucene.search.similarities.SimilarityBase} provides a basic implementation of the Similarity contract and exposes a highly simplified interface, which makes it an ideal starting point for new ranking functions. -Lucene ships the following EasySimilarity-based methods: +Lucene ships the following methods built on +{@link org.apache.lucene.search.similarities.SimilarityBase}:

-Since EasySimilarity is not optimized to the same extent as +Since {@link org.apache.lucene.search.similarities.SimilarityBase} is not +optimized to the same extent as {@link org.apache.lucene.search.similarities.DefaultSimilarity} and {@link org.apache.lucene.search.similarities.BM25Similarity}, a difference in performance is to be expected when using the methods listed above. However, @@ -100,7 +102,7 @@ either, the query norm is defined as 1 / (numTerms in field)^0.5.To make this change, implement your own {@link org.apache.lucene.search.similarities.Similarity} (likely you'll want to simply subclass an existing method, be it {@link org.apache.lucene.search.similarities.DefaultSimilarity} or a descendant of - {@link org.apache.lucene.search.similarities.EasySimilarity}) and + {@link org.apache.lucene.search.similarities.SimilarityBase}) and {@link org.apache.lucene.search.similarities.SimilarityProvider} (or use {@link org.apache.lucene.search.similarities.BasicSimilarityProvider}), and then register the new class by calling @@ -110,17 +112,17 @@ either, the query norm is defined as 1 / (numTerms in field)^0.5. -

Extending {@linkplain org.apache.lucene.search.similarities.EasySimilarity}

+

Extending {@linkplain org.apache.lucene.search.similarities.SimilarityBase}

The easiest way to quickly implement a new ranking method is to extend -{@link org.apache.lucene.search.similarities.EasySimilarity}, which provides +{@link org.apache.lucene.search.similarities.SimilarityBase}, which provides basic implementations for the low level . Subclasses are only required to -implement the {@link org.apache.lucene.search.similarities.EasySimilarity#score(EasyStats, float, float)} -and {@link org.apache.lucene.search.similarities.EasySimilarity#toString()} +implement the {@link org.apache.lucene.search.similarities.SimilarityBase#score(BasicStats, float, float)} +and {@link org.apache.lucene.search.similarities.SimilarityBase#toString()} methods.

Another options is to extend one of the frameworks -based on {@link org.apache.lucene.search.similarities.EasySimilarity}. These +based on {@link org.apache.lucene.search.similarities.SimilarityBase}. These Similarities are implemented modularly, e.g. {@link org.apache.lucene.search.similarities.DFRSimilarity} delegates computation of the three parts of its formula to the classes diff --git lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java index ed886fb..33e460b 100644 --- lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java +++ lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java @@ -39,22 +39,22 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; /** - * Index searcher implementation that takes an {@link EasyStats} instance and + * Index searcher implementation that takes an {@link BasicStats} instance and * returns statistics accordingly. Most of the methods are not implemented, so * it can only be used for Similarity unit testing. */ public class SpoofIndexSearcher extends IndexSearcher { - public SpoofIndexSearcher(EasyStats stats) { + public SpoofIndexSearcher(BasicStats stats) { super(new SpoofIndexReader(stats)); } public static class SpoofIndexReader extends IndexReader { /** The stats the reader has to return. */ - protected EasyStats stats; + protected BasicStats stats; /** The fields the reader has to return. */ protected SpoofFields fields; - public SpoofIndexReader(EasyStats stats) { + public SpoofIndexReader(BasicStats stats) { this.stats = stats; this.fields = new SpoofFields(stats); } @@ -163,7 +163,7 @@ public class SpoofIndexSearcher extends IndexSearcher { /** The stats the object has to return. */ protected SpoofTerms terms; - public SpoofFields(EasyStats stats) { + public SpoofFields(BasicStats stats) { this.terms = new SpoofTerms(stats); } @@ -183,9 +183,9 @@ public class SpoofIndexSearcher extends IndexSearcher { /** Spoof Terms class for Similarity testing. */ public static class SpoofTerms extends Terms { /** The stats the object has to return. */ - protected EasyStats stats; + protected BasicStats stats; - public SpoofTerms(EasyStats stats) { + public SpoofTerms(BasicStats stats) { this.stats = stats; } diff --git lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java deleted file mode 100644 index 456beed..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java +++ /dev/null @@ -1,586 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.OrdTermState; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.TermContext; - -/** - * Tests the {@link EasySimilarity}-based Similarities. Contains unit tests and - * integration tests for all Similarities and correctness tests for a select - * few. - *

This class maintains a list of - * {@code EasySimilarity} subclasses. Each test case performs its test on all - * items in the list. If a test case fails, the name of the Similarity that - * caused the failure is returned as part of the assertion error message.

- *

Unit testing is performed by constructing statistics manually and calling - * the {@link EasySimilarity#score(EasyStats, float, int)} method of the - * Similarities. The statistics represent corner cases of corpus distributions. - *

- *

For the integration tests, a small (8-document) collection is indexed. The - * tests verify that for a specific query, all relevant documents are returned - * in the correct order. The collection consists of two poems of English poet - * William Blake.

- *

Note: the list of Similarities is maintained by hand. If a new Similarity - * is added to the {@code org.apache.lucene.search.similarities} package, the - * list should be updated accordingly.

- *

- * In the correctness tests, the score is verified against the result of manual - * computation. Since it would be impossible to test all Similarities - * (e.g. all possible DFR combinations, all parameter values for LM), only - * the best performing setups in the original papers are verified. - *

- */ -public class TestEasySimilarity extends LuceneTestCase { - private static String FIELD_BODY = "body"; - private static String FIELD_ID = "id"; - /** The tolerance range for float equality. */ - private static float FLOAT_EPSILON = 1e-5f; - /** The DFR basic models to test. */ - private static BasicModel[] BASIC_MODELS; - /** The DFR aftereffects to test. */ - private static AfterEffect[] AFTER_EFFECTS; - /** The DFR normalizations to test. */ - private static Normalization[] NORMALIZATIONS; - /** The distributions for IB. */ - private static Distribution[] DISTRIBUTIONS; - /** Lambdas for IB. */ - private static Lambda[] LAMBDAS; - - static { - BASIC_MODELS = new BasicModel[] { - new BasicModelBE(), new BasicModelD(), new BasicModelG(), - new BasicModelIF(), new BasicModelIn(), new BasicModelIne(), - new BasicModelP() - }; - AFTER_EFFECTS = new AfterEffect[] { - new AfterEffectB(), new AfterEffectL(), new AfterEffect.NoAfterEffect() - }; - NORMALIZATIONS = new Normalization[] { - new NormalizationH1(), new NormalizationH2(), - new Normalization.NoNormalization() - }; - DISTRIBUTIONS = new Distribution[] { - new DistributionLL(), new DistributionSPL() - }; - LAMBDAS = new Lambda[] { - new LambdaDF(), new LambdaTTF() - }; - } - - private IndexSearcher searcher; - private Directory dir; - private IndexReader reader; - /** The list of similarities to test. */ - private List sims; - - @Override - public void setUp() throws Exception { - super.setUp(); - - dir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random, dir); - - for (int i = 0; i < docs.length; i++) { - Document d = new Document(); - d.add(newField(FIELD_ID, Integer.toString(i), Field.Store.YES, Field.Index.NO)); - d.add(newField(FIELD_BODY, docs[i], Field.Index.ANALYZED)); - writer.addDocument(d); - } - - reader = writer.getReader(); - searcher = newSearcher(reader); - writer.close(); - - sims = new ArrayList(); - for (BasicModel basicModel : BASIC_MODELS) { - for (AfterEffect afterEffect : AFTER_EFFECTS) { - for (Normalization normalization : NORMALIZATIONS) { - sims.add(new DFRSimilarity(basicModel, afterEffect, normalization)); - } - } - } - for (Distribution distribution : DISTRIBUTIONS) { - for (Lambda lambda : LAMBDAS) { - for (Normalization normalization : NORMALIZATIONS) { - sims.add(new IBSimilarity(distribution, lambda, normalization)); - } - } - } - sims.add(new LMDirichletSimilarity()); - sims.add(new LMJelinekMercerSimilarity(0.1f)); - sims.add(new LMJelinekMercerSimilarity(0.7f)); - } - - // ------------------------------- Unit tests -------------------------------- - - /** The default number of documents in the unit tests. */ - private static int NUMBER_OF_DOCUMENTS = 100; - /** The default total number of tokens in the field in the unit tests. */ - private static long NUMBER_OF_FIELD_TOKENS = 5000; - /** The default average field length in the unit tests. */ - private static float AVG_FIELD_LENGTH = 50; - /** The default document frequency in the unit tests. */ - private static int DOC_FREQ = 10; - /** - * The default total number of occurrences of this term across all documents - * in the unit tests. - */ - private static long TOTAL_TERM_FREQ = 70; - - /** The default tf in the unit tests. */ - private static float FREQ = 7; - /** The default document length in the unit tests. */ - private static int DOC_LEN = 40; - - /** Creates the default statistics object that the specific tests modify. */ - private EasyStats createStats() { - EasyStats stats = new EasyStats(1); - stats.setNumberOfDocuments(NUMBER_OF_DOCUMENTS); - stats.setNumberOfFieldTokens(NUMBER_OF_FIELD_TOKENS); - stats.setAvgFieldLength(AVG_FIELD_LENGTH); - stats.setDocFreq(DOC_FREQ); - stats.setTotalTermFreq(TOTAL_TERM_FREQ); - return stats; - } - - /** - * The generic test core called by all unit test methods. It calls the - * {@link EasySimilarity#score(EasyStats, float, int)} method of all - * Similarities in {@link #sims} and checks if the score is valid; i.e. it - * is a finite positive real number. - */ - private void unitTestCore(EasyStats stats, float freq, int docLen) - throws IOException { - // We have to fake everything, because computeStats() can be overridden and - // there is no way to inject false data after fillEasyStats(). - SpoofIndexSearcher searcher = new SpoofIndexSearcher(stats); - TermContext tc = new TermContext( - searcher.getIndexReader().getTopReaderContext(), - new OrdTermState(), 0, stats.getDocFreq(), stats.getTotalTermFreq()); - - for (EasySimilarity sim : sims) { - EasyStats realStats = sim.computeStats(new SpoofIndexSearcher(stats), - "spoof", stats.getTotalBoost(), tc); - float score = sim.score(realStats, freq, docLen); - float explScore = sim.explain( - realStats, 1, new Explanation(freq, "freq"), docLen).getValue(); - assertFalse("Score infinite: " + sim.toString(), Float.isInfinite(score)); - assertFalse("Score NaN: " + sim.toString(), Float.isNaN(score)); - assertTrue("Score negative: " + sim.toString(), score >= 0); - assertEquals("score() and explain() return different values: " - + sim.toString(), score, explScore, FLOAT_EPSILON); - } - } - - /** Runs the unit test with the default statistics. */ - public void testDefault() throws IOException { - unitTestCore(createStats(), FREQ, DOC_LEN); - } - - /** - * Tests correct behavior when - * {@code numberOfDocuments = numberOfFieldTokens}. - */ - public void testSparseDocuments() throws IOException { - EasyStats stats = createStats(); - stats.setNumberOfFieldTokens(stats.getNumberOfDocuments()); - stats.setTotalTermFreq(stats.getDocFreq()); - stats.setAvgFieldLength( - (float)stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); - unitTestCore(stats, FREQ, DOC_LEN); - } - - /** - * Tests correct behavior when - * {@code numberOfDocuments > numberOfFieldTokens}. - */ - public void testVerySparseDocuments() throws IOException { - EasyStats stats = createStats(); - stats.setNumberOfFieldTokens(stats.getNumberOfDocuments() * 2 / 3); - stats.setTotalTermFreq(stats.getDocFreq()); - stats.setAvgFieldLength( - (float)stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); - unitTestCore(stats, FREQ, DOC_LEN); - } - - /** - * Tests correct behavior when - * {@code NumberOfDocuments = 1}. - */ - public void testOneDocument() throws IOException { - EasyStats stats = createStats(); - stats.setNumberOfDocuments(1); - stats.setNumberOfFieldTokens(DOC_LEN); - stats.setAvgFieldLength(DOC_LEN); - stats.setDocFreq(1); - stats.setTotalTermFreq((int)FREQ); - unitTestCore(stats, FREQ, DOC_LEN); - } - - /** - * Tests correct behavior when - * {@code docFreq = numberOfDocuments}. - */ - public void testAllDocumentsRelevant() throws IOException { - EasyStats stats = createStats(); - float mult = (0.0f + stats.getNumberOfDocuments()) / stats.getDocFreq(); - stats.setTotalTermFreq((int)(stats.getTotalTermFreq() * mult)); - stats.setDocFreq(stats.getNumberOfDocuments()); - unitTestCore(stats, FREQ, DOC_LEN); - } - - /** - * Tests correct behavior when - * {@code docFreq > numberOfDocuments / 2}. - */ - public void testMostDocumentsRelevant() throws IOException { - EasyStats stats = createStats(); - float mult = (0.6f * stats.getNumberOfDocuments()) / stats.getDocFreq(); - stats.setTotalTermFreq((int)(stats.getTotalTermFreq() * mult)); - stats.setDocFreq((int)(stats.getNumberOfDocuments() * 0.6)); - unitTestCore(stats, FREQ, DOC_LEN); - } - - /** - * Tests correct behavior when - * {@code docFreq = 1}. - */ - public void testOnlyOneRelevantDocument() throws IOException { - EasyStats stats = createStats(); - stats.setDocFreq(1); - stats.setTotalTermFreq((int)FREQ + 3); - unitTestCore(stats, FREQ, DOC_LEN); - } - - /** - * Tests correct behavior when - * {@code totalTermFreq = numberOfFieldTokens}. - */ - public void testAllTermsRelevant() throws IOException { - EasyStats stats = createStats(); - stats.setTotalTermFreq(stats.getNumberOfFieldTokens()); - unitTestCore(stats, DOC_LEN, DOC_LEN); - // nocommit docLen > avglength - } - - /** - * Tests correct behavior when - * {@code totalTermFreq > numberOfDocuments}. - */ - public void testMoreTermsThanDocuments() throws IOException { - EasyStats stats = createStats(); - stats.setTotalTermFreq( - stats.getTotalTermFreq() + stats.getNumberOfDocuments()); - unitTestCore(stats, 2 * FREQ, DOC_LEN); - } - - /** - * Tests correct behavior when - * {@code totalTermFreq = numberOfDocuments}. - */ - public void testNumberOfTermsAsDocuments() throws IOException { - EasyStats stats = createStats(); - stats.setTotalTermFreq(stats.getNumberOfDocuments()); - unitTestCore(stats, FREQ, DOC_LEN); - } - - /** - * Tests correct behavior when {@code totalTermFreq = 1}. - */ - public void testOneTerm() throws IOException { - EasyStats stats = createStats(); - stats.setDocFreq(1); - stats.setTotalTermFreq(1); - unitTestCore(stats, 1, DOC_LEN); - } - - /** - * Tests correct behavior when {@code totalTermFreq = freq}. - */ - public void testOneRelevantDocument() throws IOException { - EasyStats stats = createStats(); - stats.setDocFreq(1); - stats.setTotalTermFreq((int)FREQ); - unitTestCore(stats, FREQ, DOC_LEN); - } - - /** - * Tests correct behavior when {@code numberOfFieldTokens = freq}. - */ - public void testAllTermsRelevantOnlyOneDocument() throws IOException { - EasyStats stats = createStats(); - stats.setNumberOfDocuments(10); - stats.setNumberOfFieldTokens(50); - stats.setAvgFieldLength(5); - stats.setDocFreq(1); - stats.setTotalTermFreq(50); - unitTestCore(stats, 50, 50); - } - - /** - * Tests correct behavior when there is only one document with a single term - * in the collection. - */ - public void testOnlyOneTermOneDocument() throws IOException { - EasyStats stats = createStats(); - stats.setNumberOfDocuments(1); - stats.setNumberOfFieldTokens(1); - stats.setAvgFieldLength(1); - stats.setDocFreq(1); - stats.setTotalTermFreq(1); - unitTestCore(stats, 1, 1); - } - - /** - * Tests correct behavior when there is only one term in the field, but - * more than one documents. - */ - public void testOnlyOneTerm() throws IOException { - EasyStats stats = createStats(); - stats.setNumberOfFieldTokens(1); - stats.setAvgFieldLength(1.0f / stats.getNumberOfDocuments()); - stats.setDocFreq(1); - stats.setTotalTermFreq(1); - unitTestCore(stats, 1, DOC_LEN); - } - - /** - * Tests correct behavior when {@code avgFieldLength = docLen}. - */ - public void testDocumentLengthAverage() throws IOException { - EasyStats stats = createStats(); - unitTestCore(stats, FREQ, (int)stats.getAvgFieldLength()); - } - - // ---------------------------- Correctness tests ---------------------------- - - /** Correctness test for the Dirichlet LM model. */ - public void testLMDirichlet() throws IOException { - float p = - (FREQ + 2000.0f * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f)) / - (DOC_LEN + 2000.0f); - float a = 2000.0f / (DOC_LEN + 2000.0f); - float gold = (float)( - Math.log(p / (a * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f))) + - Math.log(a)); - correctnessTestCore(new LMDirichletSimilarity(), gold); - } - - /** Correctness test for the Jelinek-Mercer LM model. */ - public void testLMJelinekMercer() throws IOException { - float p = (1 - 0.1f) * FREQ / DOC_LEN + - 0.1f * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f); - float gold = (float)(Math.log( - p / (0.1f * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f)))); - correctnessTestCore(new LMJelinekMercerSimilarity(0.1f), gold); - } - - /** - * Correctness test for the LL IB model with DF-based lambda and - * no normalization. - */ - public void testLLForIB() throws IOException { - EasySimilarity sim = new IBSimilarity(new DistributionLL(), new LambdaDF()); - correctnessTestCore(sim, 4.26267987704f); - } - - /** - * Correctness test for the SPL IB model with TTF-based lambda and - * no normalization. - */ - public void testSPLForIB() throws IOException { - EasySimilarity sim = - new IBSimilarity(new DistributionSPL(), new LambdaTTF()); - correctnessTestCore(sim, 2.24069910825f); - } - - /** Correctness test for the PL2 DFR model. */ - public void testPL2() throws IOException { - EasySimilarity sim = new DFRSimilarity( - new BasicModelP(), new AfterEffectL(), new NormalizationH2()); - float tfn = (float)(FREQ * EasySimilarity.log2( - 1 + AVG_FIELD_LENGTH / DOC_LEN)); // 8.1894750101 - float l = 1.0f / (tfn + 1.0f); // 0.108820144666 - float lambda = (1.0f * TOTAL_TERM_FREQ) / NUMBER_OF_DOCUMENTS; // 0.7 - float p = (float)(tfn * EasySimilarity.log2(tfn / lambda) + - (lambda + 1 / (12 * tfn) - tfn) * EasySimilarity.log2(Math.E) + - 0.5 * EasySimilarity.log2(2 * Math.PI * tfn)); // 21.1113611585 - float gold = l * p; // 2.29734137536 - correctnessTestCore(sim, gold); - } - - /** Correctness test for the IneB2 DFR model. */ - public void testIneB2() throws IOException { - EasySimilarity sim = new DFRSimilarity( - new BasicModelIne(), new AfterEffectB(), new NormalizationH2()); - correctnessTestCore(sim, 6.23455315685f); - } - - /** Correctness test for the GL1 DFR model. */ - public void testGL1() throws IOException { - EasySimilarity sim = new DFRSimilarity( - new BasicModelG(), new AfterEffectL(), new NormalizationH1()); - correctnessTestCore(sim, 1.22733118352f); - } - - /** Correctness test for the BEB1 DFR model. */ - public void testBEB1() throws IOException { - EasySimilarity sim = new DFRSimilarity( - new BasicModelBE(), new AfterEffectB(), new NormalizationH1()); - float tfn = FREQ * AVG_FIELD_LENGTH / DOC_LEN; // 8.75 - float b = (TOTAL_TERM_FREQ + 1) / (DOC_FREQ * (tfn + 1)); // 0.728205128205 - float n1 = NUMBER_OF_DOCUMENTS + 1 + TOTAL_TERM_FREQ - 1; // 170 - float m1 = NUMBER_OF_DOCUMENTS + 1 + TOTAL_TERM_FREQ - tfn - 2; // 160.25 - float n2 = TOTAL_TERM_FREQ; // 70 - float m2 = TOTAL_TERM_FREQ - tfn; // 61.25 - float be = (float)(-EasySimilarity.log2(NUMBER_OF_DOCUMENTS + 1 - 1) - - EasySimilarity.log2(Math.E) + // -8.08655123066 - ((m1 + 0.5f) * EasySimilarity.log2(n1 / m1) + - (n1 - m1) * EasySimilarity.log2(n1)) - // 85.9391317425 - ((m2 + 0.5f) * EasySimilarity.log2(n2 / m2) + - (n2 - m2) * EasySimilarity.log2(n2))); // 65.5270599612 - // 12.3255205506 - float gold = b * be; // 8.97550727277 - correctnessTestCore(sim, gold); - } - - /** Correctness test for the D DFR model (basic model only). */ - public void testD() throws IOException { - EasySimilarity sim = new DFRSimilarity(new BasicModelD()); - double p = 1.0 / (NUMBER_OF_DOCUMENTS + 1); // 0.009900990099 - double phi = FREQ / TOTAL_TERM_FREQ; // 0.1 - double D = phi * EasySimilarity.log2(phi / p) + // 0.209745318365 - (1 - phi) * EasySimilarity.log2((1 - phi) / (1 - p)); - float gold = (float)(TOTAL_TERM_FREQ * D + 0.5 * EasySimilarity.log2( - 1 + 2 * Math.PI * FREQ * (1 - phi))); // 17.3535930644 - correctnessTestCore(sim, gold); - } - - /** Correctness test for the In2 DFR model with no aftereffect. */ - public void testIn2() throws IOException { - EasySimilarity sim = new DFRSimilarity( - new BasicModelIn(), new NormalizationH2()); - float tfn = (float)(FREQ * EasySimilarity.log2( // 8.1894750101 - 1 + AVG_FIELD_LENGTH / DOC_LEN)); - float gold = (float)(tfn * EasySimilarity.log2( // 26.7459577898 - (NUMBER_OF_DOCUMENTS + 1) / (DOC_FREQ + 0.5))); - correctnessTestCore(sim, gold); - } - - /** Correctness test for the IFB DFR model with no normalization. */ - public void testIFB() throws IOException { - EasySimilarity sim = new DFRSimilarity( - new BasicModelIF(), new AfterEffectB()); - float B = (TOTAL_TERM_FREQ + 1) / (DOC_FREQ * (FREQ + 1)); // 0.8875 - float IF = (float)(FREQ * EasySimilarity.log2( // 8.97759389642 - 1 + (NUMBER_OF_DOCUMENTS + 1) / (TOTAL_TERM_FREQ + 0.5))); - float gold = B * IF; // 7.96761458307 - correctnessTestCore(sim, gold); - } - - /** - * The generic test core called by all correctness test methods. It calls the - * {@link EasySimilarity#score(EasyStats, float, int)} method of all - * Similarities in {@link #sims} and compares the score against the manually - * computed {@code gold}. - */ - private void correctnessTestCore(EasySimilarity sim, float gold) - throws IOException { - // We have to fake everything, because computeStats() can be overridden and - // there is no way to inject false data after fillEasyStats(). - EasyStats stats = createStats(); - SpoofIndexSearcher searcher = new SpoofIndexSearcher(stats); - TermContext tc = new TermContext( - searcher.getIndexReader().getTopReaderContext(), - new OrdTermState(), 0, stats.getDocFreq(), stats.getTotalTermFreq()); - - EasyStats realStats = sim.computeStats( - searcher, "spoof", stats.getTotalBoost(), tc); - float score = sim.score(realStats, FREQ, DOC_LEN); - assertEquals( - sim.toString() + " score not correct.", gold, score, FLOAT_EPSILON); - } - - // ---------------------------- Integration tests ---------------------------- - - /** The "collection" for the integration tests. */ - String[] docs = new String[] { - "Tiger, tiger burning bright In the forest of the night What immortal hand or eye Could frame thy fearful symmetry ?", - "In what distant depths or skies Burnt the fire of thine eyes ? On what wings dare he aspire ? What the hands the seize the fire ?", - "And what shoulder and what art Could twist the sinews of thy heart ? And when thy heart began to beat What dread hand ? And what dread feet ?", - "What the hammer? What the chain ? In what furnace was thy brain ? What the anvil ? And what dread grasp Dare its deadly terrors clasp ?", - "And when the stars threw down their spears And water'd heaven with their tear Did he smile his work to see ? Did he, who made the lamb, made thee ?", - "Tiger, tiger burning bright In the forest of the night What immortal hand or eye Dare frame thy fearful symmetry ?", - "Cruelty has a human heart And jealousy a human face Terror the human form divine And Secrecy the human dress .", - "The human dress is forg'd iron The human form a fiery forge The human face a furnace seal'd The human heart its fiery gorge ." - }; - - /** - * Tests whether all similarities return three documents for the query word - * "heart". - */ - public void testHeartList() throws IOException { - Query q = new TermQuery(new Term(FIELD_BODY, "heart")); - - for (EasySimilarity sim : sims) { - searcher.setSimilarityProvider(new BasicSimilarityProvider(sim)); - TopDocs topDocs = searcher.search(q, 1000); - assertEquals("Failed: " + sim.toString(), 3, topDocs.totalHits); - } - } - - /** Test whether all similarities return document 3 before documents 7 and 8. */ - public void testHeartRanking() throws IOException { - assumeFalse("PreFlex codec does not support the stats necessary for this test!", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); - - Query q = new TermQuery(new Term(FIELD_BODY, "heart")); - - for (EasySimilarity sim : sims) { - searcher.setSimilarityProvider(new BasicSimilarityProvider(sim)); - TopDocs topDocs = searcher.search(q, 1000); - assertEquals("Failed: " + sim.toString(), 2, topDocs.scoreDocs[0].doc); - } - } - - @Override - public void tearDown() throws Exception { - searcher.close(); - reader.close(); - dir.close(); - super.tearDown(); - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java new file mode 100644 index 0000000..2070ce6 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java @@ -0,0 +1,586 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.OrdTermState; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TermContext; + +/** + * Tests the {@link SimilarityBase}-based Similarities. Contains unit tests and + * integration tests for all Similarities and correctness tests for a select + * few. + *

This class maintains a list of + * {@code SimilarityBase} subclasses. Each test case performs its test on all + * items in the list. If a test case fails, the name of the Similarity that + * caused the failure is returned as part of the assertion error message.

+ *

Unit testing is performed by constructing statistics manually and calling + * the {@link SimilarityBase#score(BasicStats, float, int)} method of the + * Similarities. The statistics represent corner cases of corpus distributions. + *

+ *

For the integration tests, a small (8-document) collection is indexed. The + * tests verify that for a specific query, all relevant documents are returned + * in the correct order. The collection consists of two poems of English poet + * William Blake.

+ *

Note: the list of Similarities is maintained by hand. If a new Similarity + * is added to the {@code org.apache.lucene.search.similarities} package, the + * list should be updated accordingly.

+ *

+ * In the correctness tests, the score is verified against the result of manual + * computation. Since it would be impossible to test all Similarities + * (e.g. all possible DFR combinations, all parameter values for LM), only + * the best performing setups in the original papers are verified. + *

+ */ +public class TestSimilarityBase extends LuceneTestCase { + private static String FIELD_BODY = "body"; + private static String FIELD_ID = "id"; + /** The tolerance range for float equality. */ + private static float FLOAT_EPSILON = 1e-5f; + /** The DFR basic models to test. */ + private static BasicModel[] BASIC_MODELS; + /** The DFR aftereffects to test. */ + private static AfterEffect[] AFTER_EFFECTS; + /** The DFR normalizations to test. */ + private static Normalization[] NORMALIZATIONS; + /** The distributions for IB. */ + private static Distribution[] DISTRIBUTIONS; + /** Lambdas for IB. */ + private static Lambda[] LAMBDAS; + + static { + BASIC_MODELS = new BasicModel[] { + new BasicModelBE(), new BasicModelD(), new BasicModelG(), + new BasicModelIF(), new BasicModelIn(), new BasicModelIne(), + new BasicModelP() + }; + AFTER_EFFECTS = new AfterEffect[] { + new AfterEffectB(), new AfterEffectL(), new AfterEffect.NoAfterEffect() + }; + NORMALIZATIONS = new Normalization[] { + new NormalizationH1(), new NormalizationH2(), + new Normalization.NoNormalization() + }; + DISTRIBUTIONS = new Distribution[] { + new DistributionLL(), new DistributionSPL() + }; + LAMBDAS = new Lambda[] { + new LambdaDF(), new LambdaTTF() + }; + } + + private IndexSearcher searcher; + private Directory dir; + private IndexReader reader; + /** The list of similarities to test. */ + private List sims; + + @Override + public void setUp() throws Exception { + super.setUp(); + + dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random, dir); + + for (int i = 0; i < docs.length; i++) { + Document d = new Document(); + d.add(newField(FIELD_ID, Integer.toString(i), Field.Store.YES, Field.Index.NO)); + d.add(newField(FIELD_BODY, docs[i], Field.Index.ANALYZED)); + writer.addDocument(d); + } + + reader = writer.getReader(); + searcher = newSearcher(reader); + writer.close(); + + sims = new ArrayList(); + for (BasicModel basicModel : BASIC_MODELS) { + for (AfterEffect afterEffect : AFTER_EFFECTS) { + for (Normalization normalization : NORMALIZATIONS) { + sims.add(new DFRSimilarity(basicModel, afterEffect, normalization)); + } + } + } + for (Distribution distribution : DISTRIBUTIONS) { + for (Lambda lambda : LAMBDAS) { + for (Normalization normalization : NORMALIZATIONS) { + sims.add(new IBSimilarity(distribution, lambda, normalization)); + } + } + } + sims.add(new LMDirichletSimilarity()); + sims.add(new LMJelinekMercerSimilarity(0.1f)); + sims.add(new LMJelinekMercerSimilarity(0.7f)); + } + + // ------------------------------- Unit tests -------------------------------- + + /** The default number of documents in the unit tests. */ + private static int NUMBER_OF_DOCUMENTS = 100; + /** The default total number of tokens in the field in the unit tests. */ + private static long NUMBER_OF_FIELD_TOKENS = 5000; + /** The default average field length in the unit tests. */ + private static float AVG_FIELD_LENGTH = 50; + /** The default document frequency in the unit tests. */ + private static int DOC_FREQ = 10; + /** + * The default total number of occurrences of this term across all documents + * in the unit tests. + */ + private static long TOTAL_TERM_FREQ = 70; + + /** The default tf in the unit tests. */ + private static float FREQ = 7; + /** The default document length in the unit tests. */ + private static int DOC_LEN = 40; + + /** Creates the default statistics object that the specific tests modify. */ + private BasicStats createStats() { + BasicStats stats = new BasicStats(1); + stats.setNumberOfDocuments(NUMBER_OF_DOCUMENTS); + stats.setNumberOfFieldTokens(NUMBER_OF_FIELD_TOKENS); + stats.setAvgFieldLength(AVG_FIELD_LENGTH); + stats.setDocFreq(DOC_FREQ); + stats.setTotalTermFreq(TOTAL_TERM_FREQ); + return stats; + } + + /** + * The generic test core called by all unit test methods. It calls the + * {@link SimilarityBase#score(BasicStats, float, int)} method of all + * Similarities in {@link #sims} and checks if the score is valid; i.e. it + * is a finite positive real number. + */ + private void unitTestCore(BasicStats stats, float freq, int docLen) + throws IOException { + // We have to fake everything, because computeStats() can be overridden and + // there is no way to inject false data after fillBasicStats(). + SpoofIndexSearcher searcher = new SpoofIndexSearcher(stats); + TermContext tc = new TermContext( + searcher.getIndexReader().getTopReaderContext(), + new OrdTermState(), 0, stats.getDocFreq(), stats.getTotalTermFreq()); + + for (SimilarityBase sim : sims) { + BasicStats realStats = sim.computeStats(new SpoofIndexSearcher(stats), + "spoof", stats.getTotalBoost(), tc); + float score = sim.score(realStats, freq, docLen); + float explScore = sim.explain( + realStats, 1, new Explanation(freq, "freq"), docLen).getValue(); + assertFalse("Score infinite: " + sim.toString(), Float.isInfinite(score)); + assertFalse("Score NaN: " + sim.toString(), Float.isNaN(score)); + assertTrue("Score negative: " + sim.toString(), score >= 0); + assertEquals("score() and explain() return different values: " + + sim.toString(), score, explScore, FLOAT_EPSILON); + } + } + + /** Runs the unit test with the default statistics. */ + public void testDefault() throws IOException { + unitTestCore(createStats(), FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code numberOfDocuments = numberOfFieldTokens}. + */ + public void testSparseDocuments() throws IOException { + BasicStats stats = createStats(); + stats.setNumberOfFieldTokens(stats.getNumberOfDocuments()); + stats.setTotalTermFreq(stats.getDocFreq()); + stats.setAvgFieldLength( + (float)stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code numberOfDocuments > numberOfFieldTokens}. + */ + public void testVerySparseDocuments() throws IOException { + BasicStats stats = createStats(); + stats.setNumberOfFieldTokens(stats.getNumberOfDocuments() * 2 / 3); + stats.setTotalTermFreq(stats.getDocFreq()); + stats.setAvgFieldLength( + (float)stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code NumberOfDocuments = 1}. + */ + public void testOneDocument() throws IOException { + BasicStats stats = createStats(); + stats.setNumberOfDocuments(1); + stats.setNumberOfFieldTokens(DOC_LEN); + stats.setAvgFieldLength(DOC_LEN); + stats.setDocFreq(1); + stats.setTotalTermFreq((int)FREQ); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code docFreq = numberOfDocuments}. + */ + public void testAllDocumentsRelevant() throws IOException { + BasicStats stats = createStats(); + float mult = (0.0f + stats.getNumberOfDocuments()) / stats.getDocFreq(); + stats.setTotalTermFreq((int)(stats.getTotalTermFreq() * mult)); + stats.setDocFreq(stats.getNumberOfDocuments()); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code docFreq > numberOfDocuments / 2}. + */ + public void testMostDocumentsRelevant() throws IOException { + BasicStats stats = createStats(); + float mult = (0.6f * stats.getNumberOfDocuments()) / stats.getDocFreq(); + stats.setTotalTermFreq((int)(stats.getTotalTermFreq() * mult)); + stats.setDocFreq((int)(stats.getNumberOfDocuments() * 0.6)); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code docFreq = 1}. + */ + public void testOnlyOneRelevantDocument() throws IOException { + BasicStats stats = createStats(); + stats.setDocFreq(1); + stats.setTotalTermFreq((int)FREQ + 3); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code totalTermFreq = numberOfFieldTokens}. + */ + public void testAllTermsRelevant() throws IOException { + BasicStats stats = createStats(); + stats.setTotalTermFreq(stats.getNumberOfFieldTokens()); + unitTestCore(stats, DOC_LEN, DOC_LEN); + // nocommit docLen > avglength + } + + /** + * Tests correct behavior when + * {@code totalTermFreq > numberOfDocuments}. + */ + public void testMoreTermsThanDocuments() throws IOException { + BasicStats stats = createStats(); + stats.setTotalTermFreq( + stats.getTotalTermFreq() + stats.getNumberOfDocuments()); + unitTestCore(stats, 2 * FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code totalTermFreq = numberOfDocuments}. + */ + public void testNumberOfTermsAsDocuments() throws IOException { + BasicStats stats = createStats(); + stats.setTotalTermFreq(stats.getNumberOfDocuments()); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when {@code totalTermFreq = 1}. + */ + public void testOneTerm() throws IOException { + BasicStats stats = createStats(); + stats.setDocFreq(1); + stats.setTotalTermFreq(1); + unitTestCore(stats, 1, DOC_LEN); + } + + /** + * Tests correct behavior when {@code totalTermFreq = freq}. + */ + public void testOneRelevantDocument() throws IOException { + BasicStats stats = createStats(); + stats.setDocFreq(1); + stats.setTotalTermFreq((int)FREQ); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when {@code numberOfFieldTokens = freq}. + */ + public void testAllTermsRelevantOnlyOneDocument() throws IOException { + BasicStats stats = createStats(); + stats.setNumberOfDocuments(10); + stats.setNumberOfFieldTokens(50); + stats.setAvgFieldLength(5); + stats.setDocFreq(1); + stats.setTotalTermFreq(50); + unitTestCore(stats, 50, 50); + } + + /** + * Tests correct behavior when there is only one document with a single term + * in the collection. + */ + public void testOnlyOneTermOneDocument() throws IOException { + BasicStats stats = createStats(); + stats.setNumberOfDocuments(1); + stats.setNumberOfFieldTokens(1); + stats.setAvgFieldLength(1); + stats.setDocFreq(1); + stats.setTotalTermFreq(1); + unitTestCore(stats, 1, 1); + } + + /** + * Tests correct behavior when there is only one term in the field, but + * more than one documents. + */ + public void testOnlyOneTerm() throws IOException { + BasicStats stats = createStats(); + stats.setNumberOfFieldTokens(1); + stats.setAvgFieldLength(1.0f / stats.getNumberOfDocuments()); + stats.setDocFreq(1); + stats.setTotalTermFreq(1); + unitTestCore(stats, 1, DOC_LEN); + } + + /** + * Tests correct behavior when {@code avgFieldLength = docLen}. + */ + public void testDocumentLengthAverage() throws IOException { + BasicStats stats = createStats(); + unitTestCore(stats, FREQ, (int)stats.getAvgFieldLength()); + } + + // ---------------------------- Correctness tests ---------------------------- + + /** Correctness test for the Dirichlet LM model. */ + public void testLMDirichlet() throws IOException { + float p = + (FREQ + 2000.0f * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f)) / + (DOC_LEN + 2000.0f); + float a = 2000.0f / (DOC_LEN + 2000.0f); + float gold = (float)( + Math.log(p / (a * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f))) + + Math.log(a)); + correctnessTestCore(new LMDirichletSimilarity(), gold); + } + + /** Correctness test for the Jelinek-Mercer LM model. */ + public void testLMJelinekMercer() throws IOException { + float p = (1 - 0.1f) * FREQ / DOC_LEN + + 0.1f * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f); + float gold = (float)(Math.log( + p / (0.1f * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f)))); + correctnessTestCore(new LMJelinekMercerSimilarity(0.1f), gold); + } + + /** + * Correctness test for the LL IB model with DF-based lambda and + * no normalization. + */ + public void testLLForIB() throws IOException { + SimilarityBase sim = new IBSimilarity(new DistributionLL(), new LambdaDF()); + correctnessTestCore(sim, 4.26267987704f); + } + + /** + * Correctness test for the SPL IB model with TTF-based lambda and + * no normalization. + */ + public void testSPLForIB() throws IOException { + SimilarityBase sim = + new IBSimilarity(new DistributionSPL(), new LambdaTTF()); + correctnessTestCore(sim, 2.24069910825f); + } + + /** Correctness test for the PL2 DFR model. */ + public void testPL2() throws IOException { + SimilarityBase sim = new DFRSimilarity( + new BasicModelP(), new AfterEffectL(), new NormalizationH2()); + float tfn = (float)(FREQ * SimilarityBase.log2( + 1 + AVG_FIELD_LENGTH / DOC_LEN)); // 8.1894750101 + float l = 1.0f / (tfn + 1.0f); // 0.108820144666 + float lambda = (1.0f * TOTAL_TERM_FREQ) / NUMBER_OF_DOCUMENTS; // 0.7 + float p = (float)(tfn * SimilarityBase.log2(tfn / lambda) + + (lambda + 1 / (12 * tfn) - tfn) * SimilarityBase.log2(Math.E) + + 0.5 * SimilarityBase.log2(2 * Math.PI * tfn)); // 21.1113611585 + float gold = l * p; // 2.29734137536 + correctnessTestCore(sim, gold); + } + + /** Correctness test for the IneB2 DFR model. */ + public void testIneB2() throws IOException { + SimilarityBase sim = new DFRSimilarity( + new BasicModelIne(), new AfterEffectB(), new NormalizationH2()); + correctnessTestCore(sim, 6.23455315685f); + } + + /** Correctness test for the GL1 DFR model. */ + public void testGL1() throws IOException { + SimilarityBase sim = new DFRSimilarity( + new BasicModelG(), new AfterEffectL(), new NormalizationH1()); + correctnessTestCore(sim, 1.22733118352f); + } + + /** Correctness test for the BEB1 DFR model. */ + public void testBEB1() throws IOException { + SimilarityBase sim = new DFRSimilarity( + new BasicModelBE(), new AfterEffectB(), new NormalizationH1()); + float tfn = FREQ * AVG_FIELD_LENGTH / DOC_LEN; // 8.75 + float b = (TOTAL_TERM_FREQ + 1) / (DOC_FREQ * (tfn + 1)); // 0.728205128205 + float n1 = NUMBER_OF_DOCUMENTS + 1 + TOTAL_TERM_FREQ - 1; // 170 + float m1 = NUMBER_OF_DOCUMENTS + 1 + TOTAL_TERM_FREQ - tfn - 2; // 160.25 + float n2 = TOTAL_TERM_FREQ; // 70 + float m2 = TOTAL_TERM_FREQ - tfn; // 61.25 + float be = (float)(-SimilarityBase.log2(NUMBER_OF_DOCUMENTS + 1 - 1) - + SimilarityBase.log2(Math.E) + // -8.08655123066 + ((m1 + 0.5f) * SimilarityBase.log2(n1 / m1) + + (n1 - m1) * SimilarityBase.log2(n1)) - // 85.9391317425 + ((m2 + 0.5f) * SimilarityBase.log2(n2 / m2) + + (n2 - m2) * SimilarityBase.log2(n2))); // 65.5270599612 + // 12.3255205506 + float gold = b * be; // 8.97550727277 + correctnessTestCore(sim, gold); + } + + /** Correctness test for the D DFR model (basic model only). */ + public void testD() throws IOException { + SimilarityBase sim = new DFRSimilarity(new BasicModelD()); + double p = 1.0 / (NUMBER_OF_DOCUMENTS + 1); // 0.009900990099 + double phi = FREQ / TOTAL_TERM_FREQ; // 0.1 + double D = phi * SimilarityBase.log2(phi / p) + // 0.209745318365 + (1 - phi) * SimilarityBase.log2((1 - phi) / (1 - p)); + float gold = (float)(TOTAL_TERM_FREQ * D + 0.5 * SimilarityBase.log2( + 1 + 2 * Math.PI * FREQ * (1 - phi))); // 17.3535930644 + correctnessTestCore(sim, gold); + } + + /** Correctness test for the In2 DFR model with no aftereffect. */ + public void testIn2() throws IOException { + SimilarityBase sim = new DFRSimilarity( + new BasicModelIn(), new NormalizationH2()); + float tfn = (float)(FREQ * SimilarityBase.log2( // 8.1894750101 + 1 + AVG_FIELD_LENGTH / DOC_LEN)); + float gold = (float)(tfn * SimilarityBase.log2( // 26.7459577898 + (NUMBER_OF_DOCUMENTS + 1) / (DOC_FREQ + 0.5))); + correctnessTestCore(sim, gold); + } + + /** Correctness test for the IFB DFR model with no normalization. */ + public void testIFB() throws IOException { + SimilarityBase sim = new DFRSimilarity( + new BasicModelIF(), new AfterEffectB()); + float B = (TOTAL_TERM_FREQ + 1) / (DOC_FREQ * (FREQ + 1)); // 0.8875 + float IF = (float)(FREQ * SimilarityBase.log2( // 8.97759389642 + 1 + (NUMBER_OF_DOCUMENTS + 1) / (TOTAL_TERM_FREQ + 0.5))); + float gold = B * IF; // 7.96761458307 + correctnessTestCore(sim, gold); + } + + /** + * The generic test core called by all correctness test methods. It calls the + * {@link SimilarityBase#score(BasicStats, float, int)} method of all + * Similarities in {@link #sims} and compares the score against the manually + * computed {@code gold}. + */ + private void correctnessTestCore(SimilarityBase sim, float gold) + throws IOException { + // We have to fake everything, because computeStats() can be overridden and + // there is no way to inject false data after fillBasicStats(). + BasicStats stats = createStats(); + SpoofIndexSearcher searcher = new SpoofIndexSearcher(stats); + TermContext tc = new TermContext( + searcher.getIndexReader().getTopReaderContext(), + new OrdTermState(), 0, stats.getDocFreq(), stats.getTotalTermFreq()); + + BasicStats realStats = sim.computeStats( + searcher, "spoof", stats.getTotalBoost(), tc); + float score = sim.score(realStats, FREQ, DOC_LEN); + assertEquals( + sim.toString() + " score not correct.", gold, score, FLOAT_EPSILON); + } + + // ---------------------------- Integration tests ---------------------------- + + /** The "collection" for the integration tests. */ + String[] docs = new String[] { + "Tiger, tiger burning bright In the forest of the night What immortal hand or eye Could frame thy fearful symmetry ?", + "In what distant depths or skies Burnt the fire of thine eyes ? On what wings dare he aspire ? What the hands the seize the fire ?", + "And what shoulder and what art Could twist the sinews of thy heart ? And when thy heart began to beat What dread hand ? And what dread feet ?", + "What the hammer? What the chain ? In what furnace was thy brain ? What the anvil ? And what dread grasp Dare its deadly terrors clasp ?", + "And when the stars threw down their spears And water'd heaven with their tear Did he smile his work to see ? Did he, who made the lamb, made thee ?", + "Tiger, tiger burning bright In the forest of the night What immortal hand or eye Dare frame thy fearful symmetry ?", + "Cruelty has a human heart And jealousy a human face Terror the human form divine And Secrecy the human dress .", + "The human dress is forg'd iron The human form a fiery forge The human face a furnace seal'd The human heart its fiery gorge ." + }; + + /** + * Tests whether all similarities return three documents for the query word + * "heart". + */ + public void testHeartList() throws IOException { + Query q = new TermQuery(new Term(FIELD_BODY, "heart")); + + for (SimilarityBase sim : sims) { + searcher.setSimilarityProvider(new BasicSimilarityProvider(sim)); + TopDocs topDocs = searcher.search(q, 1000); + assertEquals("Failed: " + sim.toString(), 3, topDocs.totalHits); + } + } + + /** Test whether all similarities return document 3 before documents 7 and 8. */ + public void testHeartRanking() throws IOException { + assumeFalse("PreFlex codec does not support the stats necessary for this test!", + "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + + Query q = new TermQuery(new Term(FIELD_BODY, "heart")); + + for (SimilarityBase sim : sims) { + searcher.setSimilarityProvider(new BasicSimilarityProvider(sim)); + TopDocs topDocs = searcher.search(q, 1000); + assertEquals("Failed: " + sim.toString(), 2, topDocs.scoreDocs[0].doc); + } + } + + @Override + public void tearDown() throws Exception { + searcher.close(); + reader.close(); + dir.close(); + super.tearDown(); + } +}