Most basic models use the number of documents and the total term + * frequency to compute Inf1. This method provides a generic + * explanation for such models. Subclasses that use other statistics must + * override this method.
+ */ + public Explanation explain(EasyStats stats, float tfn) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(score(stats, tfn)); + result.addDetail(new Explanation(tfn, "tfn")); + result.addDetail( + new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); + result.addDetail( + new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); + return result; + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java new file mode 100644 index 0000000..a73d2b5 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java @@ -0,0 +1,22 @@ +package org.apache.lucene.search.similarities; + +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Limiting form of the Bose-Einstein model. + * @lucene.experimental + */ +public class BasicModelBE extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + long N = stats.getNumberOfDocuments(); + long F = stats.getTotalTermFreq(); + return (float)(-log2((N - 1) * Math.E) + + f(N + F -1, N + F - tfn - 2) - f(F, F - tfn)); + } + + /** The f helper function defined for BE. */ + private final double f(long n, float m) { + return (m + 0.5) * log2((double)n / m) + (n - m) * log2(n); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java new file mode 100644 index 0000000..8c5be42 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java @@ -0,0 +1,20 @@ +package org.apache.lucene.search.similarities; + +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Implements the approximation of the binomial model with the divergence + * for DFR. + * @lucene.experimental + */ +public class BasicModelD extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + long F = stats.getTotalTermFreq(); + double phi = (double)tfn / F; + double nphi = 1 - phi; + double p = 1.0 / stats.getNumberOfDocuments(); + double D = phi * log2(phi / p) + nphi * log2(nphi / (1 - p)); + return (float)(D * F + 0.5 * log2(2 * Math.PI * tfn * nphi)); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java new file mode 100644 index 0000000..59bbc73 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java @@ -0,0 +1,16 @@ +package org.apache.lucene.search.similarities; + +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Geometric as limiting form of the Bose-Einstein model. + * @lucene.experimental + */ +public class BasicModelG extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + double lambda = stats.getTotalTermFreq() / stats.getNumberOfDocuments(); + // -log(1 / (lambda + 1)) -> log(lambda + 1) + return (float)(log2(lambda + 1) + tfn * log2((1 + lambda) / lambda)); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java new file mode 100644 index 0000000..73ed7de --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java @@ -0,0 +1,17 @@ +package org.apache.lucene.search.similarities; + +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * An approximation of the I(ne) model. + * @lucene.experimental + */ +public class BasicModelIF extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + // TODO: Refactor this method to a parent class? See the other two Ix model. + int N = stats.getNumberOfDocuments(); + long F = stats.getTotalTermFreq(); + return tfn * (float)(log2((N + 1) / (F + 0.5))); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java new file mode 100644 index 0000000..abaa7a7 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java @@ -0,0 +1,30 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * The basic tf-idf model of randomness. + * @lucene.experimental + */ +public class BasicModelIn extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + int N = stats.getNumberOfDocuments(); + int n = stats.getDocFreq(); + return tfn * (float)(log2((N + 1) / (n + 0.5))); + } + + @Override + public Explanation explain(EasyStats stats, float tfn) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(score(stats, tfn)); + result.addDetail(new Explanation(tfn, "tfn")); + result.addDetail( + new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); + result.addDetail( + new Explanation(stats.getDocFreq(), "docFreq")); + return result; + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java new file mode 100644 index 0000000..b019183 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java @@ -0,0 +1,18 @@ +package org.apache.lucene.search.similarities; + +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Tf-idf model of randomness, based on a mixture of Poisson and inverse + * document frequency. + * @lucene.experimental + */ +public class BasicModelIne extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + int N = stats.getNumberOfDocuments(); + long F = stats.getTotalTermFreq(); + double ne = N * (1 - Math.pow((N - 1) / N, F)); + return tfn * (float)(log2((N + 1) / (ne + 0.5))); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java new file mode 100644 index 0000000..00081d8 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java @@ -0,0 +1,17 @@ +package org.apache.lucene.search.similarities; + +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Implements the Poisson approximation for the binomial model for DFR. + * @lucene.experimental + */ +public class BasicModelP extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + float lambda = (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); + return (float)(tfn * log2(tfn / lambda) + + (lambda + 1 / 12 / tfn - tfn) * log2(Math.E) + + 0.5 * log2(2 * Math.PI * tfn)); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java new file mode 100644 index 0000000..4aa9c76 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java @@ -0,0 +1,75 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; + +/** + * Implements the divergence from randomness (DFR) framework + * introduced in Gianni Amati and Cornelis Joost Van Rijsbergen. 2002. + * Probabilistic models of information retrieval based on measuring the + * divergence from randomness. ACM Trans. Inf. Syst. 20, 4 (October 2002), + * 357-389. + *The DFR scoring formula is composed of three separate components: the + * basic model, the aftereffect and an additional + * normalization component, represented by the classes + * {@code BasicModel}, {@code AfterEffect} and {@code Normalization}, + * respectively. The names of these classes were chosen to match the names of + * their counterparts in the Terrier IR engine.
+ *Note that qtf, the multiplicity of term-occurrence in the query, + * is not handled by this implementation.
+ * + * @see BasicModel + * @see AfterEffect + * @see Normalization + * @lucene.experimental + */ +public class DFRSimilarity extends EasySimilarity { + /** For {@link #log2(double)}. Precomputed for efficiency reasons. */ + private static final double LOG_2 = Math.log(2); + + /** The basic model for information content. */ + protected final BasicModel basicModel; + /** The first normalization of the information content. */ + protected final AfterEffect afterEffect; + /** The term frequency normalization. */ + protected final Normalization normalization; + + public DFRSimilarity(ClassUnlike for DFR, the natural logarithm is used, as + * it is faster to compute and the original paper does not express any + * preference to a specific base.
+ * @lucene.experimental + */ +public abstract class DistributionLL extends Distribution { + @Override + public float score(EasyStats stats, float tfn, float lambda) { + return (float)-Math.log(lambda / (tfn + lambda)); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java new file mode 100644 index 0000000..80310b4 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java @@ -0,0 +1,17 @@ +package org.apache.lucene.search.similarities; + +/** + * The smoothed power-law (SPL) distribution for the information-based framework + * that is described in the original paper. + *Unlike for DFR, the natural logarithm is used, as + * it is faster to compute and the original paper does not express any + * preference to a specific base.
+ * @lucene.experimental + */ +public abstract class DistributionSPL extends Distribution { + @Override + public float score(EasyStats stats, float tfn, float lambda) { + return (float)-Math.log( + (Math.pow(lambda, (tfn / (tfn + 1))) - lambda) / (1 - lambda)); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java new file mode 100644 index 0000000..83b6b05 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java @@ -0,0 +1,205 @@ +package org.apache.lucene.search.similarities; + +import java.io.IOException; + +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Similarity; +import org.apache.lucene.util.SmallFloat; +import org.apache.lucene.util.TermContext; + +/** + * A subclass of {@code Similarity} that provides a simplified API for its + * descendants. + * @lucene.experimental + */ +public abstract class EasySimilarity extends Similarity { + @Override + public EasyStats computeStats(IndexSearcher searcher, String fieldName, + float queryBoost, TermContext... termContexts) throws IOException { + EasyStats stats = new EasyStats(queryBoost); + IndexReader reader = searcher.getIndexReader(); + int numberOfDocuments = reader.maxDoc(); + long numberOfFieldTokens = reader.hasNorms(fieldName) + ? reader.getSumOfNorms(fieldName) : 0; + // nocommit The norm should be smooth enough for this to work! + int avgFieldLengthNorm = + (int)(numberOfFieldTokens / (double)numberOfDocuments); + float avgFieldLength = decodeNormValue((byte)avgFieldLengthNorm); + // Is this OK? + int docFreq = 0; + long totalTermFreq = 0; + for (final TermContext context : termContexts) { + docFreq += context.docFreq(); + totalTermFreq += context.totalTermFreq(); + } + // Another source. What about the root context? Are the others OK? + long sumTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), + fieldName).getSumTotalTermFreq(); + + stats.setNumberOfDocuments(numberOfDocuments); + stats.setNumberOfFieldTokens(numberOfFieldTokens); + stats.setAvgFieldLength(avgFieldLength); + stats.setDocFreq(docFreq); + stats.setTotalTermFreq(totalTermFreq); + stats.setSumTotalTermFreq(sumTotalTermFreq); + // nocommit uniqueTermCount? + return stats; + } + + /** Uses the {@code 1 / (distance + 1)} formula. */ + @Override + public float sloppyFreq(int distance) { + // nocommit to separate class + return 1.0f / (distance + 1); + } + + /** Encodes the document length. */ + @Override + public byte computeNorm(FieldInvertState state) { + return encodeNormValue(state.getLength()); + } + + /** Decodes a normalization factor stored in an index. + * @see #encodeNormValue(float) + */ + // nocommit to protected? + // nocommit is int OK? + public int decodeNormValue(byte norm) { + // SmallFloat seems OK, because tf is smoothed anyway. + return (int)SmallFloat.byte315ToFloat(norm); + } + + /** Encodes the length to a byte via SmallInt. */ + // nocommit to protected? + public byte encodeNormValue(int length) { + // SmallFloat seems OK, because tf is smoothed anyway. + return SmallFloat.floatToByte315(length); + } + + /** + * Scores the document {@code doc}. + *Subclasses must apply their scoring formula in this class.
+ * @param stats the corpus level statistics. + * @param freq the term frequency. + * @param norm the current document's field norm. + * @return the score. + */ + protected abstract float score(EasyStats stats, float freq, byte norm); + + /** + * Subclasses should implement this method to explain the score. {@code expl} + * already contains the score, the name of the class and the doc id, as well + * as the term frequency and its explanation; subclasses can add additional + * clauses to explain details of their scoring formulae. + *The default implementation does nothing.
+ * + * @param expl the explanation to extend with details. + * @param stats the corpus level statistics. + * @param doc the document id. + * @param freq the term frequency. + * @param norm the current document's field norm. + */ + protected void explain( + Explanation expl, EasyStats stats, int doc, float freq, byte norm) {} + + /** + * Explains the score. The implementation here provides a basic explanation + * in the format score(name-of-similarity, doc=doc-id, + * freq=term-frequency), computed from:, and + * attaches the score (computed via the {@link #score(EasyStats, float, byte)} + * method) and the explanation for the term frequency. Subclasses content with + * this format may add additional details in + * {@link #explain(Explanation, EasyStats, int, float, byte)}. + * + * @param stats the corpus level statistics. + * @param doc the document id. + * @param freq the term frequency and its explanation. + * @param norm the current document's field norm. + * @return the explanation. + */ + protected Explanation explain( + EasyStats stats, int doc, Explanation freq, byte norm) { + Explanation result = new Explanation(); + result.setValue(score(stats, freq.getValue(), norm)); + result.setDescription("score(" + getClass().getSimpleName() + + ", doc=" + doc + ", freq=" + freq.getValue() +"), computed from:"); + result.addDetail(freq); + + explain(result, stats, doc, freq.getValue(), norm); + + return result; + } + + @Override + public ExactDocScorer exactDocScorer(Stats stats, String fieldName, + AtomicReaderContext context) throws IOException { + return new EasyExactDocScorer((EasyStats) stats, + context.reader.norms(fieldName)); + } + + @Override + public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, + AtomicReaderContext context) throws IOException { + return new EasySloppyDocScorer((EasyStats) stats, + context.reader.norms(fieldName)); + } + + // --------------------------------- Classes --------------------------------- + + /** Delegates the {@link #score(int, int)} and + * {@link #explain(int, Explanation)} methods to + * {@link EasySimilarity#score(EasyStats, float, byte)} and + * {@link EasySimilarity#explain(EasyStats, int, Explanation, byte)}, + * respectively. + */ + private class EasyExactDocScorer extends ExactDocScorer { + private final EasyStats stats; + private final byte[] norms; + + EasyExactDocScorer(EasyStats stats, byte norms[]) { + this.stats = stats; + this.norms = norms; + } + + @Override + public float score(int doc, int freq) { + return EasySimilarity.this.score(stats, freq, norms[doc]); + } + + @Override + public Explanation explain(int doc, Explanation freq) { + return EasySimilarity.this.explain(stats, doc, freq, norms[doc]); + } + } + + /** Delegates the {@link #score(int, int)} and + * {@link #explain(int, Explanation)} methods to + * {@link EasySimilarity#score(EasyStats, float, byte)} and + * {@link EasySimilarity#explain(EasyStats, int, Explanation, byte)}, + * respectively. + */ + private class EasySloppyDocScorer extends SloppyDocScorer { + private final EasyStats stats; + private final byte[] norms; + + EasySloppyDocScorer(EasyStats stats, byte norms[]) { + this.stats = stats; + this.norms = norms; + } + + // todo: optimize + @Override + public float score(int doc, float freq) { + return EasySimilarity.this.score(stats, freq, norms[doc]); + } + @Override + public Explanation explain(int doc, Explanation freq) { + return EasySimilarity.this.explain(stats, doc, freq, norms[doc]); + } + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java new file mode 100644 index 0000000..f84fe95 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java @@ -0,0 +1,137 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Similarity; + +/** + * Stores all statistics commonly used ranking methods. + * @lucene.experimental + */ +public class EasyStats extends Similarity.Stats { + /** The number of documents. */ + protected int numberOfDocuments; // TODO: to long? + /** The total number of tokens in the field. */ + protected long numberOfFieldTokens; + /** The average field length. */ + protected float avgFieldLength; + /** The document frequency. */ + protected int docFreq; + /** The total number of occurrences of this term across all documents. */ + // TODO: same field? + protected long totalTermFreq; + /** The total number of terms across all documents. */ + // TODO: same field? + protected long sumTotalTermFreq; + /** The number of unique terms. */ + // nocommit might be per-segment only + protected long uniqueTermCount; + + // -------------------------- Boost-related stuff -------------------------- + + /** Query's inner boost. */ + protected final float queryBoost; + /** Any outer query's boost. */ + protected float topLevelBoost; + + /** Constructor. Sets the query boost. */ + public EasyStats(float queryBoost) { + this.queryBoost = queryBoost; + } + + // ------------------------- Getter/setter methods ------------------------- + + /** Returns the number of documents. */ + public int getNumberOfDocuments() { + return numberOfDocuments; + } + + /** Sets the number of documents. */ + public void setNumberOfDocuments(int numberOfDocuments) { + this.numberOfDocuments = numberOfDocuments; + } + + /** Returns the total number of tokens in the field. */ + public long getNumberOfFieldTokens() { + return numberOfFieldTokens; + } + + /** Sets the total number of tokens in the field. */ + public void setNumberOfFieldTokens(long numberOfFieldTokens) { + this.numberOfFieldTokens = numberOfFieldTokens; + } + + /** Returns the average field length. */ + public float getAvgFieldLength() { + return avgFieldLength; + } + + /** Sets the average field length. */ + public void setAvgFieldLength(float avgFieldLength) { + this.avgFieldLength = avgFieldLength; + } + + /** Returns the document frequency. */ + public int getDocFreq() { + return docFreq; + } + + /** Sets the document frequency. */ + public void setDocFreq(int docFreq) { + this.docFreq = docFreq; + } + + /** Returns the total number of occurrences of this term across all documents. */ + public long getTotalTermFreq() { + return totalTermFreq; + } + + /** Sets the total number of occurrences of this term across all documents. */ + public void setTotalTermFreq(long totalTermFreq) { + this.totalTermFreq = totalTermFreq; + } + + /** Returns the total number of terms across all documents. */ + public long getSumTotalTermFreq() { + return sumTotalTermFreq; + } + + /** Sets the total number of terms across all documents. */ + public void setSumTotalTermFreq(long sumTotalTermFreq) { + this.sumTotalTermFreq = sumTotalTermFreq; + } + + /** Returns the number of unique terms. */ + public long getUniqueTermCount() { + return uniqueTermCount; + } + + /** Sets the number of unique terms. */ + public void setUniqueTermCount(long uniqueTermCount) { + this.uniqueTermCount = uniqueTermCount; + } + + // -------------------------- Boost-related stuff -------------------------- + + /** The square of the raw normalization value. + * @see #rawNormalizationValue() */ + @Override + public float getValueForNormalization() { + float rawValue = rawNormalizationValue(); + return rawValue * rawValue; + } + + /** Computes the raw normalization value. This basic implementation returns + * the query boost. Subclasses may override this method to include other + * factors (such as idf), or to save the value for inclusion in + * {@link #normalize(float, float)}, etc. + */ + protected float rawNormalizationValue() { + return queryBoost; + } + + /** No normalization is done. {@code topLevelBoost} is saved in the object, + * however. */ + @Override + public void normalize(float queryNorm, float topLevelBoost) { + this.topLevelBoost = topLevelBoost; + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java new file mode 100644 index 0000000..2c750d1 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java @@ -0,0 +1,66 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; + +/** + * Provides a framework for the family of information-based models, as described + * in Stéphane Clinchant and Eric Gaussier. 2010. Information-based + * models for ad hoc IR. In Proceeding of the 33rd international ACM SIGIR + * conference on Research and development in information retrieval (SIGIR '10). + * ACM, New York, NY, USA, 234-241. + *The retrieval function is of the form RSV(q, d) = ∑ + * -xqw log Prob(Xw ≥ + * tdw | λw), where + *
The framework described in the paper has many similarities to the DFR + * framework (see {@link DFRSimilarity}). It is possible that the two + * Similarities will be merged at one point.
+ * @lucene.experimental + */ +public class IBSimilarity extends EasySimilarity { + /** The probabilistic distribution used to model term occurrence. */ + protected final Distribution distribution; + /** The lambda (λw) parameter. */ + protected final Lambda lambda; + /** The term frequency normalization. */ + protected final Normalization normalization; + + public IBSimilarity(ClassThe default normalization methods use the field length of the document + * and the average field length to compute the normalized term frequency. + * This method provides a generic explanation for such methods. + * Subclasses that use other statistics must override this method.
+ */ + public Explanation explain(EasyStats stats, float tf, int len) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(tfn(stats, tf, len)); + result.addDetail(new Explanation(tf, "tf")); + result.addDetail( + new Explanation(stats.getAvgFieldLength(), "avgFieldLength")); + result.addDetail(new Explanation(len, "len")); + return result; + } + + /** Implementation used when there is no normalization. */ + public static final class NoNormalization extends Normalization { + @Override + public final float tfn(EasyStats stats, float tf, int len) { + return tf; + } + + @Override + public final Explanation explain(EasyStats stats, float tf, int len) { + return new Explanation(1, "no normalization"); + } + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java new file mode 100644 index 0000000..242ee78 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java @@ -0,0 +1,11 @@ +package org.apache.lucene.search.similarities; + +/** + * Normalization model that assumes a uniform distribution of the term frequency. + */ +public class NormalizationH1 extends Normalization { + @Override + public float tfn(EasyStats stats, float tf, int len) { + return tf * stats.getAvgFieldLength() / len; + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java new file mode 100644 index 0000000..bcab41a --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java @@ -0,0 +1,14 @@ +package org.apache.lucene.search.similarities; + +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Normalization model in which the term frequency is inversely related to the + * length. + */ +public class NormalizationH2 extends Normalization { + @Override + public float tfn(EasyStats stats, float tf, int len) { + return (float)(tf * log2(1 + stats.getAvgFieldLength() / len)); + } +}