diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java new file mode 100644 index 0000000..1fd31f1 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java @@ -0,0 +1,35 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; + +/** + * This class acts as the base class for the implementations of the first + * normalization of the informative content in the DFR framework. This + * component is also called the after effect and is defined by the + * formula Inf2 = 1 - Prob2, where + * Prob2 measures the information gain. + * + * @see DFRSimilarity + * @lucene.experimental + */ +public abstract class AfterEffect { + /** Returns the aftereffect score. */ + public abstract float score(EasyStats stats, float tfn); + + /** Returns an explanation for the score. */ + public abstract Explanation explain(EasyStats stats, float tfn); + + /** Implementation used when there is no aftereffect. */ + public static final class NoAfterEffect extends AfterEffect { + @Override + public final float score(EasyStats stats, float tfn) { + return 1f; + } + + @Override + public final Explanation explain(EasyStats stats, float tfn) { + // nocommit implement + throw new UnsupportedOperationException(); + } + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java new file mode 100644 index 0000000..5914d27 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java @@ -0,0 +1,22 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; + +/** + * Model of the information gain based on the ration of two Bernoulli processes. + * @lucene.experimental + */ +public class AfterEffectB extends AfterEffect { + @Override + public float score(EasyStats stats, float tfn) { + long F = stats.getTotalTermFreq(); + int n = stats.getDocFreq(); + return (F + 1) / (n * (tfn + 1)); + } + + @Override + public Explanation explain(EasyStats stats, float tfn) { + // nocommit implement + throw new UnsupportedOperationException(); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java new file mode 100644 index 0000000..873e4a7 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java @@ -0,0 +1,20 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; + +/** + * Model of the information gain based on Laplace's law of succession. + * @lucene.experimental + */ +public class AfterEffectL extends AfterEffect { + @Override + public float score(EasyStats stats, float tfn) { + return 1 / (tfn + 1); + } + + @Override + public Explanation explain(EasyStats stats, float tfn) { + // nocommit implement + throw new UnsupportedOperationException(); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java new file mode 100644 index 0000000..0a5a42d --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java @@ -0,0 +1,20 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; + +/** + * This class acts as the base class for the specific basic model + * implementations in the DFR framework. Basic models compute the + * informative content Inf1 = -log2Prob1 + * . + * + * @see DFRSimilarity + * @lucene.experimental + */ +public abstract class BasicModel { + /** Returns the informative content score. */ + public abstract float score(EasyStats stats, float tfn); + + /** Returns an explanation for the score. */ + public abstract Explanation explain(EasyStats stats, float tfn); +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java new file mode 100644 index 0000000..86bc54b --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java @@ -0,0 +1,29 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Limiting form of the Bose-Einstein model. + * @lucene.experimental + */ +public class BasicModelBE extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + long N = stats.getNumberOfDocuments(); + long F = stats.getTotalTermFreq(); + return (float)(-log2((N - 1) * Math.E) + + f(N + F -1, N + F - tfn - 2) - f(F, F - tfn)); + } + + /** The f helper function defined for BE. */ + private final double f(long n, float m) { + return (m + 0.5) * log2((double)n / m) + (n - m) * log2(n); + } + + @Override + public Explanation explain(EasyStats stats, float tfn) { + // nocommit implement + throw new UnsupportedOperationException(); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java new file mode 100644 index 0000000..d5b7918 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java @@ -0,0 +1,28 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Implements the approximation of the binomial model with the divergence + * for DFR. + * @lucene.experimental + */ +public class BasicModelD extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + // nocommit clear this up + long F = stats.getTotalTermFreq(); + double phi = (double)tfn / F; + double nphi = 1 - phi; + double p = 1.0 / stats.getNumberOfDocuments(); + double D = phi * log2(phi / p) + nphi * log2(nphi / (1 - p)); + return (float)(D * F + 0.5 * log2(2 * Math.PI * tfn * nphi)); + } + + @Override + public Explanation explain(EasyStats stats, float tfn) { + // nocommit implement + throw new UnsupportedOperationException(); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java new file mode 100644 index 0000000..aaea35d --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java @@ -0,0 +1,23 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Geometric as limiting form of the Bose-Einstein model. + * @lucene.experimental + */ +public class BasicModelG extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + double lambda = stats.getTotalTermFreq() / stats.getNumberOfDocuments(); + // -log(1 / (lambda + 1)) -> log(lambda + 1) + return (float)(log2(lambda + 1) + tfn * log2((1 + lambda) / lambda)); + } + + @Override + public Explanation explain(EasyStats stats, float tfn) { + // nocommit implement + throw new UnsupportedOperationException(); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java new file mode 100644 index 0000000..9a7ab6d --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java @@ -0,0 +1,24 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * An approximation of the I(ne) model. + * @lucene.experimental + */ +public class BasicModelIF extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + // TODO: Refactor this method to a parent class? See the other two Ix model. + int N = stats.getNumberOfDocuments(); + long F = stats.getTotalTermFreq(); + return tfn * (float)(log2((N + 1) / (F + 0.5))); + } + + @Override + public Explanation explain(EasyStats stats, float tfn) { + // nocommit implement + throw new UnsupportedOperationException(); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java new file mode 100644 index 0000000..ac44b0d --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java @@ -0,0 +1,23 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * The basic tf-idf model of randomness. + * @lucene.experimental + */ +public class BasicModelIn extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + int N = stats.getNumberOfDocuments(); + int n = stats.getDocFreq(); + return tfn * (float)(log2((N + 1) / (n + 0.5))); + } + + @Override + public Explanation explain(EasyStats stats, float tfn) { + // nocommit implement + throw new UnsupportedOperationException(); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java new file mode 100644 index 0000000..e8ba551 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java @@ -0,0 +1,25 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Tf-idf model of randomness, based on a mixture of Poisson and inverse + * document frequency. + * @lucene.experimental + */ +public class BasicModelIne extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + int N = stats.getNumberOfDocuments(); + long F = stats.getTotalTermFreq(); + double ne = N * (1 - Math.pow((N - 1) / N, F)); + return tfn * (float)(log2((N + 1) / (ne + 0.5))); + } + + @Override + public Explanation explain(EasyStats stats, float tfn) { + // nocommit implement + throw new UnsupportedOperationException(); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java new file mode 100644 index 0000000..3e70dc9 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java @@ -0,0 +1,24 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Implements the Poisson approximation for the binomial model for DFR. + * @lucene.experimental + */ +public class BasicModelP extends BasicModel { + @Override + public float score(EasyStats stats, float tfn) { + float lambda = (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); + return (float)(tfn * log2(tfn / lambda) + + (lambda + 1 / 12 / tfn - tfn) * log2(Math.E) + + 0.5 * log2(2 * Math.PI * tfn)); + } + + @Override + public Explanation explain(EasyStats stats, float tfn) { + // nocommit implement + throw new UnsupportedOperationException(); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java new file mode 100644 index 0000000..cbd6bda --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java @@ -0,0 +1,57 @@ +package org.apache.lucene.search.similarities; + +/** + * Implements the divergence from randomness (DFR) framework + * introduced in Gianni Amati and Cornelis Joost Van Rijsbergen. 2002. + * Probabilistic models of information retrieval based on measuring the + * divergence from randomness. ACM Trans. Inf. Syst. 20, 4 (October 2002), + * 357-389. + *

The DFR scoring formula is composed of three separate components: the + * basic model, the aftereffect and an additional + * normalization component, represented by the classes + * {@code BasicModel}, {@code AfterEffect} and {@code Normalization}, + * respectively. The names of these classes were chosen to match the names of + * their counterparts in the Terrier IR engine.

+ *

Note that qtf, the multiplicity of term-occurrence in the query, + * is not handled by this implementation.

+ * + * @see BasicModel + * @see AfterEffect + * @see Normalization + * @lucene.experimental + */ +public class DFRSimilarity extends EasySimilarity { + /** For {@link #log2(double)}. Precomputed for efficiency reasons. */ + private static final double LOG_2 = Math.log(2); + + /** The basic model for information content. */ + protected final BasicModel basicModel; + /** The first normalization of the information content. */ + protected final AfterEffect afterEffect; + /** The term frequency normalization. */ + protected final Normalization normalization; + + public DFRSimilarity(Class basicModelClass, + Class afterEffectClass, + Class normalizationClass) + throws InstantiationException, IllegalAccessException { + basicModel = basicModelClass.newInstance(); + afterEffect = afterEffectClass != null ? afterEffectClass.newInstance() + : new AfterEffect.NoAfterEffect(); + normalization = normalizationClass != null ? normalizationClass.newInstance() + : new Normalization.NoNormalization(); + } + + @Override + protected float score(EasyStats stats, float freq, byte norm) { + // nocommit byte[] norms to byte norm + float tfn = normalization.tfn(stats, freq, decodeNormValue(norm)); + return basicModel.score(stats, tfn) * afterEffect.score(stats, tfn); + } + + // nocommit Put this to a 'util' class + /** Returns the base two logarithm of {@code x}. */ + public static double log2(double x) { + return Math.log(x) / LOG_2; + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java new file mode 100644 index 0000000..38d0c57 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java @@ -0,0 +1,146 @@ +package org.apache.lucene.search.similarities; + +import java.io.IOException; + +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Similarity; +import org.apache.lucene.util.SmallFloat; +import org.apache.lucene.util.TermContext; + +/** + * A subclass of {@code Similarity} that provides a simplified API for its + * descendants. + * @lucene.experimental + */ +public abstract class EasySimilarity extends Similarity { + @Override + public EasyStats computeStats(IndexSearcher searcher, String fieldName, + float queryBoost, TermContext... termContexts) throws IOException { + EasyStats stats = new EasyStats(queryBoost); + IndexReader reader = searcher.getIndexReader(); + // Is this really OK? Will maxDoc change when the deleted-document-aware + // properties are recomputed? Or can it be completely inaccurate? + int numberOfDocuments = reader.maxDoc(); + long numberOfFieldTokens = reader.hasNorms(fieldName) + ? reader.getSumOfNorms(fieldName) : 0; + // nocommit The norm should be smooth enough for this to work! + int avgFieldLengthNorm = + (int)(numberOfFieldTokens / (double)numberOfDocuments); + float avgFieldLength = decodeNormValue((byte)avgFieldLengthNorm); + // Is this OK? + int docFreq = 0; + long totalTermFreq = 0; + for (final TermContext context : termContexts) { + docFreq += context.docFreq(); + totalTermFreq += context.totalTermFreq(); + } + // Another source. What about the root context? Are the others OK? + long sumTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), + fieldName).getSumTotalTermFreq(); + + stats.setNumberOfDocuments(numberOfDocuments); + stats.setNumberOfFieldTokens(numberOfFieldTokens); + stats.setAvgFieldLength(avgFieldLength); + stats.setDocFreq(docFreq); + stats.setTotalTermFreq(totalTermFreq); + stats.setSumTotalTermFreq(sumTotalTermFreq); + // nocommit uniqueTermCount? + return stats; + } + + /** Uses the {@code 1 / (distance + 1)} formula. */ + @Override + public float sloppyFreq(int distance) { + // nocommit to separate class + return 1.0f / (distance + 1); + } + + /** Encodes the document length. */ + @Override + public byte computeNorm(FieldInvertState state) { + return encodeNormValue(state.getLength()); + } + + /** Decodes a normalization factor stored in an index. + * @see #encodeNormValue(float) + */ + // nocommit to protected? + // nocommit is int OK? + public int decodeNormValue(byte norm) { + // SmallFloat seems OK, because tf is smoothed anyway. + return (int)SmallFloat.byte315ToFloat(norm); + } + + /** Encodes the length to a byte via SmallInt. */ + // nocommit to protected? + public byte encodeNormValue(int length) { + // SmallFloat seems OK, because tf is smoothed anyway. + return SmallFloat.floatToByte315(length); + } + + /** + * Scores the document {@code doc}. + *

Subclasses must apply their scoring formula in this class.

+ * @param stats the corpus level statistics. + * @param doc the document id. + * @param freq the term frequency. + * @param norm the current document's field norm. + * @return the score. + */ + protected abstract float score(EasyStats stats, float freq, byte norm); + + @Override + public ExactDocScorer exactDocScorer(Stats stats, String fieldName, + AtomicReaderContext context) throws IOException { + return new EasyExactDocScorer((EasyStats) stats, + context.reader.norms(fieldName)); + } + + @Override + public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, + AtomicReaderContext context) throws IOException { + return new EasySloppyDocScorer((EasyStats) stats, + context.reader.norms(fieldName)); + } + + // --------------------------------- Classes --------------------------------- + + /** Calls the {@code score()} method. */ + // nocommit Javadoc + private class EasyExactDocScorer extends ExactDocScorer { + private final EasyStats stats; + private final byte[] norms; + + EasyExactDocScorer(EasyStats stats, byte norms[]) { + this.stats = stats; + this.norms = norms; + } + + @Override + public float score(int doc, int freq) { + return EasySimilarity.this.score(stats, freq, norms[doc]); + } + } + + /** Calls the {@code score()} method. */ + // nocommit Javadoc + private class EasySloppyDocScorer extends SloppyDocScorer { + private final EasyStats stats; + private final byte[] norms; + + EasySloppyDocScorer(EasyStats stats, byte norms[]) { + this.stats = stats; + this.norms = norms; + } + + // todo: optimize + @Override + public float score(int doc, float freq) { + return EasySimilarity.this.score(stats, freq, norms[doc]); + } + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java new file mode 100644 index 0000000..f84fe95 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java @@ -0,0 +1,137 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Similarity; + +/** + * Stores all statistics commonly used ranking methods. + * @lucene.experimental + */ +public class EasyStats extends Similarity.Stats { + /** The number of documents. */ + protected int numberOfDocuments; // TODO: to long? + /** The total number of tokens in the field. */ + protected long numberOfFieldTokens; + /** The average field length. */ + protected float avgFieldLength; + /** The document frequency. */ + protected int docFreq; + /** The total number of occurrences of this term across all documents. */ + // TODO: same field? + protected long totalTermFreq; + /** The total number of terms across all documents. */ + // TODO: same field? + protected long sumTotalTermFreq; + /** The number of unique terms. */ + // nocommit might be per-segment only + protected long uniqueTermCount; + + // -------------------------- Boost-related stuff -------------------------- + + /** Query's inner boost. */ + protected final float queryBoost; + /** Any outer query's boost. */ + protected float topLevelBoost; + + /** Constructor. Sets the query boost. */ + public EasyStats(float queryBoost) { + this.queryBoost = queryBoost; + } + + // ------------------------- Getter/setter methods ------------------------- + + /** Returns the number of documents. */ + public int getNumberOfDocuments() { + return numberOfDocuments; + } + + /** Sets the number of documents. */ + public void setNumberOfDocuments(int numberOfDocuments) { + this.numberOfDocuments = numberOfDocuments; + } + + /** Returns the total number of tokens in the field. */ + public long getNumberOfFieldTokens() { + return numberOfFieldTokens; + } + + /** Sets the total number of tokens in the field. */ + public void setNumberOfFieldTokens(long numberOfFieldTokens) { + this.numberOfFieldTokens = numberOfFieldTokens; + } + + /** Returns the average field length. */ + public float getAvgFieldLength() { + return avgFieldLength; + } + + /** Sets the average field length. */ + public void setAvgFieldLength(float avgFieldLength) { + this.avgFieldLength = avgFieldLength; + } + + /** Returns the document frequency. */ + public int getDocFreq() { + return docFreq; + } + + /** Sets the document frequency. */ + public void setDocFreq(int docFreq) { + this.docFreq = docFreq; + } + + /** Returns the total number of occurrences of this term across all documents. */ + public long getTotalTermFreq() { + return totalTermFreq; + } + + /** Sets the total number of occurrences of this term across all documents. */ + public void setTotalTermFreq(long totalTermFreq) { + this.totalTermFreq = totalTermFreq; + } + + /** Returns the total number of terms across all documents. */ + public long getSumTotalTermFreq() { + return sumTotalTermFreq; + } + + /** Sets the total number of terms across all documents. */ + public void setSumTotalTermFreq(long sumTotalTermFreq) { + this.sumTotalTermFreq = sumTotalTermFreq; + } + + /** Returns the number of unique terms. */ + public long getUniqueTermCount() { + return uniqueTermCount; + } + + /** Sets the number of unique terms. */ + public void setUniqueTermCount(long uniqueTermCount) { + this.uniqueTermCount = uniqueTermCount; + } + + // -------------------------- Boost-related stuff -------------------------- + + /** The square of the raw normalization value. + * @see #rawNormalizationValue() */ + @Override + public float getValueForNormalization() { + float rawValue = rawNormalizationValue(); + return rawValue * rawValue; + } + + /** Computes the raw normalization value. This basic implementation returns + * the query boost. Subclasses may override this method to include other + * factors (such as idf), or to save the value for inclusion in + * {@link #normalize(float, float)}, etc. + */ + protected float rawNormalizationValue() { + return queryBoost; + } + + /** No normalization is done. {@code topLevelBoost} is saved in the object, + * however. */ + @Override + public void normalize(float queryNorm, float topLevelBoost) { + this.topLevelBoost = topLevelBoost; + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/Normalization.java lucene/src/test/org/apache/lucene/search/similarities/Normalization.java new file mode 100644 index 0000000..1b42d80 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/Normalization.java @@ -0,0 +1,33 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; + +/** + * This class acts as the base class for the implementations of the term + * frequency normalization methods in the DFR framework. + * + * @see DFRSimilarity + * @lucene.experimental + */ +public abstract class Normalization { + /** Returns the normalized term frequency. + * @param len the field length. */ + public abstract float tfn(EasyStats stats, float tf, int len); + + /** Returns an explanation for the frequency. */ + public abstract Explanation explain(); + + /** Implementation used when there is no normalization. */ + public static final class NoNormalization extends Normalization { + @Override + public final float tfn(EasyStats stats, float tf, int len) { + return tf; + } + + @Override + public final Explanation explain() { + // nocommit implement + throw new UnsupportedOperationException(); + } + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java new file mode 100644 index 0000000..1eba8d1 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java @@ -0,0 +1,19 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; + +/** + * Normalization model that assumes a uniform distribution of the term frequency. + */ +public class NormalizationH1 extends Normalization { + @Override + public float tfn(EasyStats stats, float tf, int len) { + return tf * stats.getAvgFieldLength() / len; + } + + @Override + public Explanation explain() { + // nocommit implement + throw new UnsupportedOperationException(); + } +} diff --git lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java new file mode 100644 index 0000000..ecc3be0 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java @@ -0,0 +1,21 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.Explanation; +import static org.apache.lucene.search.similarities.DFRSimilarity.log2; + +/** + * Normalization model in which the term frequency is inversely related to the + * length. + */ +public class NormalizationH2 extends Normalization { + @Override + public float tfn(EasyStats stats, float tf, int len) { + return (float)(tf * log2(1 + stats.getAvgFieldLength() / len)); + } + + @Override + public Explanation explain() { + // nocommit implement + throw new UnsupportedOperationException(); + } +}