diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java index 8c2662f..faee912 100644 --- lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java +++ lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java @@ -47,5 +47,17 @@ public abstract class AfterEffect { public final Explanation explain(EasyStats stats, float tfn) { return new Explanation(1, "no aftereffect"); } + + @Override + public String toString() { + return ""; + } } + + /** + * Subclasses must override this method to return the code of the + * after effect formula. Refer to the original paper for the list. + */ + @Override + public abstract String toString(); } diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java index 0edc2e2..90079b2 100644 --- lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java +++ lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java @@ -41,4 +41,9 @@ public class AfterEffectB extends AfterEffect { result.addDetail(new Explanation(stats.getDocFreq(), "docFreq")); return result; } + + @Override + public String toString() { + return "B"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java index b20d784..3a1f223 100644 --- lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java +++ lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java @@ -37,4 +37,9 @@ public class AfterEffectL extends AfterEffect { result.addDetail(new Explanation(tfn, "tfn")); return result; } + + @Override + public String toString() { + return "L"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java index d07a6fe..e811869 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java @@ -50,4 +50,11 @@ public abstract class BasicModel { new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); return result; } + + /** + * Subclasses must override this method to return the code of the + * basic model formula. Refer to the original paper for the list. + */ + @Override + public abstract String toString(); } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java index 366fce3..ea55a46 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java @@ -36,4 +36,9 @@ public class BasicModelBE extends BasicModel { private final double f(long n, float m) { return (m + 0.5) * log2((double)n / m) + (n - m) * log2(n); } + + @Override + public String toString() { + return "Be"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java index b477bc9..cb8dc7d 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java @@ -34,4 +34,9 @@ public class BasicModelD extends BasicModel { double D = phi * log2(phi / p) + nphi * log2(nphi / (1 - p)); return (float)(D * F + 0.5 * log2(2 * Math.PI * tfn * nphi)); } + + @Override + public String toString() { + return "D"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java index 864156e..129d76d 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java @@ -30,4 +30,9 @@ public class BasicModelG extends BasicModel { // -log(1 / (lambda + 1)) -> log(lambda + 1) return (float)(log2(lambda + 1) + tfn * log2((1 + lambda) / lambda)); } + + @Override + public String toString() { + return "G"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java index f096b0b..5cc1db5 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java @@ -31,4 +31,9 @@ public class BasicModelIF extends BasicModel { long F = stats.getTotalTermFreq(); return tfn * (float)(log2((N + 1) / (F + 0.5))); } + + @Override + public String toString() { + return "I(F)"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java index 4556850..3d7bf96 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java @@ -44,4 +44,9 @@ public class BasicModelIn extends BasicModel { new Explanation(stats.getDocFreq(), "docFreq")); return result; } + + @Override + public String toString() { + return "I(n)"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java index 042ff99..7989cb9 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java @@ -32,4 +32,9 @@ public class BasicModelIne extends BasicModel { double ne = N * (1 - Math.pow((N - 1) / N, F)); return tfn * (float)(log2((N + 1) / (ne + 0.5))); } + + @Override + public String toString() { + return "I(ne)"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java index 308f5d6..654133a 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java @@ -31,4 +31,9 @@ public class BasicModelP extends BasicModel { + (lambda + 1 / 12 / tfn - tfn) * log2(Math.E) + 0.5 * log2(2 * Math.PI * tfn)); } + + @Override + public String toString() { + return "P"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java index c52a85c..78d1a2c 100644 --- lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java @@ -98,4 +98,10 @@ public class DFRSimilarity extends EasySimilarity { expl.addDetail(basicModel.explain(stats, tfn)); expl.addDetail(afterEffect.explain(stats, tfn)); } + + @Override + public String toString() { + return "DFR " + basicModel.toString() + afterEffect.toString() + + normalization.toString(); + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/Distribution.java lucene/src/test/org/apache/lucene/search/similarities/Distribution.java index 11f2bcb..aa2702a 100644 --- lucene/src/test/org/apache/lucene/search/similarities/Distribution.java +++ lucene/src/test/org/apache/lucene/search/similarities/Distribution.java @@ -35,4 +35,11 @@ public abstract class Distribution { return new Explanation( score(stats, tfn, lambda), getClass().getSimpleName()); } + + /** + * Subclasses must override this method to return the name of the + * distribution. + */ + @Override + public abstract String toString(); } diff --git lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java index 782dda9..f7a4a4a 100644 --- lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java +++ lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java @@ -24,9 +24,14 @@ package org.apache.lucene.search.similarities; * preference to a specific base.

* @lucene.experimental */ -public abstract class DistributionLL extends Distribution { +public class DistributionLL extends Distribution { @Override public final float score(EasyStats stats, float tfn, float lambda) { return (float)-Math.log(lambda / (tfn + lambda)); } + + @Override + public String toString() { + return "LL"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java index b1e26aa..fa9ed02 100644 --- lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java +++ lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java @@ -25,10 +25,15 @@ package org.apache.lucene.search.similarities; * preference to a specific base.

* @lucene.experimental */ -public abstract class DistributionSPL extends Distribution { +public class DistributionSPL extends Distribution { @Override public final float score(EasyStats stats, float tfn, float lambda) { return (float)-Math.log( (Math.pow(lambda, (tfn / (tfn + 1))) - lambda) / (1 - lambda)); } + + @Override + public String toString() { + return "SPL"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java index c16505c..7a14180 100644 --- lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java @@ -57,10 +57,9 @@ public abstract class EasySimilarity extends Similarity { String fieldName, TermContext... termContexts) throws IOException { IndexReader reader = searcher.getIndexReader(); int numberOfDocuments = reader.maxDoc(); - long sumTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), + long numberOfFieldTokens = MultiFields.getTerms(searcher.getIndexReader(), fieldName).getSumTotalTermFreq(); - long numberOfFieldTokens = sumTotalTermFreq; // nocommit: these are the same stat? - float avgFieldLength = (float)sumTotalTermFreq / numberOfDocuments; + float avgFieldLength = (float)numberOfFieldTokens / numberOfDocuments; // nocommit This is for phrases, and it doesn't really work... have to // find a method that makes sense @@ -76,33 +75,9 @@ public abstract class EasySimilarity extends Similarity { stats.setAvgFieldLength(avgFieldLength); stats.setDocFreq(docFreq); stats.setTotalTermFreq(totalTermFreq); - stats.setSumTotalTermFreq(sumTotalTermFreq); // nocommit uniqueTermCount? (LUCENE-3290) } - /** Encodes the document length. */ - @Override - public byte computeNorm(FieldInvertState state) { - return encodeNormValue(state.getLength()); - } - - /** Decodes a normalization factor stored in an index. - * @see #encodeNormValue(float) - */ - // nocommit to protected? - // nocommit is int OK? - public int decodeNormValue(byte norm) { - // SmallFloat seems OK, because tf is smoothed anyway. - return (int)SmallFloat.byte315ToFloat(norm); - } - - /** Encodes the length to a byte via SmallInt. */ - // nocommit to protected? - public byte encodeNormValue(int length) { - // SmallFloat seems OK, because tf is smoothed anyway. - return SmallFloat.floatToByte315(length); - } - /** * Scores the document {@code doc}. *

Subclasses must apply their scoring formula in this class.

@@ -171,6 +146,52 @@ public abstract class EasySimilarity extends Similarity { context.reader.norms(fieldName)); } + /** + * Subclasses must override this method to return the name of the Similarity + * and preferably the values of parameters (if any) as well. + */ + @Override + public abstract String toString(); // nocommit: to Similarity? + + // ------------------------------ Norm handling ------------------------------ + + /** Cache of decoded bytes. */ + private static final float[] NORM_TABLE = new float[256]; + + static { + for (int i = 0; i < 256; i++) + NORM_TABLE[i] = SmallFloat.byte315ToFloat((byte)i); + } + + /** Encodes the document length in the same way as {@link TFIDFSimilarity}. */ + @Override + public byte computeNorm(FieldInvertState state) { + final int numTerms; + // nocommit: to include discountOverlaps? +// if (discountOverlaps) +// numTerms = state.getLength() - state.getNumOverlap(); +// else + numTerms = state.getLength(); +// return encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms)))); + return encodeNormValue(numTerms); + } + + /** Decodes a normalization factor (document length) stored in an index. + * @see #encodeNormValue(float) + */ + // nocommit to protected? + // nocommit is int OK? + public int decodeNormValue(byte norm) { + float floatNorm = NORM_TABLE[norm & 0xFF]; // & 0xFF maps negative bytes to positive above 127 + return (int)(1.0 / (floatNorm * floatNorm)); + } + + /** Encodes the length to a byte via SmallFloat. */ + // nocommit to protected? + public byte encodeNormValue(int length) { + return SmallFloat.floatToByte315((float)(1.0 / Math.sqrt(length))); + } + // ----------------------------- Static methods ------------------------------ /** Returns the base two logarithm of {@code x}. */ diff --git lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java index 49526b0..8b0e8aa 100644 --- lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java +++ lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java @@ -17,6 +17,8 @@ package org.apache.lucene.search.similarities; * limitations under the License. */ +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Terms; import org.apache.lucene.search.Similarity; /** @@ -33,11 +35,7 @@ public class EasyStats extends Similarity.Stats { /** The document frequency. */ protected int docFreq; /** The total number of occurrences of this term across all documents. */ - // TODO: same field? protected long totalTermFreq; - /** The total number of terms across all documents. */ - // TODO: same field? - protected long sumTotalTermFreq; /** The number of unique terms. */ // nocommit might be per-segment only protected long uniqueTermCount; @@ -70,12 +68,18 @@ public class EasyStats extends Similarity.Stats { this.numberOfDocuments = numberOfDocuments; } - /** Returns the total number of tokens in the field. */ + /** + * Returns the total number of tokens in the field. + * @see Terms#getSumTotalTermFreq() + */ public long getNumberOfFieldTokens() { return numberOfFieldTokens; } - /** Sets the total number of tokens in the field. */ + /** + * Sets the total number of tokens in the field. + * @see Terms#getSumTotalTermFreq() + */ public void setNumberOfFieldTokens(long numberOfFieldTokens) { this.numberOfFieldTokens = numberOfFieldTokens; } @@ -110,16 +114,6 @@ public class EasyStats extends Similarity.Stats { this.totalTermFreq = totalTermFreq; } - /** Returns the total number of terms across all documents. */ - public long getSumTotalTermFreq() { - return sumTotalTermFreq; - } - - /** Sets the total number of terms across all documents. */ - public void setSumTotalTermFreq(long sumTotalTermFreq) { - this.sumTotalTermFreq = sumTotalTermFreq; - } - /** Returns the number of unique terms. */ public long getUniqueTermCount() { return uniqueTermCount; diff --git lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java index 743b1be..230c81a 100644 --- lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java @@ -49,15 +49,17 @@ public class IBSimilarity extends EasySimilarity { /** The term frequency normalization. */ protected final Normalization normalization; - public IBSimilarity(Class distributionClass, - Class lambdaClass, - Class normalizationClass) - throws InstantiationException, IllegalAccessException { - distribution = distributionClass.newInstance(); - lambda = lambdaClass.newInstance(); - normalization = (normalizationClass != null) - ? normalizationClass.newInstance() - : new Normalization.NoNormalization(); + public IBSimilarity(Distribution distribution, + Lambda lambda, + Normalization normalization) { + this.distribution = distribution; + this.lambda = lambda; + this.normalization = normalization; + } + + /** Creates an instance with no normalization. */ + public IBSimilarity(Distribution distribution, Lambda lambda) { + this(distribution, lambda, new Normalization.NoNormalization()); } @Override @@ -83,4 +85,16 @@ public class IBSimilarity extends EasySimilarity { expl.addDetail(distribution.explain( stats, normExpl.getValue(), lambdaExpl.getValue())); } + + /** + * The name of IB methods follow the pattern + * {@code IB }. The name of the + * distribution is the same as in the original paper; for the names of lambda + * parameters, refer to the javadoc of the {@link Lambda} classes. + */ + @Override + public String toString() { + return "IB " + distribution.toString() + "-" + lambda.toString() + + normalization.toString(); + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java index a57343e..bd5fe37 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java @@ -83,4 +83,9 @@ public class LMDirichletSimilarity extends LMSimilarity { public float getMu() { return mu; } + + @Override + public String getName() { + return String.format("Dirichlet(%f)", getMu()); + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java index d99db37..45db940 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java @@ -66,4 +66,9 @@ public class LMJelinekMercerSimilarity extends LMSimilarity { public float getLambda() { return lambda; } + + @Override + public String getName() { + return String.format("Jelinek-Mercer(%f)", getLambda()); + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java index 30ebbca..a9b940d 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java @@ -71,6 +71,30 @@ public abstract class LMSimilarity extends EasySimilarity { expl.addDetail(new Explanation(collectionModel.computeProbability(stats), "collection probability")); } + + /** + * Returns the name of the LM method. The values of the parameters should be + * included as well. + *

Used in {@link #toString()}

. + */ + public abstract String getName(); + + /** + * Returns the name of the LM method. If a custom collection model strategy is + * used, its name is included as well. + * @see #getName() + * @see CollectionModel#getName() + * @see DefaultCollectionModel + */ + @Override + public String toString() { + String coll = collectionModel.getName(); + if (coll != null) { + return String.format("LM %s - %s", getName(), coll); + } else { + return String.format("LM %s", getName()); + } + } /** Stores the collection distribution of the current term. */ public static class LMStats extends EasyStats { @@ -105,6 +129,9 @@ public abstract class LMSimilarity extends EasySimilarity { * strategy for the current term. */ public float computeProbability(EasyStats stats); + + /** The name of the collection model strategy. */ + public String getName(); } /** @@ -114,7 +141,12 @@ public abstract class LMSimilarity extends EasySimilarity { public static class DefaultCollectionModel implements CollectionModel { @Override public float computeProbability(EasyStats stats) { - return (float)stats.getTotalTermFreq() / stats.getSumTotalTermFreq(); + return (float)stats.getTotalTermFreq() / stats.getNumberOfFieldTokens(); + } + + @Override + public String getName() { + return null; } } } diff --git lucene/src/test/org/apache/lucene/search/similarities/Lambda.java lucene/src/test/org/apache/lucene/search/similarities/Lambda.java index 4b658f7..5df1b4a 100644 --- lucene/src/test/org/apache/lucene/search/similarities/Lambda.java +++ lucene/src/test/org/apache/lucene/search/similarities/Lambda.java @@ -30,4 +30,13 @@ public abstract class Lambda { public abstract float lambda(EasyStats stats); /** Explains the lambda parameter. */ public abstract Explanation explain(EasyStats stats); + + /** + * Subclasses must override this method to return the code of the lambda + * formula. Since the original paper is not very clear on this matter, and + * also uses the DFR naming scheme incorrectly, the codes here were chosen + * arbitrarily. + */ + @Override + public abstract String toString(); } diff --git lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java index f8d7b30..5001b0e 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java +++ lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java @@ -40,4 +40,9 @@ public class LambdaDF extends Lambda { new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); return result; } + + @Override + public String toString() { + return "D"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java index 0fd3b99..06cbb05 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java +++ lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java @@ -40,4 +40,9 @@ public class LambdaTTF extends Lambda { new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); return result; } + + @Override + public String toString() { + return "L"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/Normalization.java lucene/src/test/org/apache/lucene/search/similarities/Normalization.java index a9a9d4f..68ae759 100644 --- lucene/src/test/org/apache/lucene/search/similarities/Normalization.java +++ lucene/src/test/org/apache/lucene/search/similarities/Normalization.java @@ -59,5 +59,17 @@ public abstract class Normalization { public final Explanation explain(EasyStats stats, float tf, int len) { return new Explanation(1, "no normalization"); } + + @Override + public String toString() { + return ""; + } } + + /** + * Subclasses must override this method to return the code of the + * normalization formula. Refer to the original paper for the list. + */ + @Override + public abstract String toString(); } diff --git lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java index eab4ef6..4201c59 100644 --- lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java +++ lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java @@ -25,4 +25,9 @@ public class NormalizationH1 extends Normalization { public final float tfn(EasyStats stats, float tf, int len) { return tf * stats.getAvgFieldLength() / len; } + + @Override + public String toString() { + return "1"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java index df3d4f4..5d24eca 100644 --- lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java +++ lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java @@ -28,4 +28,9 @@ public class NormalizationH2 extends Normalization { public final float tfn(EasyStats stats, float tf, int len) { return (float)(tf * log2(1 + stats.getAvgFieldLength() / len)); } + + @Override + public String toString() { + return "2"; + } } \ No newline at end of file diff --git lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java new file mode 100644 index 0000000..da0584f --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java @@ -0,0 +1,173 @@ +package org.apache.lucene.search.similarities; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SimilarityProvider; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; + +/** + * Tests the {@link EasySimilarity}-based Similarities. + */ +public class TestEasySimilarity extends LuceneTestCase { + private static String FIELD_BODY = "body"; + private static String FIELD_ID = "id"; + /** The DFR basic models to test. */ + private static BasicModel[] BASIC_MODELS; + /** The DFR aftereffects to test. */ + private static AfterEffect[] AFTER_EFFECTS; + /** The DFR normalizations to test. */ + private static Normalization[] NORMALIZATIONS; + /** The distributions for IB. */ + private static Distribution[] DISTRIBUTIONS; + /** Lambdas for IB. */ + private static Lambda[] LAMBDAS; + + static { + BASIC_MODELS = new BasicModel[] { + new BasicModelBE(), new BasicModelD(), new BasicModelG(), + new BasicModelIF(), new BasicModelIn(), new BasicModelIne(), + new BasicModelP() + }; + AFTER_EFFECTS = new AfterEffect[] { + new AfterEffectB(), new AfterEffectL(), new AfterEffect.NoAfterEffect() + }; + NORMALIZATIONS = new Normalization[] { + new NormalizationH1(), new NormalizationH2(), + new Normalization.NoNormalization() + }; + DISTRIBUTIONS = new Distribution[] { + new DistributionLL(), new DistributionSPL() + }; + LAMBDAS = new Lambda[] { + new LambdaDF(), new LambdaTTF() + }; + } + + private IndexSearcher searcher; + private Directory dir; + private IndexReader reader; + /** The list of similarities to test. */ + private List sims; + + String[] docs = new String[] { + "Tiger, tiger burning bright In the forest of the night What immortal hand or eye Could frame thy fearful symmetry ?", + "In what distant depths or skies Burnt the fire of thine eyes ? On what wings dare he aspire ? What the hands the seize the fire ?", + "And what shoulder and what art Could twist the sinews of thy heart ? And when thy heart began to beat What dread hand ? And what dread feet ?", + "What the hammer? What the chain ? In what furnace was thy brain ? What the anvil ? And what dread grasp Dare its deadly terrors clasp ?", + "And when the stars threw down their spears And water'd heaven with their tear Did he smile his work to see ? Did he, who made the lamb, made thee ?", + "Tiger, tiger burning bright In the forest of the night What immortal hand or eye Dare frame thy fearful symmetry ?", + "Cruelty has a human heart And jealousy a human face Terror the human form divine And Secrecy the human dress .", + "The human dress is forg'd iron The human form a fiery forge The human face a furnace seal'd The human heart its fiery gorge ." + }; + + @Override + public void setUp() throws Exception { + super.setUp(); + + dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random, dir); + + for (int i = 0; i < docs.length; i++) { + Document d = new Document(); + d.add(newField(FIELD_ID, Integer.toString(i), Field.Store.YES, Field.Index.NO)); + d.add(newField(FIELD_BODY, docs[i], Field.Index.ANALYZED)); + writer.addDocument(d); + } + + reader = writer.getReader(); + searcher = newSearcher(reader); + writer.close(); + + sims = new ArrayList(); + for (BasicModel basicModel : BASIC_MODELS) { + for (AfterEffect afterEffect : AFTER_EFFECTS) { + for (Normalization normalization : NORMALIZATIONS) { + sims.add(new DFRSimilarity(basicModel, afterEffect, normalization)); + } + } + } + for (Distribution distribution : DISTRIBUTIONS) { + for (Lambda lambda : LAMBDAS) { + for (Normalization normalization : NORMALIZATIONS) { + sims.add(new IBSimilarity(distribution, lambda, normalization)); + } + } + } + sims.add(new LMDirichletSimilarity()); + sims.add(new LMJelinekMercerSimilarity(0.1f)); + sims.add(new LMJelinekMercerSimilarity(0.7f)); + } + + /** + * Tests whether all similarities return three documents for the query word + * "heart". + */ + public void testHeartList() throws IOException { + Query q = new TermQuery(new Term(FIELD_BODY, "heart")); + + for (EasySimilarity sim : sims) { + searcher.setSimilarityProvider(new EasySimilarityProvider(sim)); + TopDocs topDocs = searcher.search(q, 1000); + assertEquals("Failed: " + sim.toString(), 3, topDocs.totalHits); + } + } + + /** Test whether all similarities return document 3 before documents 7 and 8. */ + public void testHeartRanking() throws IOException { + Query q = new TermQuery(new Term(FIELD_BODY, "heart")); + + for (EasySimilarity sim : sims) { + searcher.setSimilarityProvider(new EasySimilarityProvider(sim)); + TopDocs topDocs = searcher.search(q, 1000); + assertEquals("Failed: " + sim.toString(), 2, topDocs.scoreDocs[0].doc); + } + } + + @Override + public void tearDown() throws Exception { + searcher.close(); + reader.close(); + dir.close(); + super.tearDown(); + } + + /** + * A simple Similarity provider that returns in {@code get(String field)} the + * object passed to its constructor. + */ + // nocommit a real EasySimilarityProvider with coord and queryNorm implemented? + public static class EasySimilarityProvider implements SimilarityProvider { + private EasySimilarity sim; + + public EasySimilarityProvider(EasySimilarity sim) { + this.sim = sim; + } + + @Override + public float coord(int overlap, int maxOverlap) { + return 1f; + } + + @Override + public float queryNorm(float sumOfSquaredWeights) { + return 1f; + } + + @Override + public EasySimilarity get(String field) { + return sim; + } + } +}