+ * The formula as defined the paper assigns a negative score to documents that + * contain the term, but with fewer occurrences than predicted by the collection + * language model. The Lucene implementation returns {@code 0} for such + * documents. + *
* * @lucene.experimental */ @@ -55,10 +61,10 @@ public class LMDirichletSimilarity extends LMSimilarity { @Override protected float score(EasyStats stats, float freq, int docLen) { - return stats.getTotalBoost() * - (float)(Math.log(1 + freq / - (mu * ((LMStats)stats).getCollectionProbability())) + + float score = stats.getTotalBoost() * (float)(Math.log(1 + freq / + (mu * ((LMStats)stats).getCollectionProbability())) + Math.log(mu / (docLen + mu))); + return score > 0.0f ? score : 0.0f; } @Override diff --git lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java index 971a4d1..ab8a653 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java @@ -136,12 +136,12 @@ public abstract class LMSimilarity extends EasySimilarity { /** * Models {@code p(w|C)} as the number of occurrences of the term in the - * collection, divided by the total number of tokens. + * collection, divided by the total number of tokens {@code + 1}. */ public static class DefaultCollectionModel implements CollectionModel { @Override public float computeProbability(EasyStats stats) { - return (float)stats.getTotalTermFreq() / stats.getNumberOfFieldTokens(); + return (float)stats.getTotalTermFreq() / (stats.getNumberOfFieldTokens() +1); } @Override diff --git lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java index ad7799b..97a0f7c 100644 --- lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java @@ -27,6 +27,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.OrdTermState; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.SimilarityProvider; @@ -37,11 +38,13 @@ import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.TermContext; /** - * Tests the {@link EasySimilarity}-based Similarities. Contains unit tests and - * integration tests as well. This class maintains a list of + * Tests the {@link EasySimilarity}-based Similarities. Contains unit tests and + * integration tests for all Similarities and correctness tests for a select + * few. + *This class maintains a list of * {@code EasySimilarity} subclasses. Each test case performs its test on all * items in the list. If a test case fails, the name of the Similarity that - * caused the failure is returned as part of the assertion error message. + * caused the failure is returned as part of the assertion error message.
*Unit testing is performed by constructing statistics manually and calling * the {@link EasySimilarity#score(EasyStats, float, int)} method of the * Similarities. The statistics represent corner cases of corpus distributions. @@ -53,10 +56,18 @@ import org.apache.lucene.util.TermContext; *
Note: the list of Similarities is maintained by hand. If a new Similarity * is added to the {@code org.apache.lucene.search.similarities} package, the * list should be updated accordingly.
+ *+ * In the correctness tests, the score is verified against the result of manual + * computation. Since it would be impossible to test all Similarities + * (e.g. all possible DFR combinations, all parameter values for LM), only + * the best performing setups in the original papers are verified. + *
*/ public class TestEasySimilarity extends LuceneTestCase { private static String FIELD_BODY = "body"; private static String FIELD_ID = "id"; + /** The tolerance range for float equality. */ + private static float FLOAT_EPSILON = 1e-5f; /** The DFR basic models to test. */ private static BasicModel[] BASIC_MODELS; /** The DFR aftereffects to test. */ @@ -183,23 +194,14 @@ public class TestEasySimilarity extends LuceneTestCase { for (EasySimilarity sim : sims) { EasyStats realStats = sim.computeStats(new SpoofIndexSearcher(stats), "spoof", stats.getTotalBoost(), tc); -// System.out.printf("Before: %d %d %f %d %d%n", -// realStats.getNumberOfDocuments(), realStats.getNumberOfFieldTokens(), -// realStats.getAvgFieldLength(), realStats.getDocFreq(), -// realStats.getTotalTermFreq()); -// realStats.setNumberOfDocuments(stats.getNumberOfDocuments()); -// realStats.setNumberOfFieldTokens(stats.getNumberOfFieldTokens()); -// realStats.setAvgFieldLength(stats.getAvgFieldLength()); -// realStats.setDocFreq(stats.getDocFreq()); -// realStats.setTotalTermFreq(stats.getTotalTermFreq()); -// System.out.printf("After: %d %d %f %d %d%n", -// realStats.getNumberOfDocuments(), realStats.getNumberOfFieldTokens(), -// realStats.getAvgFieldLength(), realStats.getDocFreq(), -// realStats.getTotalTermFreq()); float score = sim.score(realStats, freq, docLen); + float explScore = sim.explain( + realStats, 1, new Explanation(freq, "freq"), docLen).getValue(); assertFalse("Score infinite: " + sim.toString(), Float.isInfinite(score)); assertFalse("Score NaN: " + sim.toString(), Float.isNaN(score)); assertTrue("Score negative: " + sim.toString(), score >= 0); + assertEquals("score() and explain() return different values: " + + sim.toString(), score, explScore, FLOAT_EPSILON); } } @@ -217,7 +219,7 @@ public class TestEasySimilarity extends LuceneTestCase { stats.setNumberOfFieldTokens(stats.getNumberOfDocuments()); stats.setTotalTermFreq(stats.getDocFreq()); stats.setAvgFieldLength( - stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); + (float)stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); unitTestCore(stats, FREQ, DOC_LEN); } @@ -230,10 +232,10 @@ public class TestEasySimilarity extends LuceneTestCase { stats.setNumberOfFieldTokens(stats.getNumberOfDocuments() * 2 / 3); stats.setTotalTermFreq(stats.getDocFreq()); stats.setAvgFieldLength( - stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); + (float)stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); unitTestCore(stats, FREQ, DOC_LEN); } - + /** * Tests correct behavior when * {@code NumberOfDocuments = 1}. @@ -383,6 +385,121 @@ public class TestEasySimilarity extends LuceneTestCase { unitTestCore(stats, FREQ, (int)stats.getAvgFieldLength()); } + // ---------------------------- Correctness tests ---------------------------- + + /** Correctness test for the Dirichlet LM model. */ + public void testLMDirichlet() throws IOException { + float p = + (FREQ + 2000.0f * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f)) / + (DOC_LEN + 2000.0f); + float a = 2000.0f / (DOC_LEN + 2000.0f); + float gold = (float)( + Math.log(p / (a * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f))) + + Math.log(a)); + correctnessTestCore(new LMDirichletSimilarity(), gold); + } + + /** Correctness test for the Jelinek-Mercer LM model. */ + public void testLMJelinekMercer() throws IOException { + float p = (1 - 0.1f) * FREQ / DOC_LEN + + 0.1f * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f); + float gold = (float)(Math.log( + p / (0.1f * TOTAL_TERM_FREQ / (NUMBER_OF_FIELD_TOKENS + 1.0f)))); + correctnessTestCore(new LMJelinekMercerSimilarity(0.1f), gold); + } + + /** + * Correctness test for the LL IB model with DF-based lambda and + * no normalization. + */ + public void testLLForIB() throws IOException { + EasySimilarity sim = new IBSimilarity(new DistributionLL(), new LambdaDF()); + correctnessTestCore(sim, 4.26267987704f); + } + + /** + * Correctness test for the SPL IB model with TTF-based lambda and + * no normalization. + */ + public void testSPLForIB() throws IOException { + EasySimilarity sim = + new IBSimilarity(new DistributionSPL(), new LambdaTTF()); + correctnessTestCore(sim, 2.24069910825f); + } + + /** Correctness test for the PL2 DFR model. */ + public void testPL2() throws IOException { + EasySimilarity sim = new DFRSimilarity( + new BasicModelP(), new AfterEffectL(), new NormalizationH2()); + float tfn = (float)(FREQ * EasySimilarity.log2( + 1 + AVG_FIELD_LENGTH / DOC_LEN)); // 8.1894750101 + float l = 1.0f / (tfn + 1.0f); // 0.108820144666 + float lambda = (1.0f * TOTAL_TERM_FREQ) / NUMBER_OF_DOCUMENTS; // 0.7 + float p = (float)(tfn * EasySimilarity.log2(tfn / lambda) + + (lambda + 1 / (12 * tfn) - tfn) * EasySimilarity.log2(Math.E) + + 0.5 * EasySimilarity.log2(2 * Math.PI * tfn)); // 21.1113611585 + float gold = l * p; // 2.29734137536 + correctnessTestCore(sim, gold); + } + + /** Correctness test for the IneB2 DFR model. */ + public void testIneB2() throws IOException { + EasySimilarity sim = new DFRSimilarity( + new BasicModelIne(), new AfterEffectB(), new NormalizationH2()); + correctnessTestCore(sim, 6.23455315685f); + } + + /** Correctness test for the GL1 DFR model. */ + public void testGL1() throws IOException { + EasySimilarity sim = new DFRSimilarity( + new BasicModelG(), new AfterEffectL(), new NormalizationH1()); + correctnessTestCore(sim, 1.22733118352f); + } + + /** Correctness test for the BEB1 DFR model. */ + public void testBEB1() throws IOException { + EasySimilarity sim = new DFRSimilarity( + new BasicModelBE(), new AfterEffectB(), new NormalizationH1()); + float tfn = FREQ * AVG_FIELD_LENGTH / DOC_LEN; // 8.75 + float b = (TOTAL_TERM_FREQ + 1) / (DOC_FREQ * (tfn + 1)); // 0.728205128205 + float n1 = NUMBER_OF_DOCUMENTS + 1 + TOTAL_TERM_FREQ - 1; // 170 + float m1 = NUMBER_OF_DOCUMENTS + 1 + TOTAL_TERM_FREQ - tfn - 2; // 160.25 + float n2 = TOTAL_TERM_FREQ; // 70 + float m2 = TOTAL_TERM_FREQ - tfn; // 61.25 + float be = (float)(-EasySimilarity.log2(NUMBER_OF_DOCUMENTS + 1 - 1) - + EasySimilarity.log2(Math.E) + // -8.08655123066 + ((m1 + 0.5f) * EasySimilarity.log2(n1 / m1) + + (n1 - m1) * EasySimilarity.log2(n1)) - // 85.9391317425 + ((m2 + 0.5f) * EasySimilarity.log2(n2 / m2) + + (n2 - m2) * EasySimilarity.log2(n2))); // 65.5270599612 + // 12.3255205506 + float gold = b * be; // 8.97550727277 + correctnessTestCore(sim, gold); + } + + /** + * The generic test core called by all correctness test methods. It calls the + * {@link EasySimilarity#score(EasyStats, float, int)} method of all + * Similarities in {@link #sims} and compares the score against the manually + * computed {@code gold}. + */ + private void correctnessTestCore(EasySimilarity sim, float gold) + throws IOException { + // We have to fake everything, because computeStats() can be overridden and + // there is no way to inject false data after fillEasyStats(). + EasyStats stats = createStats(); + SpoofIndexSearcher searcher = new SpoofIndexSearcher(stats); + TermContext tc = new TermContext( + searcher.getIndexReader().getTopReaderContext(), + new OrdTermState(), 0, stats.getDocFreq(), stats.getTotalTermFreq()); + + EasyStats realStats = sim.computeStats( + searcher, "spoof", stats.getTotalBoost(), tc); + float score = sim.score(realStats, FREQ, DOC_LEN); + assertEquals( + sim.toString() + " score not correct.", gold, score, FLOAT_EPSILON); + } + // ---------------------------- Integration tests ---------------------------- /** The "collection" for the integration tests. */ @@ -459,4 +576,4 @@ public class TestEasySimilarity extends LuceneTestCase { return sim; } } -} \ No newline at end of file +}