diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java index 98594b6..7c253d9 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java @@ -36,7 +36,8 @@ public class BasicModelD extends BasicModel { double nphi = 1 - phi; double p = 1.0 / (stats.getNumberOfDocuments() + 1); double D = phi * log2(phi / p) + nphi * log2(nphi / (1 - p)); - return (float)(D * F + 0.5 * log2(2 * Math.PI * tfn * nphi)); + // nocommit return (float)(D * F + 0.5 * log2(2 * Math.PI * tfn * nphi)); + return (float)(D * F + 0.5 * log2(1 + 2 * Math.PI * tfn * nphi)); } @Override diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java index 654133a..72ae4c1 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java @@ -24,12 +24,19 @@ import static org.apache.lucene.search.similarities.EasySimilarity.log2; * @lucene.experimental */ public class BasicModelP extends BasicModel { + /** {@code log2(Math.E)}, precomputed. */ + protected static double LOG2_E = log2(Math.E); + @Override public final float score(EasyStats stats, float tfn) { float lambda = (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); - return (float)(tfn * log2(tfn / lambda) - + (lambda + 1 / 12 / tfn - tfn) * log2(Math.E) +// System.out.printf("tfn=%f, lambda=%f, log1=%f, log2=%f%n", tfn, lambda, +// tfn / lambda, 2 * Math.PI * tfn); + // nocommit + float score = (float)(tfn * log2(tfn / lambda) + + (lambda + 1 / 12 / tfn - tfn) * LOG2_E + 0.5 * log2(2 * Math.PI * tfn)); + return score > 0.0f ? score : 0.0f; } @Override diff --git lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java index 2da29fa..cec865b 100644 --- lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java @@ -94,14 +94,12 @@ public abstract class EasySimilarity extends Similarity { // nocommit: we have to provide something if codec doesnt supply these measures, // or if someone omitted frequencies for the field... negative values cause // NaN/Inf for some scorers. - if (numberOfFieldTokens == -1) { - numberOfFieldTokens = 1; - avgFieldLength = 1; + numberOfFieldTokens = 100 * numberOfDocuments; + avgFieldLength = 100; } - if (totalTermFreq == -1) { - totalTermFreq = 1; + totalTermFreq = Math.min(numberOfDocuments / 10, 10); } stats.setNumberOfDocuments(numberOfDocuments); @@ -249,7 +247,8 @@ public abstract class EasySimilarity extends Similarity { @Override public float score(int doc, int freq) { // nocommit: we have to supply something in case norms are omitted - return EasySimilarity.this.score(stats, freq, norms == null ? 1 : decodeNormValue(norms[doc])); + return EasySimilarity.this.score( + stats, freq, norms == null ? 10 * freq : decodeNormValue(norms[doc])); } @Override @@ -277,7 +276,8 @@ public abstract class EasySimilarity extends Similarity { @Override public float score(int doc, float freq) { // nocommit: we have to supply something in case norms are omitted - return EasySimilarity.this.score(stats, freq, norms == null ? 1 : decodeNormValue(norms[doc])); + return EasySimilarity.this.score(stats, freq, + norms == null ? 10 * (int)(freq + 0.5) : decodeNormValue(norms[doc])); } @Override public Explanation explain(int doc, Explanation freq) { diff --git lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java index 2c228f8..7db0136 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java @@ -25,6 +25,12 @@ import org.apache.lucene.search.Explanation; * Ad Hoc information retrieval. In Proceedings of the 24th annual international * ACM SIGIR conference on Research and development in information retrieval * (SIGIR '01). ACM, New York, NY, USA, 334-342. + *

+ * The formula as defined the paper assigns a negative score to documents that + * contain the term, but with fewer occurrences than predicted by the collection + * language model. The Lucene implementation returns {@code 0} for such + * documents. + *

* * @lucene.experimental */ @@ -55,10 +61,10 @@ public class LMDirichletSimilarity extends LMSimilarity { @Override protected float score(EasyStats stats, float freq, int docLen) { - return stats.getTotalBoost() * - (float)(Math.log(1 + freq / - (mu * ((LMStats)stats).getCollectionProbability())) + + float score = stats.getTotalBoost() * (float)(Math.log(1 + freq / + (mu * ((LMStats)stats).getCollectionProbability())) + Math.log(mu / (docLen + mu))); + return score > 0.0f ? score : 0.0f; } @Override diff --git lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java index 971a4d1..ab8a653 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java @@ -136,12 +136,12 @@ public abstract class LMSimilarity extends EasySimilarity { /** * Models {@code p(w|C)} as the number of occurrences of the term in the - * collection, divided by the total number of tokens. + * collection, divided by the total number of tokens {@code + 1}. */ public static class DefaultCollectionModel implements CollectionModel { @Override public float computeProbability(EasyStats stats) { - return (float)stats.getTotalTermFreq() / stats.getNumberOfFieldTokens(); + return (float)stats.getTotalTermFreq() / (stats.getNumberOfFieldTokens() +1); } @Override diff --git lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java index ad7799b..658b846 100644 --- lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java @@ -183,19 +183,6 @@ public class TestEasySimilarity extends LuceneTestCase { for (EasySimilarity sim : sims) { EasyStats realStats = sim.computeStats(new SpoofIndexSearcher(stats), "spoof", stats.getTotalBoost(), tc); -// System.out.printf("Before: %d %d %f %d %d%n", -// realStats.getNumberOfDocuments(), realStats.getNumberOfFieldTokens(), -// realStats.getAvgFieldLength(), realStats.getDocFreq(), -// realStats.getTotalTermFreq()); -// realStats.setNumberOfDocuments(stats.getNumberOfDocuments()); -// realStats.setNumberOfFieldTokens(stats.getNumberOfFieldTokens()); -// realStats.setAvgFieldLength(stats.getAvgFieldLength()); -// realStats.setDocFreq(stats.getDocFreq()); -// realStats.setTotalTermFreq(stats.getTotalTermFreq()); -// System.out.printf("After: %d %d %f %d %d%n", -// realStats.getNumberOfDocuments(), realStats.getNumberOfFieldTokens(), -// realStats.getAvgFieldLength(), realStats.getDocFreq(), -// realStats.getTotalTermFreq()); float score = sim.score(realStats, freq, docLen); assertFalse("Score infinite: " + sim.toString(), Float.isInfinite(score)); assertFalse("Score NaN: " + sim.toString(), Float.isNaN(score)); @@ -217,7 +204,7 @@ public class TestEasySimilarity extends LuceneTestCase { stats.setNumberOfFieldTokens(stats.getNumberOfDocuments()); stats.setTotalTermFreq(stats.getDocFreq()); stats.setAvgFieldLength( - stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); + (float)stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); unitTestCore(stats, FREQ, DOC_LEN); } @@ -230,10 +217,10 @@ public class TestEasySimilarity extends LuceneTestCase { stats.setNumberOfFieldTokens(stats.getNumberOfDocuments() * 2 / 3); stats.setTotalTermFreq(stats.getDocFreq()); stats.setAvgFieldLength( - stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); + (float)stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); unitTestCore(stats, FREQ, DOC_LEN); } - + /** * Tests correct behavior when * {@code NumberOfDocuments = 1}. @@ -459,4 +446,4 @@ public class TestEasySimilarity extends LuceneTestCase { return sim; } } -} \ No newline at end of file +}