diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java index f096b0b..66c3b72 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java @@ -26,7 +26,6 @@ import static org.apache.lucene.search.similarities.EasySimilarity.log2; public class BasicModelIF extends BasicModel { @Override public final float score(EasyStats stats, float tfn) { - // TODO: Refactor this method to a parent class? See the other two Ix model. int N = stats.getNumberOfDocuments(); long F = stats.getTotalTermFreq(); return tfn * (float)(log2((N + 1) / (F + 0.5))); diff --git lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java index 204bbdc..c7473ae 100644 --- lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java @@ -82,13 +82,13 @@ public abstract class EasySimilarity extends Similarity { fieldName).getSumTotalTermFreq(); float avgFieldLength = (float)numberOfFieldTokens / numberOfDocuments; - // nocommit This is for phrases, and it doesn't really work... have to - // find a method that makes sense - int docFreq = 0; - long totalTermFreq = 0; + // nocommit Take the minimum of term frequencies for phrases. This is not + // correct though, we'll need something like a scorePhrase(MultiStats ...) + int docFreq = Integer.MAX_VALUE; + long totalTermFreq = Integer.MAX_VALUE; for (final TermContext context : termContexts) { - docFreq += context.docFreq(); - totalTermFreq += context.totalTermFreq(); + docFreq = Math.min(docFreq, context.docFreq()); + totalTermFreq = Math.min(totalTermFreq, context.totalTermFreq()); } stats.setNumberOfDocuments(numberOfDocuments); @@ -96,7 +96,6 @@ public abstract class EasySimilarity extends Similarity { stats.setAvgFieldLength(avgFieldLength); stats.setDocFreq(docFreq); stats.setTotalTermFreq(totalTermFreq); - // nocommit uniqueTermCount? (LUCENE-3290) } /** @@ -193,14 +192,12 @@ public abstract class EasySimilarity extends Similarity { /** Decodes a normalization factor (document length) stored in an index. * @see #encodeNormValue(float) */ - // nocommit to protected? - public int decodeNormValue(byte norm) { + protected int decodeNormValue(byte norm) { return NORM_TABLE[norm & 0xFF]; // & 0xFF maps negative bytes to positive above 127 } /** Encodes the length to a byte via SmallFloat. */ - // nocommit to protected? - public byte encodeNormValue(float length) { + protected byte encodeNormValue(float length) { return SmallFloat.floatToByte315((float)(1.0 / Math.sqrt(length))); } @@ -208,7 +205,7 @@ public abstract class EasySimilarity extends Similarity { /** Returns the base two logarithm of {@code x}. */ public static double log2(double x) { - // Put this to a 'util' class? + // Put this to a 'util' class if we need more of these. return Math.log(x) / LOG_2; } @@ -216,8 +213,8 @@ public abstract class EasySimilarity extends Similarity { /** Delegates the {@link #score(int, int)} and * {@link #explain(int, Explanation)} methods to - * {@link EasySimilarity#score(EasyStats, float, byte)} and - * {@link EasySimilarity#explain(EasyStats, int, Explanation, byte)}, + * {@link EasySimilarity#score(EasyStats, float, int)} and + * {@link EasySimilarity#explain(EasyStats, int, Explanation, int)}, * respectively. */ private class EasyExactDocScorer extends ExactDocScorer { @@ -243,8 +240,8 @@ public abstract class EasySimilarity extends Similarity { /** Delegates the {@link #score(int, int)} and * {@link #explain(int, Explanation)} methods to - * {@link EasySimilarity#score(EasyStats, float, byte)} and - * {@link EasySimilarity#explain(EasyStats, int, Explanation, byte)}, + * {@link EasySimilarity#score(EasyStats, float, int)} and + * {@link EasySimilarity#explain(EasyStats, int, Explanation, int)}, * respectively. */ private class EasySloppyDocScorer extends SloppyDocScorer { @@ -256,7 +253,6 @@ public abstract class EasySimilarity extends Similarity { this.norms = norms; } - // todo: optimize @Override public float score(int doc, float freq) { return EasySimilarity.this.score(stats, freq, decodeNormValue(norms[doc])); @@ -272,7 +268,6 @@ public abstract class EasySimilarity extends Similarity { return 1.0f / (distance + 1); } - // nocommit: do we care about exposing this? @Override public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { return 1f; diff --git lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java index 8b0e8aa..3761933 100644 --- lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java +++ lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java @@ -27,7 +27,7 @@ import org.apache.lucene.search.Similarity; */ public class EasyStats extends Similarity.Stats { /** The number of documents. */ - protected int numberOfDocuments; // TODO: to long? + protected int numberOfDocuments; /** The total number of tokens in the field. */ protected long numberOfFieldTokens; /** The average field length. */ @@ -36,9 +36,6 @@ public class EasyStats extends Similarity.Stats { protected int docFreq; /** The total number of occurrences of this term across all documents. */ protected long totalTermFreq; - /** The number of unique terms. */ - // nocommit might be per-segment only - protected long uniqueTermCount; // -------------------------- Boost-related stuff -------------------------- @@ -114,16 +111,6 @@ public class EasyStats extends Similarity.Stats { this.totalTermFreq = totalTermFreq; } - /** Returns the number of unique terms. */ - public long getUniqueTermCount() { - return uniqueTermCount; - } - - /** Sets the number of unique terms. */ - public void setUniqueTermCount(long uniqueTermCount) { - this.uniqueTermCount = uniqueTermCount; - } - // -------------------------- Boost-related stuff -------------------------- /** The square of the raw normalization value. diff --git lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java index 60f7a68..07402dc 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java @@ -67,8 +67,8 @@ public class LMDirichletSimilarity extends LMSimilarity { if (stats.getTotalBoost() != 1.0f) { expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); } - - // nocommit: mu? + + expl.addDetail(new Explanation(mu, "mu")); Explanation weightExpl = new Explanation(); weightExpl.setValue((float)Math.log(1 + freq / (mu * ((LMStats)stats).getCollectionProbability()))); diff --git lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java index 2b1375f..e047dad 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java @@ -62,6 +62,7 @@ public class LMJelinekMercerSimilarity extends LMSimilarity { if (stats.getTotalBoost() != 1.0f) { expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); } + expl.addDetail(new Explanation(lambda, "lambda")); super.explain(expl, stats, doc, freq, docLen); }