diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java index 8c2662f..faee912 100644 --- lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java +++ lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java @@ -47,5 +47,17 @@ public abstract class AfterEffect { public final Explanation explain(EasyStats stats, float tfn) { return new Explanation(1, "no aftereffect"); } + + @Override + public String toString() { + return ""; + } } + + /** + * Subclasses must override this method to return the code of the + * after effect formula. Refer to the original paper for the list. + */ + @Override + public abstract String toString(); } diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java index 0edc2e2..90079b2 100644 --- lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java +++ lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java @@ -41,4 +41,9 @@ public class AfterEffectB extends AfterEffect { result.addDetail(new Explanation(stats.getDocFreq(), "docFreq")); return result; } + + @Override + public String toString() { + return "B"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java index b20d784..3a1f223 100644 --- lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java +++ lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java @@ -37,4 +37,9 @@ public class AfterEffectL extends AfterEffect { result.addDetail(new Explanation(tfn, "tfn")); return result; } + + @Override + public String toString() { + return "L"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java index d07a6fe..e811869 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java @@ -50,4 +50,11 @@ public abstract class BasicModel { new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); return result; } + + /** + * Subclasses must override this method to return the code of the + * basic model formula. Refer to the original paper for the list. + */ + @Override + public abstract String toString(); } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java index 366fce3..12fb133 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java @@ -20,14 +20,18 @@ package org.apache.lucene.search.similarities; import static org.apache.lucene.search.similarities.EasySimilarity.log2; /** - * Limiting form of the Bose-Einstein model. + * Limiting form of the Bose-Einstein model. The formula used in Lucene differs + * slightly from the one in the original paper: to avoid underflow for small + * values of {@code N} and {@code F}, {@code N} is increased by {@code 1} and + * {@code F} is ensured to be at least {@code tfn + 1}. * @lucene.experimental */ public class BasicModelBE extends BasicModel { @Override public final float score(EasyStats stats, float tfn) { - long N = stats.getNumberOfDocuments(); - long F = stats.getTotalTermFreq(); + long N = stats.getNumberOfDocuments() + 1; +// long F = stats.getTotalTermFreq() + 1; + long F = Math.max(stats.getTotalTermFreq(), (long)(tfn + 0.5) + 1); return (float)(-log2((N - 1) * Math.E) + f(N + F -1, N + F - tfn - 2) - f(F, F - tfn)); } @@ -36,4 +40,9 @@ public class BasicModelBE extends BasicModel { private final double f(long n, float m) { return (m + 0.5) * log2((double)n / m) + (n - m) * log2(n); } + + @Override + public String toString() { + return "Be"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java index b477bc9..98594b6 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java @@ -21,17 +21,26 @@ import static org.apache.lucene.search.similarities.EasySimilarity.log2; /** * Implements the approximation of the binomial model with the divergence - * for DFR. + * for DFR. The formula used in Lucene differs slightly from the one in the + * original paper: to avoid underflow for small values of {@code N} and + * {@code F}, {@code N} is increased by {@code 1} and + * {@code F} is ensured to be at least {@code tfn + 1}. * @lucene.experimental */ public class BasicModelD extends BasicModel { @Override public final float score(EasyStats stats, float tfn) { - long F = stats.getTotalTermFreq(); + long F = Math.max(stats.getTotalTermFreq(), (long)(tfn + 0.5) + 1); +// long F = stats.getTotalTermFreq() + 1; double phi = (double)tfn / F; double nphi = 1 - phi; - double p = 1.0 / stats.getNumberOfDocuments(); + double p = 1.0 / (stats.getNumberOfDocuments() + 1); double D = phi * log2(phi / p) + nphi * log2(nphi / (1 - p)); return (float)(D * F + 0.5 * log2(2 * Math.PI * tfn * nphi)); } + + @Override + public String toString() { + return "D"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java index 0804e84..4a8e75a 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java @@ -30,4 +30,9 @@ public class BasicModelG extends BasicModel { // -log(1 / (lambda + 1)) -> log(lambda + 1) return (float)(log2(lambda + 1) + tfn * log2((1 + lambda) / lambda)); } + + @Override + public String toString() { + return "G"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java index 66c3b72..c073a8c 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java @@ -30,4 +30,9 @@ public class BasicModelIF extends BasicModel { long F = stats.getTotalTermFreq(); return tfn * (float)(log2((N + 1) / (F + 0.5))); } + + @Override + public String toString() { + return "I(F)"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java index 4556850..3d7bf96 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java @@ -44,4 +44,9 @@ public class BasicModelIn extends BasicModel { new Explanation(stats.getDocFreq(), "docFreq")); return result; } + + @Override + public String toString() { + return "I(n)"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java index 4b24790..e9b51ac 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java @@ -32,4 +32,9 @@ public class BasicModelIne extends BasicModel { double ne = N * (1 - Math.pow((N - 1) / (double)N, F)); return tfn * (float)(log2((N + 1) / (ne + 0.5))); } + + @Override + public String toString() { + return "I(ne)"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java index 308f5d6..654133a 100644 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java +++ lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java @@ -31,4 +31,9 @@ public class BasicModelP extends BasicModel { + (lambda + 1 / 12 / tfn - tfn) * log2(Math.E) + 0.5 * log2(2 * Math.PI * tfn)); } + + @Override + public String toString() { + return "P"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java index 16119de..ec6c420 100644 --- lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java @@ -97,4 +97,10 @@ public class DFRSimilarity extends EasySimilarity { expl.addDetail(basicModel.explain(stats, tfn)); expl.addDetail(afterEffect.explain(stats, tfn)); } + + @Override + public String toString() { + return "DFR " + basicModel.toString() + afterEffect.toString() + + normalization.toString(); + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/Distribution.java lucene/src/test/org/apache/lucene/search/similarities/Distribution.java index 11f2bcb..aa2702a 100644 --- lucene/src/test/org/apache/lucene/search/similarities/Distribution.java +++ lucene/src/test/org/apache/lucene/search/similarities/Distribution.java @@ -35,4 +35,11 @@ public abstract class Distribution { return new Explanation( score(stats, tfn, lambda), getClass().getSimpleName()); } + + /** + * Subclasses must override this method to return the name of the + * distribution. + */ + @Override + public abstract String toString(); } diff --git lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java index de5375b..f7a4a4a 100644 --- lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java +++ lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java @@ -29,4 +29,9 @@ public class DistributionLL extends Distribution { public final float score(EasyStats stats, float tfn, float lambda) { return (float)-Math.log(lambda / (tfn + lambda)); } + + @Override + public String toString() { + return "LL"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java index 892d7dc..1027543 100644 --- lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java +++ lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java @@ -28,7 +28,15 @@ package org.apache.lucene.search.similarities; public class DistributionSPL extends Distribution { @Override public final float score(EasyStats stats, float tfn, float lambda) { + if (lambda == 1f) { + lambda = 0.99f; + } return (float)-Math.log( (Math.pow(lambda, (tfn / (tfn + 1))) - lambda) / (1 - lambda)); } + + @Override + public String toString() { + return "SPL"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java index c7473ae..46ae8cf 100644 --- lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java @@ -165,6 +165,13 @@ public abstract class EasySimilarity extends Similarity { return new EasySloppyDocScorer((EasyStats) stats, context.reader.norms(fieldName)); } + + /** + * Subclasses must override this method to return the name of the Similarity + * and preferably the values of parameters (if any) as well. + */ + @Override + public abstract String toString(); // nocommit: to Similarity? // ------------------------------ Norm handling ------------------------------ diff --git lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java index 3761933..89b3263 100644 --- lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java +++ lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java @@ -51,6 +51,7 @@ public class EasyStats extends Similarity.Stats { /** Constructor. Sets the query boost. */ public EasyStats(float queryBoost) { this.queryBoost = queryBoost; + this.totalBoost = queryBoost; } // ------------------------- Getter/setter methods ------------------------- diff --git lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java index 15a967e..15c9de7 100644 --- lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java @@ -84,4 +84,16 @@ public class IBSimilarity extends EasySimilarity { expl.addDetail(distribution.explain( stats, normExpl.getValue(), lambdaExpl.getValue())); } + + /** + * The name of IB methods follow the pattern + * {@code IB }. The name of the + * distribution is the same as in the original paper; for the names of lambda + * parameters, refer to the javadoc of the {@link Lambda} classes. + */ + @Override + public String toString() { + return "IB " + distribution.toString() + "-" + lambda.toString() + + normalization.toString(); + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java index 07402dc..2c228f8 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java @@ -83,4 +83,9 @@ public class LMDirichletSimilarity extends LMSimilarity { public float getMu() { return mu; } + + @Override + public String getName() { + return String.format("Dirichlet(%f)", getMu()); + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java index e047dad..b181367 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java @@ -70,4 +70,9 @@ public class LMJelinekMercerSimilarity extends LMSimilarity { public float getLambda() { return lambda; } + + @Override + public String getName() { + return String.format("Jelinek-Mercer(%f)", getLambda()); + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java index 0abea21..971a4d1 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java @@ -71,6 +71,30 @@ public abstract class LMSimilarity extends EasySimilarity { expl.addDetail(new Explanation(collectionModel.computeProbability(stats), "collection probability")); } + + /** + * Returns the name of the LM method. The values of the parameters should be + * included as well. + *

Used in {@link #toString()}

. + */ + public abstract String getName(); + + /** + * Returns the name of the LM method. If a custom collection model strategy is + * used, its name is included as well. + * @see #getName() + * @see CollectionModel#getName() + * @see DefaultCollectionModel + */ + @Override + public String toString() { + String coll = collectionModel.getName(); + if (coll != null) { + return String.format("LM %s - %s", getName(), coll); + } else { + return String.format("LM %s", getName()); + } + } /** Stores the collection distribution of the current term. */ public static class LMStats extends EasyStats { @@ -105,6 +129,9 @@ public abstract class LMSimilarity extends EasySimilarity { * strategy for the current term. */ public float computeProbability(EasyStats stats); + + /** The name of the collection model strategy. */ + public String getName(); } /** @@ -116,5 +143,10 @@ public abstract class LMSimilarity extends EasySimilarity { public float computeProbability(EasyStats stats) { return (float)stats.getTotalTermFreq() / stats.getNumberOfFieldTokens(); } + + @Override + public String getName() { + return null; + } } } diff --git lucene/src/test/org/apache/lucene/search/similarities/Lambda.java lucene/src/test/org/apache/lucene/search/similarities/Lambda.java index 4b658f7..5df1b4a 100644 --- lucene/src/test/org/apache/lucene/search/similarities/Lambda.java +++ lucene/src/test/org/apache/lucene/search/similarities/Lambda.java @@ -30,4 +30,13 @@ public abstract class Lambda { public abstract float lambda(EasyStats stats); /** Explains the lambda parameter. */ public abstract Explanation explain(EasyStats stats); + + /** + * Subclasses must override this method to return the code of the lambda + * formula. Since the original paper is not very clear on this matter, and + * also uses the DFR naming scheme incorrectly, the codes here were chosen + * arbitrarily. + */ + @Override + public abstract String toString(); } diff --git lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java index f8d7b30..4786ec6 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java +++ lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java @@ -26,18 +26,23 @@ import org.apache.lucene.search.Explanation; public class LambdaDF extends Lambda { @Override public final float lambda(EasyStats stats) { - return (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); + return (float)stats.getDocFreq() / stats.getNumberOfDocuments(); } - + @Override public final Explanation explain(EasyStats stats) { Explanation result = new Explanation(); result.setDescription(getClass().getSimpleName() + ", computed from: "); result.setValue(lambda(stats)); result.addDetail( - new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); + new Explanation(stats.getDocFreq(), "docFreq")); result.addDetail( new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); return result; } + + @Override + public String toString() { + return "D"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java index 0fd3b99..f07f146 100644 --- lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java +++ lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java @@ -23,21 +23,26 @@ import org.apache.lucene.search.Explanation; * Computes lambda as {@code docFreq / numberOfDocuments}. * @lucene.experimental */ -public class LambdaTTF extends Lambda { +public class LambdaTTF extends Lambda { @Override public final float lambda(EasyStats stats) { - return (float)stats.getDocFreq() / stats.getNumberOfDocuments(); + return (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); } - + @Override public final Explanation explain(EasyStats stats) { Explanation result = new Explanation(); result.setDescription(getClass().getSimpleName() + ", computed from: "); result.setValue(lambda(stats)); result.addDetail( - new Explanation(stats.getDocFreq(), "docFreq")); + new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); result.addDetail( new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); return result; } + + @Override + public String toString() { + return "L"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/Normalization.java lucene/src/test/org/apache/lucene/search/similarities/Normalization.java index a9a9d4f..68ae759 100644 --- lucene/src/test/org/apache/lucene/search/similarities/Normalization.java +++ lucene/src/test/org/apache/lucene/search/similarities/Normalization.java @@ -59,5 +59,17 @@ public abstract class Normalization { public final Explanation explain(EasyStats stats, float tf, int len) { return new Explanation(1, "no normalization"); } + + @Override + public String toString() { + return ""; + } } + + /** + * Subclasses must override this method to return the code of the + * normalization formula. Refer to the original paper for the list. + */ + @Override + public abstract String toString(); } diff --git lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java index eab4ef6..4201c59 100644 --- lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java +++ lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java @@ -25,4 +25,9 @@ public class NormalizationH1 extends Normalization { public final float tfn(EasyStats stats, float tf, int len) { return tf * stats.getAvgFieldLength() / len; } + + @Override + public String toString() { + return "1"; + } } diff --git lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java index df3d4f4..5d24eca 100644 --- lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java +++ lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java @@ -28,4 +28,9 @@ public class NormalizationH2 extends Normalization { public final float tfn(EasyStats stats, float tf, int len) { return (float)(tf * log2(1 + stats.getAvgFieldLength() / len)); } + + @Override + public String toString() { + return "2"; + } } \ No newline at end of file diff --git lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java new file mode 100644 index 0000000..ed886fb --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java @@ -0,0 +1,214 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Comparator; +import java.util.Map; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.Fields; +import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.TermFreqVector; +import org.apache.lucene.index.TermVectorMapper; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PerDocValues; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; + +/** + * Index searcher implementation that takes an {@link EasyStats} instance and + * returns statistics accordingly. Most of the methods are not implemented, so + * it can only be used for Similarity unit testing. + */ +public class SpoofIndexSearcher extends IndexSearcher { + public SpoofIndexSearcher(EasyStats stats) { + super(new SpoofIndexReader(stats)); + } + + public static class SpoofIndexReader extends IndexReader { + /** The stats the reader has to return. */ + protected EasyStats stats; + /** The fields the reader has to return. */ + protected SpoofFields fields; + + public SpoofIndexReader(EasyStats stats) { + this.stats = stats; + this.fields = new SpoofFields(stats); + } + + @Override + public int numDocs() { + return stats.getNumberOfDocuments(); + } + + @Override + public int maxDoc() { + return stats.getNumberOfDocuments(); + } + + @Override + public Fields fields() throws IOException { + return fields; + } + + @Override + public Collection getFieldNames(FieldOption fldOption) { + return Arrays.asList(new String[]{"spoof"}); + } + + @Override + public ReaderContext getTopReaderContext() { + return new AtomicReaderContext(this); + } + + @Override + public boolean hasDeletions() { + return false; + } + + // ------------------------ Not implemented methods ------------------------ + + @Override + public TermFreqVector[] getTermFreqVectors(int docNumber) + throws IOException { + return null; + } + + @Override + public TermFreqVector getTermFreqVector(int docNumber, String field) + throws IOException { + return null; + } + + @Override + public void getTermFreqVector(int docNumber, String field, + TermVectorMapper mapper) throws IOException { + } + + @Override + public void getTermFreqVector(int docNumber, TermVectorMapper mapper) + throws IOException { + } + + @Override + public Document document(int n, FieldSelector fieldSelector) + throws CorruptIndexException, IOException { + return null; + } + + @Override + public byte[] norms(String field) throws IOException { + return null; + } + + @Override + protected void doSetNorm(int doc, String field, byte value) + throws CorruptIndexException, IOException { + } + + @Override + public PerDocValues perDocValues() throws IOException { + return null; + } + + @Override + protected void doDelete(int docNum) throws CorruptIndexException, + IOException { + } + + @Override + protected void doUndeleteAll() throws CorruptIndexException, IOException { + } + + @Override + protected void doCommit(Map commitUserData) + throws IOException { + } + + @Override + protected void doClose() throws IOException { + } + + @Override + public Bits getLiveDocs() { + return null; + } + } + + /** Spoof Fields class for Similarity testing. */ + public static class SpoofFields extends Fields { + /** The stats the object has to return. */ + protected SpoofTerms terms; + + public SpoofFields(EasyStats stats) { + this.terms = new SpoofTerms(stats); + } + + @Override + public Terms terms(String field) throws IOException { + return terms; + } + + // ------------------------ Not implemented methods ------------------------ + + @Override + public FieldsEnum iterator() throws IOException { + return null; + } + } + + /** Spoof Terms class for Similarity testing. */ + public static class SpoofTerms extends Terms { + /** The stats the object has to return. */ + protected EasyStats stats; + + public SpoofTerms(EasyStats stats) { + this.stats = stats; + } + + @Override + public long getSumTotalTermFreq() throws IOException { + return stats.getNumberOfFieldTokens(); + } + + @Override + public long getSumDocFreq() throws IOException { + return stats.getDocFreq(); + } + + // ------------------------ Not implemented methods ------------------------ + + @Override + public TermsEnum iterator() throws IOException { + return null; + } + + @Override + public Comparator getComparator() throws IOException { + return null; + } + } +} \ No newline at end of file diff --git lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java new file mode 100644 index 0000000..fd2bf10 --- /dev/null +++ lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java @@ -0,0 +1,462 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.OrdTermState; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.SimilarityProvider; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.TermContext; + +/** + * Tests the {@link EasySimilarity}-based Similarities. Contains unit tests and + * integration tests as well. This class maintains a list of + * {@code EasySimilarity} subclasses. Each test case performs its test on all + * items in the list. If a test case fails, the name of the Similarity that + * caused the failure is returned as part of the assertion error message. + *

Unit testing is performed by constructing statistics manually and calling + * the {@link EasySimilarity#score(EasyStats, float, int)} method of the + * Similarities. The statistics represent corner cases of corpus distributions. + *

+ *

For the integration tests, a small (8-document) collection is indexed. The + * tests verify that for a specific query, all relevant documents are returned + * in the correct order. The collection consists of two poems of English poet + * William Blake.

+ *

Note: the list of Similarities is maintained by hand. If a new Similarity + * is added to the {@code org.apache.lucene.search.similarities} package, the + * list should be updated accordingly.

+ */ +public class TestEasySimilarity extends LuceneTestCase { + private static String FIELD_BODY = "body"; + private static String FIELD_ID = "id"; + /** The DFR basic models to test. */ + private static BasicModel[] BASIC_MODELS; + /** The DFR aftereffects to test. */ + private static AfterEffect[] AFTER_EFFECTS; + /** The DFR normalizations to test. */ + private static Normalization[] NORMALIZATIONS; + /** The distributions for IB. */ + private static Distribution[] DISTRIBUTIONS; + /** Lambdas for IB. */ + private static Lambda[] LAMBDAS; + + static { + BASIC_MODELS = new BasicModel[] { + new BasicModelBE(), new BasicModelD(), new BasicModelG(), + new BasicModelIF(), new BasicModelIn(), new BasicModelIne(), + new BasicModelP() + }; + AFTER_EFFECTS = new AfterEffect[] { + new AfterEffectB(), new AfterEffectL(), new AfterEffect.NoAfterEffect() + }; + NORMALIZATIONS = new Normalization[] { + new NormalizationH1(), new NormalizationH2(), + new Normalization.NoNormalization() + }; + DISTRIBUTIONS = new Distribution[] { + new DistributionLL(), new DistributionSPL() + }; + LAMBDAS = new Lambda[] { + new LambdaDF(), new LambdaTTF() + }; + } + + private IndexSearcher searcher; + private Directory dir; + private IndexReader reader; + /** The list of similarities to test. */ + private List sims; + + @Override + public void setUp() throws Exception { + super.setUp(); + + dir = newDirectory(); + RandomIndexWriter writer = new RandomIndexWriter(random, dir); + + for (int i = 0; i < docs.length; i++) { + Document d = new Document(); + d.add(newField(FIELD_ID, Integer.toString(i), Field.Store.YES, Field.Index.NO)); + d.add(newField(FIELD_BODY, docs[i], Field.Index.ANALYZED)); + writer.addDocument(d); + } + + reader = writer.getReader(); + searcher = newSearcher(reader); + writer.close(); + + sims = new ArrayList(); + for (BasicModel basicModel : BASIC_MODELS) { + for (AfterEffect afterEffect : AFTER_EFFECTS) { + for (Normalization normalization : NORMALIZATIONS) { + sims.add(new DFRSimilarity(basicModel, afterEffect, normalization)); + } + } + } + for (Distribution distribution : DISTRIBUTIONS) { + for (Lambda lambda : LAMBDAS) { + for (Normalization normalization : NORMALIZATIONS) { + sims.add(new IBSimilarity(distribution, lambda, normalization)); + } + } + } + sims.add(new LMDirichletSimilarity()); + sims.add(new LMJelinekMercerSimilarity(0.1f)); + sims.add(new LMJelinekMercerSimilarity(0.7f)); + } + + // ------------------------------- Unit tests -------------------------------- + + /** The default number of documents in the unit tests. */ + private static int NUMBER_OF_DOCUMENTS = 100; + /** The default total number of tokens in the field in the unit tests. */ + private static long NUMBER_OF_FIELD_TOKENS = 5000; + /** The default average field length in the unit tests. */ + private static float AVG_FIELD_LENGTH = 50; + /** The default document frequency in the unit tests. */ + private static int DOC_FREQ = 10; + /** + * The default total number of occurrences of this term across all documents + * in the unit tests. + */ + private static long TOTAL_TERM_FREQ = 70; + + /** The default tf in the unit tests. */ + private static float FREQ = 7; + /** The default document length in the unit tests. */ + private static int DOC_LEN = 40; + + /** Creates the default statistics object that the specific tests modify. */ + private EasyStats createStats() { + EasyStats stats = new EasyStats(1); + stats.setNumberOfDocuments(NUMBER_OF_DOCUMENTS); + stats.setNumberOfFieldTokens(NUMBER_OF_FIELD_TOKENS); + stats.setAvgFieldLength(AVG_FIELD_LENGTH); + stats.setDocFreq(DOC_FREQ); + stats.setTotalTermFreq(TOTAL_TERM_FREQ); + return stats; + } + + /** + * The generic test core called by all unit test methods. It calls the + * {@link EasySimilarity#score(EasyStats, float, int)} method of all + * Similarities in {@link #sims} and checks if the score is valid; i.e. it + * is a finite positive real number. + */ + private void unitTestCore(EasyStats stats, float freq, int docLen) + throws IOException { + // We have to fake everything, because computeStats() can be overridden and + // there is no way to inject false data after fillEasyStats(). + SpoofIndexSearcher searcher = new SpoofIndexSearcher(stats); + TermContext tc = new TermContext( + searcher.getIndexReader().getTopReaderContext(), + new OrdTermState(), 0, stats.getDocFreq(), stats.getTotalTermFreq()); + + for (EasySimilarity sim : sims) { + EasyStats realStats = sim.computeStats(new SpoofIndexSearcher(stats), + "spoof", stats.getTotalBoost(), tc); +// System.out.printf("Before: %d %d %f %d %d%n", +// realStats.getNumberOfDocuments(), realStats.getNumberOfFieldTokens(), +// realStats.getAvgFieldLength(), realStats.getDocFreq(), +// realStats.getTotalTermFreq()); +// realStats.setNumberOfDocuments(stats.getNumberOfDocuments()); +// realStats.setNumberOfFieldTokens(stats.getNumberOfFieldTokens()); +// realStats.setAvgFieldLength(stats.getAvgFieldLength()); +// realStats.setDocFreq(stats.getDocFreq()); +// realStats.setTotalTermFreq(stats.getTotalTermFreq()); +// System.out.printf("After: %d %d %f %d %d%n", +// realStats.getNumberOfDocuments(), realStats.getNumberOfFieldTokens(), +// realStats.getAvgFieldLength(), realStats.getDocFreq(), +// realStats.getTotalTermFreq()); + float score = sim.score(realStats, freq, docLen); + assertFalse("Score infinite: " + sim.toString(), Float.isInfinite(score)); + assertFalse("Score NaN: " + sim.toString(), Float.isNaN(score)); + assertTrue("Score negative: " + sim.toString(), score >= 0); + } + } + + /** Runs the unit test with the default statistics. */ + public void testDefault() throws IOException { + unitTestCore(createStats(), FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code numberOfDocuments = numberOfFieldTokens}. + */ + public void testSparseDocuments() throws IOException { + EasyStats stats = createStats(); + stats.setNumberOfFieldTokens(stats.getNumberOfDocuments()); + stats.setTotalTermFreq(stats.getDocFreq()); + stats.setAvgFieldLength( + stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code numberOfDocuments > numberOfFieldTokens}. + */ + public void testVerySparseDocuments() throws IOException { + EasyStats stats = createStats(); + stats.setNumberOfFieldTokens(stats.getNumberOfDocuments() * 2 / 3); + stats.setTotalTermFreq(stats.getDocFreq()); + stats.setAvgFieldLength( + stats.getNumberOfFieldTokens() / stats.getNumberOfDocuments()); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code NumberOfDocuments = 1}. + */ + public void testOneDocument() throws IOException { + EasyStats stats = createStats(); + stats.setNumberOfDocuments(1); + stats.setNumberOfFieldTokens(DOC_LEN); + stats.setAvgFieldLength(DOC_LEN); + stats.setDocFreq(1); + stats.setTotalTermFreq((int)FREQ); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code docFreq = numberOfDocuments}. + */ + public void testAllDocumentsRelevant() throws IOException { + EasyStats stats = createStats(); + float mult = (0.0f + stats.getNumberOfDocuments()) / stats.getDocFreq(); + stats.setTotalTermFreq((int)(stats.getTotalTermFreq() * mult)); + stats.setDocFreq(stats.getNumberOfDocuments()); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code docFreq > numberOfDocuments / 2}. + */ + public void testMostDocumentsRelevant() throws IOException { + EasyStats stats = createStats(); + float mult = (0.6f * stats.getNumberOfDocuments()) / stats.getDocFreq(); + stats.setTotalTermFreq((int)(stats.getTotalTermFreq() * mult)); + stats.setDocFreq((int)(stats.getNumberOfDocuments() * 0.6)); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code docFreq = 1}. + */ + public void testOnlyOneRelevantDocument() throws IOException { + EasyStats stats = createStats(); + stats.setDocFreq(1); + stats.setTotalTermFreq((int)FREQ + 3); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code totalTermFreq = numberOfFieldTokens}. + */ + public void testAllTermsRelevant() throws IOException { + EasyStats stats = createStats(); + stats.setTotalTermFreq(stats.getNumberOfFieldTokens()); + unitTestCore(stats, DOC_LEN, DOC_LEN); + // nocommit docLen > avglength + } + + /** + * Tests correct behavior when + * {@code totalTermFreq > numberOfDocuments}. + */ + public void testMoreTermsThanDocuments() throws IOException { + EasyStats stats = createStats(); + stats.setTotalTermFreq( + stats.getTotalTermFreq() + stats.getNumberOfDocuments()); + unitTestCore(stats, 2 * FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when + * {@code totalTermFreq = numberOfDocuments}. + */ + public void testNumberOfTermsAsDocuments() throws IOException { + EasyStats stats = createStats(); + stats.setTotalTermFreq(stats.getNumberOfDocuments()); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when {@code totalTermFreq = 1}. + */ + public void testOneTerm() throws IOException { + EasyStats stats = createStats(); + stats.setDocFreq(1); + stats.setTotalTermFreq(1); + unitTestCore(stats, 1, DOC_LEN); + } + + /** + * Tests correct behavior when {@code totalTermFreq = freq}. + */ + public void testOneRelevantDocument() throws IOException { + EasyStats stats = createStats(); + stats.setDocFreq(1); + stats.setTotalTermFreq((int)FREQ); + unitTestCore(stats, FREQ, DOC_LEN); + } + + /** + * Tests correct behavior when {@code numberOfFieldTokens = freq}. + */ + public void testAllTermsRelevantOnlyOneDocument() throws IOException { + EasyStats stats = createStats(); + stats.setNumberOfDocuments(10); + stats.setNumberOfFieldTokens(50); + stats.setAvgFieldLength(5); + stats.setDocFreq(1); + stats.setTotalTermFreq(50); + unitTestCore(stats, 50, 50); + } + + /** + * Tests correct behavior when there is only one document with a single term + * in the collection. + */ + public void testOnlyOneTermOneDocument() throws IOException { + EasyStats stats = createStats(); + stats.setNumberOfDocuments(1); + stats.setNumberOfFieldTokens(1); + stats.setAvgFieldLength(1); + stats.setDocFreq(1); + stats.setTotalTermFreq(1); + unitTestCore(stats, 1, 1); + } + + /** + * Tests correct behavior when there is only one term in the field, but + * more than one documents. + */ + public void testOnlyOneTerm() throws IOException { + EasyStats stats = createStats(); + stats.setNumberOfFieldTokens(1); + stats.setAvgFieldLength(1.0f / stats.getNumberOfDocuments()); + stats.setDocFreq(1); + stats.setTotalTermFreq(1); + unitTestCore(stats, 1, DOC_LEN); + } + + /** + * Tests correct behavior when {@code avgFieldLength = docLen}. + */ + public void testDocumentLengthAverage() throws IOException { + EasyStats stats = createStats(); + unitTestCore(stats, FREQ, (int)stats.getAvgFieldLength()); + } + + // ---------------------------- Integration tests ---------------------------- + + /** The "collection" for the integration tests. */ + String[] docs = new String[] { + "Tiger, tiger burning bright In the forest of the night What immortal hand or eye Could frame thy fearful symmetry ?", + "In what distant depths or skies Burnt the fire of thine eyes ? On what wings dare he aspire ? What the hands the seize the fire ?", + "And what shoulder and what art Could twist the sinews of thy heart ? And when thy heart began to beat What dread hand ? And what dread feet ?", + "What the hammer? What the chain ? In what furnace was thy brain ? What the anvil ? And what dread grasp Dare its deadly terrors clasp ?", + "And when the stars threw down their spears And water'd heaven with their tear Did he smile his work to see ? Did he, who made the lamb, made thee ?", + "Tiger, tiger burning bright In the forest of the night What immortal hand or eye Dare frame thy fearful symmetry ?", + "Cruelty has a human heart And jealousy a human face Terror the human form divine And Secrecy the human dress .", + "The human dress is forg'd iron The human form a fiery forge The human face a furnace seal'd The human heart its fiery gorge ." + }; + + /** + * Tests whether all similarities return three documents for the query word + * "heart". + */ + public void testHeartList() throws IOException { + Query q = new TermQuery(new Term(FIELD_BODY, "heart")); + + for (EasySimilarity sim : sims) { + searcher.setSimilarityProvider(new EasySimilarityProvider(sim)); + TopDocs topDocs = searcher.search(q, 1000); + assertEquals("Failed: " + sim.toString(), 3, topDocs.totalHits); + } + } + + /** Test whether all similarities return document 3 before documents 7 and 8. */ + public void testHeartRanking() throws IOException { + Query q = new TermQuery(new Term(FIELD_BODY, "heart")); + + for (EasySimilarity sim : sims) { + searcher.setSimilarityProvider(new EasySimilarityProvider(sim)); + TopDocs topDocs = searcher.search(q, 1000); + assertEquals("Failed: " + sim.toString(), 2, topDocs.scoreDocs[0].doc); + } + } + + @Override + public void tearDown() throws Exception { + searcher.close(); + reader.close(); + dir.close(); + super.tearDown(); + } + + // ------------------------- Helper class definitions ------------------------ + + /** + * A simple Similarity provider that returns in {@code get(String field)} the + * object passed to its constructor. + */ + // nocommit a real EasySimilarityProvider with coord and queryNorm implemented? + public static class EasySimilarityProvider implements SimilarityProvider { + private EasySimilarity sim; + + public EasySimilarityProvider(EasySimilarity sim) { + this.sim = sim; + } + + @Override + public float coord(int overlap, int maxOverlap) { + return 1f; + } + + @Override + public float queryNorm(float sumOfSquaredWeights) { + return 1f; + } + + @Override + public EasySimilarity get(String field) { + return sim; + } + } +}