Index: lucene/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java (revision 1163308) +++ lucene/src/test/org/apache/lucene/search/TestSloppyPhraseQuery.java (working copy) @@ -25,8 +25,10 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.store.Directory; public class TestSloppyPhraseQuery extends LuceneTestCase { @@ -116,12 +118,16 @@ query.setSlop(slop); Directory ramDir = newDirectory(); - RandomIndexWriter writer = new RandomIndexWriter(random, ramDir, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)); + IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)); + // TODO: generalize tests to not rely upon how the default sim works + iwc.setSimilarityProvider(new DefaultSimilarityProvider()); + RandomIndexWriter writer = new RandomIndexWriter(random, ramDir, iwc); writer.addDocument(doc); IndexReader reader = writer.getReader(); IndexSearcher searcher = newSearcher(reader); + searcher.setSimilarityProvider(new DefaultSimilarityProvider()); TopDocs td = searcher.search(query,null,10); //System.out.println("slop: "+slop+" query: "+query+" doc: "+doc+" Expecting number of hits: "+expectedNumResults+" maxScore="+td.getMaxScore()); assertEquals("slop: "+slop+" query: "+query+" doc: "+doc+" Wrong number of hits", expectedNumResults, td.totalHits); Index: lucene/src/test/org/apache/lucene/search/TestSort.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestSort.java (revision 1163308) +++ lucene/src/test/org/apache/lucene/search/TestSort.java (working copy) @@ -86,6 +86,7 @@ @BeforeClass public static void beforeClass() throws Exception { NUM_STRINGS = atLeast(6000); + setNeedsIndexTimeBoost(); } // document data: // the tracer field is used to determine which document was hit Index: lucene/src/test/org/apache/lucene/search/TestExplanations.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestExplanations.java (revision 1163308) +++ lucene/src/test/org/apache/lucene/search/TestExplanations.java (working copy) @@ -71,6 +71,7 @@ @BeforeClass public static void beforeClassTestExplanations() throws Exception { + setNeedsIndexTimeBoost(); directory = newDirectory(); RandomIndexWriter writer= new RandomIndexWriter(random, directory, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); for (int i = 0; i < docFields.length; i++) { Index: lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java (revision 1163308) +++ lucene/src/test/org/apache/lucene/search/TestMatchAllDocsQuery.java (working copy) @@ -29,15 +29,21 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; +import org.junit.BeforeClass; /** * Tests MatchAllDocsQuery. * */ public class TestMatchAllDocsQuery extends LuceneTestCase { - private Analyzer analyzer = new MockAnalyzer(random); - + + @BeforeClass + public static void beforeClass() throws Exception { + setNeedsIndexTimeBoost(); + } + public void testQuery() throws Exception { + Analyzer analyzer = new MockAnalyzer(random); Directory dir = newDirectory(); IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer).setMaxBufferedDocs(2).setMergePolicy(newLogMergePolicy())); Index: lucene/src/test/org/apache/lucene/search/TestDocBoost.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestDocBoost.java (revision 1163308) +++ lucene/src/test/org/apache/lucene/search/TestDocBoost.java (working copy) @@ -27,13 +27,18 @@ import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; +import org.junit.BeforeClass; /** Document boost unit test. * * */ public class TestDocBoost extends LuceneTestCase { - + @BeforeClass + public static void beforeClass() throws Exception { + setNeedsIndexTimeBoost(); + } + public void testDocBoost() throws Exception { Directory store = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random, store, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy())); Index: lucene/src/test/org/apache/lucene/index/TestIndexableField.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexableField.java (revision 1163308) +++ lucene/src/test/org/apache/lucene/index/TestIndexableField.java (working copy) @@ -38,9 +38,15 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import org.junit.BeforeClass; public class TestIndexableField extends LuceneTestCase { + @BeforeClass + public static void beforeClass() throws Exception { + setNeedsIndexTimeBoost(); + } + private class MyField implements IndexableField { private final int counter; Index: lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java =================================================================== --- lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (revision 1163308) +++ lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (working copy) @@ -75,6 +75,8 @@ * In these cases boosting a document higher may actually have the reverse effect, * so subclasses can return false here so that the user will get an error instead. */ + // TODO: should we bother? index-time boosting sometimes doesnt work as a user + // might expect with the default sim too even, because of floating point quantization. protected boolean supportsIndexTimeBoost() { return true; } Index: lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java (revision 1163309) +++ lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java (working copy) @@ -18,39 +18,14 @@ */ import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Random; -import org.apache.lucene.search.similarities.AfterEffect; -import org.apache.lucene.search.similarities.AfterEffectB; -import org.apache.lucene.search.similarities.AfterEffectL; -import org.apache.lucene.search.similarities.BM25Similarity; -import org.apache.lucene.search.similarities.BasicModel; -import org.apache.lucene.search.similarities.BasicModelBE; -import org.apache.lucene.search.similarities.BasicModelD; -import org.apache.lucene.search.similarities.BasicModelG; -import org.apache.lucene.search.similarities.BasicModelIF; -import org.apache.lucene.search.similarities.BasicModelIn; -import org.apache.lucene.search.similarities.BasicModelIne; -import org.apache.lucene.search.similarities.BasicModelP; -import org.apache.lucene.search.similarities.DFRSimilarity; -import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.DefaultSimilarityProvider; -import org.apache.lucene.search.similarities.Distribution; -import org.apache.lucene.search.similarities.DistributionLL; -import org.apache.lucene.search.similarities.DistributionSPL; -import org.apache.lucene.search.similarities.IBSimilarity; -import org.apache.lucene.search.similarities.LMDirichletSimilarity; -import org.apache.lucene.search.similarities.LMJelinekMercerSimilarity; -import org.apache.lucene.search.similarities.Lambda; -import org.apache.lucene.search.similarities.LambdaDF; -import org.apache.lucene.search.similarities.LambdaTTF; -import org.apache.lucene.search.similarities.Normalization; -import org.apache.lucene.search.similarities.NormalizationH1; -import org.apache.lucene.search.similarities.NormalizationH2; import org.apache.lucene.search.similarities.Similarity; public class RandomSimilarityProvider extends DefaultSimilarityProvider { @@ -60,11 +35,11 @@ final boolean shouldCoord; final boolean shouldQueryNorm; - public RandomSimilarityProvider(Random random) { + public RandomSimilarityProvider(Random random, Collection sims) { perFieldSeed = random.nextInt(); shouldCoord = random.nextBoolean(); shouldQueryNorm = random.nextBoolean(); - knownSims = new ArrayList(allSims); + knownSims = new ArrayList(sims); Collections.shuffle(knownSims, random); } @@ -97,58 +72,6 @@ return sim; } - // all the similarities that we rotate through - /** The DFR basic models to test. */ - static BasicModel[] BASIC_MODELS = { - new BasicModelBE(), new BasicModelD(), new BasicModelG(), - new BasicModelIF(), new BasicModelIn(), new BasicModelIne(), - new BasicModelP() - }; - /** The DFR aftereffects to test. */ - static AfterEffect[] AFTER_EFFECTS = { - new AfterEffectB(), new AfterEffectL(), new AfterEffect.NoAfterEffect() - }; - /** The DFR normalizations to test. */ - static Normalization[] NORMALIZATIONS = { - new NormalizationH1(), new NormalizationH2(), - // TODO: if we enable NoNormalization, we have to deal with - // a couple tests (e.g. TestDocBoost, TestSort) that expect length normalization - // new Normalization.NoNormalization() - }; - /** The distributions for IB. */ - static Distribution[] DISTRIBUTIONS = { - new DistributionLL(), new DistributionSPL() - }; - /** Lambdas for IB. */ - static Lambda[] LAMBDAS = { - new LambdaDF(), new LambdaTTF() - }; - static List allSims; - static { - allSims = new ArrayList(); - allSims.add(new DefaultSimilarity()); - allSims.add(new BM25Similarity()); - /* TODO: enable DFR sims - for (BasicModel basicModel : BASIC_MODELS) { - for (AfterEffect afterEffect : AFTER_EFFECTS) { - for (Normalization normalization : NORMALIZATIONS) { - allSims.add(new DFRSimilarity(basicModel, afterEffect, normalization)); - } - } - } */ - for (Distribution distribution : DISTRIBUTIONS) { - for (Lambda lambda : LAMBDAS) { - for (Normalization normalization : NORMALIZATIONS) { - allSims.add(new IBSimilarity(distribution, lambda, normalization)); - } - } - } - /* TODO: enable Dirichlet - allSims.add(new LMDirichletSimilarity()); */ - allSims.add(new LMJelinekMercerSimilarity(0.1f)); - allSims.add(new LMJelinekMercerSimilarity(0.7f)); - } - @Override public synchronized String toString() { return "RandomSimilarityProvider(queryNorm=" + shouldQueryNorm + ",coord=" + shouldCoord + "): " + previousMappings.toString(); Index: lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (revision 1163308) +++ lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -53,7 +53,7 @@ import org.apache.lucene.search.AssertingIndexSearcher; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.RandomSimilarityProvider; -import org.apache.lucene.search.similarities.SimilarityProvider; +import org.apache.lucene.search.similarities.*; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.FlushInfo; @@ -397,7 +397,7 @@ savedTimeZone = TimeZone.getDefault(); timeZone = TEST_TIMEZONE.equals("random") ? randomTimeZone(random) : TimeZone.getTimeZone(TEST_TIMEZONE); TimeZone.setDefault(timeZone); - similarityProvider = new RandomSimilarityProvider(random); + similarityProvider = new RandomSimilarityProvider(random, allSims); testsFailed = false; } @@ -1045,6 +1045,70 @@ logmp.setMergeFactor(mergeFactor); return logmp; } + + // all the similarities that we rotate through + /** The DFR basic models to test. */ + static BasicModel[] BASIC_MODELS = { + new BasicModelBE(), new BasicModelD(), new BasicModelG(), + new BasicModelIF(), new BasicModelIn(), new BasicModelIne(), + new BasicModelP() + }; + /** The DFR aftereffects to test. */ + static AfterEffect[] AFTER_EFFECTS = { + new AfterEffectB(), new AfterEffectL(), new AfterEffect.NoAfterEffect() + }; + /** The DFR normalizations to test. */ + static Normalization[] NORMALIZATIONS = { + new NormalizationH1(), new NormalizationH2(), + // TODO: if we enable NoNormalization, we have to deal with + // a couple tests (e.g. TestDocBoost, TestSort) that expect length normalization + // new Normalization.NoNormalization() + }; + /** The distributions for IB. */ + static Distribution[] DISTRIBUTIONS = { + new DistributionLL(), new DistributionSPL() + }; + /** Lambdas for IB. */ + static Lambda[] LAMBDAS = { + new LambdaDF(), new LambdaTTF() + }; + public static List allSims; + public static List boostableSims; + static { + allSims = new ArrayList(); + allSims.add(new DefaultSimilarity()); + allSims.add(new BM25Similarity()); + for (BasicModel basicModel : BASIC_MODELS) { + for (AfterEffect afterEffect : AFTER_EFFECTS) { + for (Normalization normalization : NORMALIZATIONS) { + allSims.add(new DFRSimilarity(basicModel, afterEffect, normalization)); + } + } + } + for (Distribution distribution : DISTRIBUTIONS) { + for (Lambda lambda : LAMBDAS) { + for (Normalization normalization : NORMALIZATIONS) { + allSims.add(new IBSimilarity(distribution, lambda, normalization)); + } + } + } + /* TODO: enable Dirichlet + allSims.add(new LMDirichletSimilarity()); */ + allSims.add(new LMJelinekMercerSimilarity(0.1f)); + allSims.add(new LMJelinekMercerSimilarity(0.7f)); + boostableSims = new ArrayList(); + for (Similarity sim : allSims) { + if (!(sim instanceof DFRSimilarity)) { + boostableSims.add(sim); + } + } + } + + /** if your test uses index-time boosting, set this in a beforeClass annotated method + * before creating any indexwriters etc. */ + public static void setNeedsIndexTimeBoost() { + similarityProvider = new RandomSimilarityProvider(random, boostableSims); + } /** * Returns a new Directory instance. Use this when the test does not