Index: solr/src/java/org/apache/solr/search/SolrSimilarity.java =================================================================== --- solr/src/java/org/apache/solr/search/SolrSimilarity.java (revision 1056696) +++ solr/src/java/org/apache/solr/search/SolrSimilarity.java (working copy) @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.search; - -import org.apache.lucene.search.DefaultSimilarity; - -import java.util.HashMap; - -/** - */ -// don't make it public for now... easier to change later. - -// This class is currently unused. -class SolrSimilarity extends DefaultSimilarity { - private final HashMap lengthNormConfig = new HashMap(); - - public float lengthNorm(String fieldName, int numTerms) { - // Float f = lengthNormConfig. - // if (lengthNormDisabled.) - return super.lengthNorm(fieldName, numTerms); - } -} Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1056696) +++ lucene/CHANGES.txt (working copy) @@ -81,6 +81,9 @@ Searchable are collapsed into IndexSearcher; contrib/remote and MultiSearcher have been removed. (Mike McCandless) +* LUCENE-2854: Deprecated SimilarityDelegator and + Similarity.lengthNorm. (Robert Muir, Mike McCandless) + Changes in runtime behavior * LUCENE-1923: Made IndexReader.toString() produce something Index: lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (revision 1056696) +++ lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (working copy) @@ -26,6 +26,7 @@ import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Payload; import org.apache.lucene.index.RandomIndexWriter; @@ -306,8 +307,8 @@ //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! //Make everything else 1 so we see the effect of the payload //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - @Override public float lengthNorm(String fieldName, int numTerms) { - return 1.0f; + @Override public float computeNorm(String fieldName, FieldInvertState state) { + return state.getBoost(); } @Override public float queryNorm(float sumOfSquaredWeights) { Index: lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (revision 1056696) +++ lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java (working copy) @@ -34,6 +34,7 @@ import org.apache.lucene.analysis.LowerCaseTokenizer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Payload; import org.apache.lucene.index.RandomIndexWriter; @@ -298,8 +299,8 @@ //Make everything else 1 so we see the effect of the payload //!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! @Override - public float lengthNorm(String fieldName, int numTerms) { - return 1; + public float computeNorm(String fieldName, FieldInvertState state) { + return state.getBoost(); } @Override Index: lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java (revision 1056696) +++ lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java (working copy) @@ -83,7 +83,7 @@ } }}; - BooleanScorer bs = new BooleanScorer(null, sim, 1, Arrays.asList(scorers), null, scorers.length); + BooleanScorer bs = new BooleanScorer(null, false, sim, 1, Arrays.asList(scorers), null, scorers.length); assertEquals("should have received 3000", 3000, bs.nextDoc()); assertEquals("should have received NO_MORE_DOCS", DocIdSetIterator.NO_MORE_DOCS, bs.nextDoc()); Index: lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java (revision 1056696) +++ lucene/src/test/org/apache/lucene/search/TestMultiSearcher.java (working copy) @@ -23,6 +23,8 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.SetBasedFieldSelector; + +import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -379,7 +381,7 @@ @Override public float coord(int overlap, int maxOverlap) { return 1.0f; } @Override - public float lengthNorm(String fieldName, int numTokens) { return 1.0f; } + public float computeNorm(String fieldName, FieldInvertState state) { return state.getBoost(); } @Override public float queryNorm(float sumOfSquaredWeights) { return 1.0f; } @Override Index: lucene/src/test/org/apache/lucene/search/JustCompileSearch.java =================================================================== --- lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (revision 1056696) +++ lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.Term; import org.apache.lucene.util.PriorityQueue; @@ -384,7 +385,7 @@ } @Override - public float lengthNorm(String fieldName, int numTokens) { + public float computeNorm(String fieldName, FieldInvertState state) { throw new UnsupportedOperationException(UNSUPPORTED_MSG); } Index: lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (revision 1056696) +++ lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (working copy) @@ -22,6 +22,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.store.Directory; @@ -58,8 +59,9 @@ } @Override - public float lengthNorm(String fieldName, int numTerms) { - return 1.0f; + public float computeNorm(String fieldName, FieldInvertState state) { + // Disable length norm + return state.getBoost(); } @Override Index: lucene/src/test/org/apache/lucene/search/TestSimilarity.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestSimilarity.java (revision 1056696) +++ lucene/src/test/org/apache/lucene/search/TestSimilarity.java (working copy) @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Collection; +import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; @@ -38,7 +39,7 @@ public class TestSimilarity extends LuceneTestCase { public static class SimpleSimilarity extends Similarity { - @Override public float lengthNorm(String field, int numTerms) { return 1.0f; } + @Override public float computeNorm(String field, FieldInvertState state) { return state.getBoost(); } @Override public float queryNorm(float sumOfSquaredWeights) { return 1.0f; } @Override public float tf(float freq) { return freq; } @Override public float sloppyFreq(int distance) { return 2.0f; } Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1056696) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -44,6 +44,8 @@ import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.SimilarityDelegator; import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; Index: lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (revision 1056696) +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (working copy) @@ -30,6 +30,7 @@ import org.apache.lucene.document.Field.Store; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.SegmentReader.Norm; +import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.Similarity; import org.apache.lucene.store.Directory; @@ -42,8 +43,9 @@ private class SimilarityOne extends DefaultSimilarity { @Override - public float lengthNorm(String fieldName, int numTerms) { - return 1; + public float computeNorm(String fieldName, FieldInvertState state) { + // diable length norm + return state.getBoost(); } } Index: lucene/src/test/org/apache/lucene/index/TestOmitTf.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestOmitTf.java (revision 1056696) +++ lucene/src/test/org/apache/lucene/index/TestOmitTf.java (working copy) @@ -41,7 +41,7 @@ public class TestOmitTf extends LuceneTestCase { public static class SimpleSimilarity extends Similarity { - @Override public float lengthNorm(String field, int numTerms) { return 1.0f; } + @Override public float computeNorm(String field, FieldInvertState state) { return state.getBoost(); } @Override public float queryNorm(float sumOfSquaredWeights) { return 1.0f; } @Override public float tf(float freq) { return freq; } @Override public float sloppyFreq(int distance) { return 2.0f; } Index: lucene/src/test/org/apache/lucene/index/TestNorms.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestNorms.java (revision 1056696) +++ lucene/src/test/org/apache/lucene/index/TestNorms.java (working copy) @@ -41,8 +41,9 @@ private class SimilarityOne extends DefaultSimilarity { @Override - public float lengthNorm(String fieldName, int numTerms) { - return 1; + public float computeNorm(String fieldName, FieldInvertState state) { + // Disable length norm + return state.getBoost(); } } Index: lucene/src/java/org/apache/lucene/search/BooleanScorer2.java =================================================================== --- lucene/src/java/org/apache/lucene/search/BooleanScorer2.java (revision 1056696) +++ lucene/src/java/org/apache/lucene/search/BooleanScorer2.java (working copy) @@ -42,14 +42,12 @@ int maxCoord = 0; // to be increased for each non prohibited scorer int nrMatchers; // to be increased by score() of match counting scorers. - void init() { // use after all scorers have been added. + void init(Similarity sim, boolean disableCoord) { // use after all scorers have been added. coordFactors = new float[optionalScorers.size() + requiredScorers.size() + 1]; - Similarity sim = getSimilarity(); for (int i = 0; i < coordFactors.length; i++) { - coordFactors[i] = sim.coord(i, maxCoord); + coordFactors[i] = disableCoord ? 1.0f : sim.coord(i, maxCoord); } } - } private final Coordinator coordinator; @@ -82,9 +80,9 @@ * @param optional * the list of optional scorers. */ - public BooleanScorer2(Weight weight, Similarity similarity, int minNrShouldMatch, + public BooleanScorer2(Weight weight, boolean disableCoord, Similarity similarity, int minNrShouldMatch, List required, List prohibited, List optional, int maxCoord) throws IOException { - super(similarity, weight); + super(null, weight); // Similarity not used if (minNrShouldMatch < 0) { throw new IllegalArgumentException("Minimum number of optional scorers should not be negative"); } @@ -96,8 +94,8 @@ requiredScorers = required; prohibitedScorers = prohibited; - coordinator.init(); - countingSumScorer = makeCountingSumScorer(); + coordinator.init(similarity, disableCoord); + countingSumScorer = makeCountingSumScorer(disableCoord, similarity); } /** Count a scorer as a single match. */ @@ -109,7 +107,7 @@ private float lastDocScore = Float.NaN; SingleMatchScorer(Scorer scorer) { - super(scorer.getSimilarity()); + super(null); // No similarity used. this.scorer = scorer; } @@ -164,12 +162,12 @@ }; } - private static final Similarity defaultSimilarity = Similarity.getDefault(); - - private Scorer countingConjunctionSumScorer(List requiredScorers) throws IOException { + private Scorer countingConjunctionSumScorer(boolean disableCoord, + Similarity similarity, + List requiredScorers) throws IOException { // each scorer from the list counted as a single matcher final int requiredNrMatchers = requiredScorers.size(); - return new ConjunctionScorer(defaultSimilarity, requiredScorers) { + return new ConjunctionScorer(disableCoord ? 1.0f : similarity.coord(requiredScorers.size(), requiredScorers.size()), requiredScorers) { private int lastScoredDoc = -1; // Save the score of lastScoredDoc, so that we don't compute it more than // once in score(). @@ -192,8 +190,10 @@ }; } - private Scorer dualConjunctionSumScorer(Scorer req1, Scorer req2) throws IOException { // non counting. - return new ConjunctionScorer(defaultSimilarity, new Scorer[]{req1, req2}); + private Scorer dualConjunctionSumScorer(boolean disableCoord, + Similarity similarity, + Scorer req1, Scorer req2) throws IOException { // non counting. + return new ConjunctionScorer(disableCoord ? 1.0f : similarity.coord(2, 2), new Scorer[]{req1, req2}); // All scorers match, so defaultSimilarity always has 1 as // the coordination factor. // Therefore the sum of the scores of two scorers @@ -203,13 +203,14 @@ /** Returns the scorer to be used for match counting and score summing. * Uses requiredScorers, optionalScorers and prohibitedScorers. */ - private Scorer makeCountingSumScorer() throws IOException { // each scorer counted as a single matcher + private Scorer makeCountingSumScorer(boolean disableCoord, + Similarity similarity) throws IOException { // each scorer counted as a single matcher return (requiredScorers.size() == 0) - ? makeCountingSumScorerNoReq() - : makeCountingSumScorerSomeReq(); + ? makeCountingSumScorerNoReq(disableCoord, similarity) + : makeCountingSumScorerSomeReq(disableCoord, similarity); } - private Scorer makeCountingSumScorerNoReq() throws IOException { // No required scorers + private Scorer makeCountingSumScorerNoReq(boolean disableCoord, Similarity similarity) throws IOException { // No required scorers // minNrShouldMatch optional scorers are required, but at least 1 int nrOptRequired = (minNrShouldMatch < 1) ? 1 : minNrShouldMatch; Scorer requiredCountingSumScorer; @@ -217,24 +218,27 @@ requiredCountingSumScorer = countingDisjunctionSumScorer(optionalScorers, nrOptRequired); else if (optionalScorers.size() == 1) requiredCountingSumScorer = new SingleMatchScorer(optionalScorers.get(0)); - else - requiredCountingSumScorer = countingConjunctionSumScorer(optionalScorers); + else { + requiredCountingSumScorer = countingConjunctionSumScorer(disableCoord, similarity, optionalScorers); + } return addProhibitedScorers(requiredCountingSumScorer); } - private Scorer makeCountingSumScorerSomeReq() throws IOException { // At least one required scorer. + private Scorer makeCountingSumScorerSomeReq(boolean disableCoord, Similarity similarity) throws IOException { // At least one required scorer. if (optionalScorers.size() == minNrShouldMatch) { // all optional scorers also required. ArrayList allReq = new ArrayList(requiredScorers); allReq.addAll(optionalScorers); - return addProhibitedScorers(countingConjunctionSumScorer(allReq)); + return addProhibitedScorers(countingConjunctionSumScorer(disableCoord, similarity, allReq)); } else { // optionalScorers.size() > minNrShouldMatch, and at least one required scorer Scorer requiredCountingSumScorer = requiredScorers.size() == 1 ? new SingleMatchScorer(requiredScorers.get(0)) - : countingConjunctionSumScorer(requiredScorers); + : countingConjunctionSumScorer(disableCoord, similarity, requiredScorers); if (minNrShouldMatch > 0) { // use a required disjunction scorer over the optional scorers return addProhibitedScorers( dualConjunctionSumScorer( // non counting + disableCoord, + similarity, requiredCountingSumScorer, countingDisjunctionSumScorer( optionalScorers, Index: lucene/src/java/org/apache/lucene/search/Similarity.java =================================================================== --- lucene/src/java/org/apache/lucene/search/Similarity.java (revision 1056696) +++ lucene/src/java/org/apache/lucene/search/Similarity.java (working copy) @@ -462,12 +462,14 @@ * {@link org.apache.lucene.document.Fieldable#setBoost(float) field.setBoost()} * before adding the field to a document. * - *
  • {@link #lengthNorm(String, int) lengthNorm(field)} - computed + *
  • lengthNorm - computed * when the document is added to the index in accordance with the number of tokens * of this field in the document, so that shorter fields contribute more to the score. * LengthNorm is computed by the Similarity class in effect at indexing. *
    • * + * The {@link #computeNorm} method is responsible for + * combining all of these factors into a single float. * *

    * When a document is added to the index, all the above factors are multiplied. @@ -480,7 +482,7 @@ * norm(t,d)   =   * {@link org.apache.lucene.document.Document#getBoost() doc.getBoost()} *  ·  - * {@link #lengthNorm(String, int) lengthNorm(field)} + * lengthNorm *  ·  * * @@ -591,12 +593,23 @@ } /** - * Compute the normalization value for a field, given the accumulated + * Computes the normalization value for a field, given the accumulated * state of term processing for this field (see {@link FieldInvertState}). * *

    Implementations should calculate a float value based on the field * state and then return that value. * + *

    Matches in longer fields are less precise, so implementations of this + * method usually return smaller values when state.getLength() is large, + * and larger values when state.getLength() is small. + * + *

    Note that the return values are computed under + * {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document)} + * and then stored using + * {@link #encodeNormValue(float)}. + * Thus they have limited precision, and documents + * must be re-indexed if this method is altered. + * *

    For backward compatibility this method by default calls * {@link #lengthNorm(String, int)} passing * {@link FieldInvertState#getLength()} as the second argument, and @@ -608,9 +621,7 @@ * @param state current processing state for this field * @return the calculated float norm */ - public float computeNorm(String field, FieldInvertState state) { - return (state.getBoost() * lengthNorm(field, state.getLength())); - } + public abstract float computeNorm(String field, FieldInvertState state); /** Computes the normalization value for a field given the total number of * terms contained in a field. These values, together with field boosts, are @@ -634,8 +645,14 @@ * @return a normalization factor for hits on this field of this document * * @see org.apache.lucene.document.Field#setBoost(float) + * + * @deprecated Please override computeNorm instead */ - public abstract float lengthNorm(String fieldName, int numTokens); + @Deprecated + // nocommit -- try removing this & make sure all compiles + public final float lengthNorm(String fieldName, int numTokens) { + throw new UnsupportedOperationException("please use computeNorm instead"); + } /** Computes the normalization value for a query given the sum of the squared * weights of each of the query terms. This value is multiplied into the Index: lucene/src/java/org/apache/lucene/search/ConjunctionScorer.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ConjunctionScorer.java (revision 1056696) +++ lucene/src/java/org/apache/lucene/search/ConjunctionScorer.java (working copy) @@ -29,14 +29,14 @@ private final float coord; private int lastDoc = -1; - public ConjunctionScorer(Similarity similarity, Collection scorers) throws IOException { - this(similarity, scorers.toArray(new Scorer[scorers.size()])); + public ConjunctionScorer(float coord, Collection scorers) throws IOException { + this(coord, scorers.toArray(new Scorer[scorers.size()])); } - public ConjunctionScorer(Similarity similarity, Scorer... scorers) throws IOException { - super(similarity); + public ConjunctionScorer(float coord, Scorer... scorers) throws IOException { + super(null); this.scorers = scorers; - coord = similarity.coord(scorers.length, scorers.length); + this.coord = coord; for (int i = 0; i < scorers.length; i++) { if (scorers[i].nextDoc() == NO_MORE_DOCS) { Index: lucene/src/java/org/apache/lucene/search/SimilarityDelegator.java =================================================================== --- lucene/src/java/org/apache/lucene/search/SimilarityDelegator.java (revision 1056696) +++ lucene/src/java/org/apache/lucene/search/SimilarityDelegator.java (working copy) @@ -21,7 +21,10 @@ /** Expert: Delegating scoring implementation. Useful in {@link * Query#getSimilarity(Searcher)} implementations, to override only certain - * methods of a Searcher's Similarity implementation.. */ + * methods of a Searcher's Similarity implementation.. + * @deprecated this class will be removed in 4.0. Please + * subclass {@link Similarity} or {@link DefaultSimilarity} instead. */ +@Deprecated public class SimilarityDelegator extends Similarity { private Similarity delegee; @@ -38,13 +41,8 @@ public float computeNorm(String fieldName, FieldInvertState state) { return delegee.computeNorm(fieldName, state); } - + @Override - public float lengthNorm(String fieldName, int numTerms) { - return delegee.lengthNorm(fieldName, numTerms); - } - - @Override public float queryNorm(float sumOfSquaredWeights) { return delegee.queryNorm(sumOfSquaredWeights); } Index: lucene/src/java/org/apache/lucene/search/BooleanQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/BooleanQuery.java (revision 1056696) +++ lucene/src/java/org/apache/lucene/search/BooleanQuery.java (working copy) @@ -62,10 +62,12 @@ } private ArrayList clauses = new ArrayList(); - private boolean disableCoord; + private final boolean disableCoord; /** Constructs an empty boolean query. */ - public BooleanQuery() {} + public BooleanQuery() { + disableCoord = false; + } /** Constructs an empty boolean query. * @@ -86,22 +88,6 @@ */ public boolean isCoordDisabled() { return disableCoord; } - // Implement coord disabling. - // Inherit javadoc. - @Override - public Similarity getSimilarity(Searcher searcher) { - Similarity result = super.getSimilarity(searcher); - if (disableCoord) { // disable coord as requested - result = new SimilarityDelegator(result) { - @Override - public float coord(int overlap, int maxOverlap) { - return 1.0f; - } - }; - } - return result; - } - /** * Specifies a minimum number of the optional BooleanClauses * which must be satisfied. @@ -178,10 +164,12 @@ protected Similarity similarity; protected ArrayList weights; protected int maxCoord; // num optional + num required + private final boolean disableCoord; - public BooleanWeight(Searcher searcher) + public BooleanWeight(Searcher searcher, boolean disableCoord) throws IOException { this.similarity = getSimilarity(searcher); + this.disableCoord = disableCoord; weights = new ArrayList(clauses.size()); for (int i = 0 ; i < clauses.size(); i++) { BooleanClause c = clauses.get(i); @@ -284,10 +272,10 @@ sumExpl.setMatch(0 < coord ? Boolean.TRUE : Boolean.FALSE); sumExpl.setValue(sum); - float coordFactor = similarity.coord(coord, maxCoord); - if (coordFactor == 1.0f) // coord is no-op + final float coordFactor = disableCoord ? 1.0f : similarity.coord(coord, maxCoord); + if (coordFactor == 1.0f) { return sumExpl; // eliminate wrapper - else { + } else { ComplexExplanation result = new ComplexExplanation(sumExpl.isMatch(), sum*coordFactor, "product of:"); @@ -323,7 +311,7 @@ // Check if we can return a BooleanScorer if (!scoreDocsInOrder && topScorer && required.size() == 0 && prohibited.size() < 32) { - return new BooleanScorer(this, similarity, minNrShouldMatch, optional, prohibited, maxCoord); + return new BooleanScorer(this, disableCoord, similarity, minNrShouldMatch, optional, prohibited, maxCoord); } if (required.size() == 0 && optional.size() == 0) { @@ -337,7 +325,7 @@ } // Return a BooleanScorer2 - return new BooleanScorer2(this, similarity, minNrShouldMatch, required, prohibited, optional, maxCoord); + return new BooleanScorer2(this, disableCoord, similarity, minNrShouldMatch, required, prohibited, optional, maxCoord); } @Override @@ -363,7 +351,7 @@ @Override public Weight createWeight(Searcher searcher) throws IOException { - return new BooleanWeight(searcher); + return new BooleanWeight(searcher, disableCoord); } @Override Index: lucene/src/java/org/apache/lucene/search/BooleanScorer.java =================================================================== --- lucene/src/java/org/apache/lucene/search/BooleanScorer.java (revision 1056696) +++ lucene/src/java/org/apache/lucene/search/BooleanScorer.java (working copy) @@ -197,9 +197,9 @@ private Bucket current; private int doc = -1; - BooleanScorer(Weight weight, Similarity similarity, int minNrShouldMatch, + BooleanScorer(Weight weight, boolean disableCoord, Similarity similarity, int minNrShouldMatch, List optionalScorers, List prohibitedScorers, int maxCoord) throws IOException { - super(similarity, weight); + super(null, weight); // Similarity not used this.minNrShouldMatch = minNrShouldMatch; if (optionalScorers != null && optionalScorers.size() > 0) { @@ -222,9 +222,8 @@ } coordFactors = new float[optionalScorers.size() + 1]; - Similarity sim = getSimilarity(); for (int i = 0; i < coordFactors.length; i++) { - coordFactors[i] = sim.coord(i, maxCoord); + coordFactors[i] = disableCoord ? 1.0f : similarity.coord(i, maxCoord); } } Index: lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java =================================================================== --- lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java (revision 1056696) +++ lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java (working copy) @@ -37,15 +37,9 @@ numTerms = state.getLength() - state.getNumOverlap(); else numTerms = state.getLength(); - return (state.getBoost() * lengthNorm(field, numTerms)); + return state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms))); } - /** Implemented as 1/sqrt(numTerms). */ - @Override - public float lengthNorm(String fieldName, int numTerms) { - return (float)(1.0 / Math.sqrt(numTerms)); - } - /** Implemented as 1/sqrt(sumOfSquaredWeights). */ @Override public float queryNorm(float sumOfSquaredWeights) { Index: lucene/src/java/org/apache/lucene/index/FieldInvertState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInvertState.java (revision 1056696) +++ lucene/src/java/org/apache/lucene/index/FieldInvertState.java (working copy) @@ -73,6 +73,10 @@ return length; } + public void setLength(int length) { + this.length = length; + } + /** * Get the number of terms with positionIncrement == 0. * @return the numOverlap @@ -81,6 +85,10 @@ return numOverlap; } + public void setNumOverlap(int numOverlap) { + this.numOverlap = numOverlap; + } + /** * Get end offset of the last processed term. * @return the offset @@ -99,6 +107,10 @@ return boost; } + public void setBoost(float boost) { + this.boost = boost; + } + public AttributeSource getAttributeSource() { return attributeSource; } Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java =================================================================== --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (revision 1056696) +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (working copy) @@ -38,6 +38,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Document; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; @@ -238,9 +239,10 @@ termsInDocument += eFieldTermDocInfoFactoriesByTermText.getValue().size(); if (eFieldTermDocInfoFactoriesByTermText.getKey().indexed && !eFieldTermDocInfoFactoriesByTermText.getKey().omitNorms) { - float norm = eFieldTermDocInfoFactoriesByTermText.getKey().boost; - norm *= document.getDocument().getBoost(); - norm *= similarity.lengthNorm(eFieldTermDocInfoFactoriesByTermText.getKey().fieldName, eFieldTermDocInfoFactoriesByTermText.getKey().fieldLength); + final FieldInvertState invertState = new FieldInvertState(); + invertState.setBoost(eFieldTermDocInfoFactoriesByTermText.getKey().boost * document.getDocument().getBoost()); + invertState.setLength(eFieldTermDocInfoFactoriesByTermText.getKey().fieldLength); + final float norm = similarity.computeNorm(eFieldTermDocInfoFactoriesByTermText.getKey().fieldName, invertState); normsByFieldNameAndDocumentNumber.get(eFieldTermDocInfoFactoriesByTermText.getKey().fieldName)[document.getDocumentNumber()] = similarity.encodeNormValue(norm); } else { System.currentTimeMillis(); Index: lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (revision 1056696) +++ lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java (working copy) @@ -43,8 +43,8 @@ /** inverts the normal notion of lengthNorm */ public static Similarity s = new DefaultSimilarity() { @Override - public float lengthNorm(String fieldName, int numTokens) { - return numTokens; + public float computeNorm(String fieldName, FieldInvertState state) { + return state.getBoost() * (discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength()); } }; Index: lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (revision 1056696) +++ lucene/contrib/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (working copy) @@ -21,13 +21,14 @@ import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.Similarity; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.index.FieldInvertState; /** * Test of the SweetSpotSimilarity */ public class SweetSpotSimilarityTest extends LuceneTestCase { - public void testSweetSpotLengthNorm() { + public void testSweetSpotComputeNorm() { SweetSpotSimilarity ss = new SweetSpotSimilarity(); ss.setLengthNormFactors(1,1,0.5f); @@ -37,10 +38,13 @@ // base case, should degrade - + final FieldInvertState invertState = new FieldInvertState(); + invertState.setBoost(1.0f); for (int i = 1; i < 1000; i++) { + invertState.setLength(i); assertEquals("base case: i="+i, - d.lengthNorm("foo",i), s.lengthNorm("foo",i), + d.computeNorm("foo", invertState), + s.computeNorm("foo", invertState), 0.0f); } @@ -49,14 +53,21 @@ ss.setLengthNormFactors(3,10,0.5f); for (int i = 3; i <=10; i++) { + invertState.setLength(i); assertEquals("3,10: spot i="+i, - 1.0f, s.lengthNorm("foo",i), + 1.0f, + s.computeNorm("foo", invertState), 0.0f); } for (int i = 10; i < 1000; i++) { + invertState.setLength(i-9); + final float normD = d.computeNorm("foo", invertState); + invertState.setLength(i); + final float normS = s.computeNorm("foo", invertState); assertEquals("3,10: 10 terms, Searcher searcher) throws IOException { - return new IDFExplanation() { - @Override - public float getIdf() { - return 1.0f; - } - @Override - public String explain() { - return "Inexplicable"; - } - }; - } - } - - public void testSimilarity() throws Exception { - RAMDirectory store = new RAMDirectory(); - IndexWriter writer = new IndexWriter(store, new SimpleAnalyzer(), true, - IndexWriter.MaxFieldLength.LIMITED); - writer.setSimilarity(new SimpleSimilarity()); - - Document d1 = new Document(); - d1.add(new Field("field", "a c", Field.Store.YES, Field.Index.ANALYZED)); - - Document d2 = new Document(); - d2.add(new Field("field", "a b c", Field.Store.YES, Field.Index.ANALYZED)); - - writer.addDocument(d1); - writer.addDocument(d2); - writer.optimize(); - writer.close(); - - Searcher searcher = new IndexSearcher(store, true); - searcher.setSimilarity(new SimpleSimilarity()); - - Term a = new Term("field", "a"); - Term b = new Term("field", "b"); - Term c = new Term("field", "c"); - - searcher.search(new TermQuery(b), new Collector() { - private Scorer scorer; - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - @Override - public final void collect(int doc) throws IOException { - assertEquals(1.0f, scorer.score()); - } - @Override - public void setNextReader(IndexReader reader, int docBase) {} - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - - BooleanQuery bq = new BooleanQuery(); - bq.add(new TermQuery(a), BooleanClause.Occur.SHOULD); - bq.add(new TermQuery(b), BooleanClause.Occur.SHOULD); - //System.out.println(bq.toString("field")); - searcher.search(bq, new Collector() { - private int base = 0; - private Scorer scorer; - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - @Override - public final void collect(int doc) throws IOException { - //System.out.println("Doc=" + doc + " score=" + score); - assertEquals((float)doc+base+1, scorer.score()); - } - @Override - public void setNextReader(IndexReader reader, int docBase) { - base = docBase; - } - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - - PhraseQuery pq = new PhraseQuery(); - pq.add(a); - pq.add(c); - //System.out.println(pq.toString("field")); - searcher.search(pq, - new Collector() { - private Scorer scorer; - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - @Override - public final void collect(int doc) throws IOException { - //System.out.println("Doc=" + doc + " score=" + score); - assertEquals(1.0f, scorer.score()); - } - @Override - public void setNextReader(IndexReader reader, int docBase) {} - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - - pq.setSlop(2); - //System.out.println(pq.toString("field")); - searcher.search(pq, new Collector() { - private Scorer scorer; - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - @Override - public final void collect(int doc) throws IOException { - //System.out.println("Doc=" + doc + " score=" + score); - assertEquals(2.0f, scorer.score()); - } - @Override - public void setNextReader(IndexReader reader, int docBase) {} - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - } -} Index: lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (revision 1056696) +++ lucene/backwards/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (working copy) @@ -42,8 +42,8 @@ private class SimilarityOne extends DefaultSimilarity { @Override - public float lengthNorm(String fieldName, int numTerms) { - return 1; + public float computeNorm(String fieldName, FieldInvertState state) { + return state.getBoost(); } } Index: lucene/backwards/src/test/org/apache/lucene/index/TestOmitTf.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestOmitTf.java (revision 1056696) +++ lucene/backwards/src/test/org/apache/lucene/index/TestOmitTf.java (working copy) @@ -41,27 +41,6 @@ public class TestOmitTf extends LuceneTestCase { - public static class SimpleSimilarity extends Similarity { - @Override public float lengthNorm(String field, int numTerms) { return 1.0f; } - @Override public float queryNorm(float sumOfSquaredWeights) { return 1.0f; } - @Override public float tf(float freq) { return freq; } - @Override public float sloppyFreq(int distance) { return 2.0f; } - @Override public float idf(int docFreq, int numDocs) { return 1.0f; } - @Override public float coord(int overlap, int maxOverlap) { return 1.0f; } - @Override public IDFExplanation idfExplain(Collection terms, Searcher searcher) throws IOException { - return new IDFExplanation() { - @Override - public float getIdf() { - return 1.0f; - } - @Override - public String explain() { - return "Inexplicable"; - } - }; - } - } - // Tests whether the DocumentWriter correctly enable the // omitTermFreqAndPositions bit in the FieldInfo public void testOmitTermFreqAndPositions() throws Exception { @@ -241,150 +220,6 @@ ram.close(); } - // Test scores with one field with Term Freqs and one without, otherwise with equal content - public void testBasic() throws Exception { - Directory dir = new MockRAMDirectory(); - Analyzer analyzer = new StandardAnalyzer(TEST_VERSION_CURRENT); - IndexWriter writer = new IndexWriter(dir, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); - writer.setMergeFactor(2); - writer.setMaxBufferedDocs(2); - writer.setSimilarity(new SimpleSimilarity()); - - - StringBuilder sb = new StringBuilder(265); - String term = "term"; - for(int i = 0; i<30; i++){ - Document d = new Document(); - sb.append(term).append(" "); - String content = sb.toString(); - Field noTf = new Field("noTf", content + (i%2==0 ? "" : " notf"), Field.Store.NO, Field.Index.ANALYZED); - noTf.setOmitTermFreqAndPositions(true); - d.add(noTf); - - Field tf = new Field("tf", content + (i%2==0 ? " tf" : ""), Field.Store.NO, Field.Index.ANALYZED); - d.add(tf); - - writer.addDocument(d); - //System.out.println(d); - } - - writer.optimize(); - // flush - writer.close(); - _TestUtil.checkIndex(dir); - - /* - * Verify the index - */ - Searcher searcher = new IndexSearcher(dir, true); - searcher.setSimilarity(new SimpleSimilarity()); - - Term a = new Term("noTf", term); - Term b = new Term("tf", term); - Term c = new Term("noTf", "notf"); - Term d = new Term("tf", "tf"); - TermQuery q1 = new TermQuery(a); - TermQuery q2 = new TermQuery(b); - TermQuery q3 = new TermQuery(c); - TermQuery q4 = new TermQuery(d); - - - searcher.search(q1, - new CountingHitCollector() { - private Scorer scorer; - @Override - public final void setScorer(Scorer scorer) { - this.scorer = scorer; - } - @Override - public final void collect(int doc) throws IOException { - //System.out.println("Q1: Doc=" + doc + " score=" + score); - float score = scorer.score(); - assertTrue(score==1.0f); - super.collect(doc); - } - }); - //System.out.println(CountingHitCollector.getCount()); - - - searcher.search(q2, - new CountingHitCollector() { - private Scorer scorer; - @Override - public final void setScorer(Scorer scorer) { - this.scorer = scorer; - } - @Override - public final void collect(int doc) throws IOException { - //System.out.println("Q2: Doc=" + doc + " score=" + score); - float score = scorer.score(); - assertTrue(score==1.0f+doc); - super.collect(doc); - } - }); - //System.out.println(CountingHitCollector.getCount()); - - - - - - searcher.search(q3, - new CountingHitCollector() { - private Scorer scorer; - @Override - public final void setScorer(Scorer scorer) { - this.scorer = scorer; - } - @Override - public final void collect(int doc) throws IOException { - //System.out.println("Q1: Doc=" + doc + " score=" + score); - float score = scorer.score(); - assertTrue(score==1.0f); - assertFalse(doc%2==0); - super.collect(doc); - } - }); - //System.out.println(CountingHitCollector.getCount()); - - - searcher.search(q4, - new CountingHitCollector() { - private Scorer scorer; - @Override - public final void setScorer(Scorer scorer) { - this.scorer = scorer; - } - @Override - public final void collect(int doc) throws IOException { - float score = scorer.score(); - //System.out.println("Q1: Doc=" + doc + " score=" + score); - assertTrue(score==1.0f); - assertTrue(doc%2==0); - super.collect(doc); - } - }); - //System.out.println(CountingHitCollector.getCount()); - - - - BooleanQuery bq = new BooleanQuery(); - bq.add(q1,Occur.MUST); - bq.add(q4,Occur.MUST); - - searcher.search(bq, - new CountingHitCollector() { - @Override - public final void collect(int doc) throws IOException { - //System.out.println("BQ: Doc=" + doc + " score=" + score); - super.collect(doc); - } - }); - assertTrue(15 == CountingHitCollector.getCount()); - - searcher.close(); - dir.close(); - } - public static class CountingHitCollector extends Collector { static int count=0; static int sum=0; Index: lucene/backwards/src/test/org/apache/lucene/index/TestNorms.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestNorms.java (revision 1056696) +++ lucene/backwards/src/test/org/apache/lucene/index/TestNorms.java (working copy) @@ -30,7 +30,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; -import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -42,8 +41,8 @@ private class SimilarityOne extends DefaultSimilarity { @Override - public float lengthNorm(String fieldName, int numTerms) { - return 1; + public float computeNorm(String fieldName, FieldInvertState state) { + return state.getBoost(); } }