state.getBoost()*lengthNorm(numTerms), where
- * numTerms is {@link FieldInvertState#getLength()} if {@link
- * #setDiscountOverlaps} is false, else it's {@link
- * FieldInvertState#getLength()} - {@link
- * FieldInvertState#getNumOverlap()}.
- *
- * @lucene.experimental */
- @Override
- public byte computeNorm(FieldInvertState state) {
- final int numTerms;
- if (discountOverlaps)
- numTerms = state.getLength() - state.getNumOverlap();
- else
- numTerms = state.getLength();
- return encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms))));
- }
-
- /** Implemented as sqrt(freq). */
- @Override
- public float tf(float freq) {
- return (float)Math.sqrt(freq);
- }
-
- /** Implemented as 1 / (distance + 1). */
- @Override
- public float sloppyFreq(int distance) {
- return 1.0f / (distance + 1);
- }
-
- /** The default implementation returns 1 */
- @Override
- public float scorePayload(int doc, int start, int end, BytesRef payload) {
- return 1;
- }
-
- /** Implemented as log(numDocs/(docFreq+1)) + 1. */
- @Override
- public float idf(int docFreq, int numDocs) {
- return (float)(Math.log(numDocs/(double)(docFreq+1)) + 1.0);
- }
-
- // Default true
- protected boolean discountOverlaps = true;
-
- /** Determines whether overlap tokens (Tokens with
- * 0 position increment) are ignored when computing
- * norm. By default this is true, meaning overlap
- * tokens do not count when computing norms.
- *
- * @lucene.experimental
- *
- * @see #computeNorm
- */
- public void setDiscountOverlaps(boolean v) {
- discountOverlaps = v;
- }
-
- /** @see #setDiscountOverlaps */
- public boolean getDiscountOverlaps() {
- return discountOverlaps;
- }
-}
diff --git lucene/src/java/org/apache/lucene/search/DefaultSimilarityProvider.java lucene/src/java/org/apache/lucene/search/DefaultSimilarityProvider.java
deleted file mode 100644
index 0b336f7..0000000
--- lucene/src/java/org/apache/lucene/search/DefaultSimilarityProvider.java
+++ /dev/null
@@ -1,41 +0,0 @@
-package org.apache.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Expert: Default scoring provider.
- *
- * Returns {@link DefaultSimilarity} for every field
- */
-public class DefaultSimilarityProvider implements SimilarityProvider {
- private static final Similarity impl = new DefaultSimilarity();
-
- /** Implemented as overlap / maxOverlap. */
- public float coord(int overlap, int maxOverlap) {
- return overlap / (float)maxOverlap;
- }
-
- /** Implemented as 1/sqrt(sumOfSquaredWeights). */
- public float queryNorm(float sumOfSquaredWeights) {
- return (float)(1.0 / Math.sqrt(sumOfSquaredWeights));
- }
-
- public Similarity get(String field) {
- return impl;
- }
-}
diff --git lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java
index 08cf2c3..0a0454b 100644
--- lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java
+++ lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.util.Arrays;
import org.apache.lucene.index.*;
+import org.apache.lucene.search.similarities.Similarity;
final class ExactPhraseScorer extends Scorer {
private final int endMinus1;
diff --git lucene/src/java/org/apache/lucene/search/IndexSearcher.java lucene/src/java/org/apache/lucene/search/IndexSearcher.java
index 81b90f2..7583eeb 100644
--- lucene/src/java/org/apache/lucene/search/IndexSearcher.java
+++ lucene/src/java/org/apache/lucene/search/IndexSearcher.java
@@ -37,6 +37,8 @@ import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Weight.ScorerContext;
+import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
+import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory; // javadoc
import org.apache.lucene.util.ReaderUtil;
diff --git lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
index 1a65b65..a6d2b6c 100644
--- lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
+++ lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
@@ -26,7 +26,8 @@ import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.DocsAndPositionsEnum;
-import org.apache.lucene.search.Similarity.SloppyDocScorer;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
diff --git lucene/src/java/org/apache/lucene/search/PhraseQuery.java lucene/src/java/org/apache/lucene/search/PhraseQuery.java
index 73c054d..124719f 100644
--- lucene/src/java/org/apache/lucene/search/PhraseQuery.java
+++ lucene/src/java/org/apache/lucene/search/PhraseQuery.java
@@ -29,7 +29,8 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.Similarity.SloppyDocScorer;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
diff --git lucene/src/java/org/apache/lucene/search/PhraseScorer.java lucene/src/java/org/apache/lucene/search/PhraseScorer.java
index f50ae07..3afcb67 100644
--- lucene/src/java/org/apache/lucene/search/PhraseScorer.java
+++ lucene/src/java/org/apache/lucene/search/PhraseScorer.java
@@ -19,6 +19,8 @@ package org.apache.lucene.search;
import java.io.IOException;
+import org.apache.lucene.search.similarities.Similarity;
+
/** Expert: Scoring functionality for phrase queries.
*
A document is considered matching if it contains the phrase-query terms
* at "valid" positions. What "valid positions" are
diff --git lucene/src/java/org/apache/lucene/search/Similarity.java lucene/src/java/org/apache/lucene/search/Similarity.java
deleted file mode 100644
index fe184be..0000000
--- lucene/src/java/org/apache/lucene/search/Similarity.java
+++ /dev/null
@@ -1,224 +0,0 @@
-package org.apache.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-import java.io.IOException;
-
-import org.apache.lucene.document.IndexDocValuesField; // javadoc
-import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.IndexReader; // javadoc
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.index.Terms; // javadoc
-import org.apache.lucene.search.spans.SpanQuery; // javadoc
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.SmallFloat; // javadoc
-import org.apache.lucene.util.TermContext;
-
-
-/**
- * Similarity defines the components of Lucene scoring.
- *
- * Expert: Scoring API. - *
- * This is a low-level API, you should only extend this API if you want to implement - * an information retrieval model. If you are instead looking for a convenient way - * to alter Lucene's scoring, consider extending a higher-level implementation - * such as {@link TFIDFSimilarity}, which implements the vector space model with this API, or - * just tweaking the default implementation: {@link DefaultSimilarity}. - *
- * Similarity determines how Lucene weights terms, and Lucene interacts with - * this class at both index-time and - * query-time. - *
- * - * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState)}, allowing - * the Similarity implementation to return a per-document byte for the field that will - * be later accessible via {@link IndexReader#norms(String)}. Lucene makes no assumption - * about what is in this byte, but it is most useful for encoding length normalization - * information. - *
- * Implementations should carefully consider how the normalization byte is encoded: while - * Lucene's classical {@link TFIDFSimilarity} encodes a combination of index-time boost - * and length normalization information with {@link SmallFloat}, this might not be suitable - * for all purposes. - *
- * Many formulas require the use of average document length, which can be computed via a - * combination of {@link Terms#getSumTotalTermFreq()} and {@link IndexReader#maxDoc()}, - *
- * Because index-time boost is handled entirely at the application level anyway, - * an application can alternatively store the index-time boost separately using an - * {@link IndexDocValuesField}, and access this at query-time with - * {@link IndexReader#docValues(String)}. - *
- * Finally, using index-time boosts (either via folding into the normalization byte or - * via IndexDocValues), is an inefficient way to boost the scores of different fields if the - * boost will be the same for every document, instead the Similarity can simply take a constant - * boost parameter C, and the SimilarityProvider can return different instances with - * different boosts depending upon field name. - *
- * - * At query-time, Queries interact with the Similarity via these steps: - *
- * - * When {@link IndexSearcher#explain(Query, int)} is called, queries consult the Similarity's DocScorer for an - * explanation of how it computed its score. The query passes in a the document id and an explanation of how the frequency - * was computed. - * - * @see org.apache.lucene.index.IndexWriterConfig#setSimilarityProvider(SimilarityProvider) - * @see IndexSearcher#setSimilarityProvider(SimilarityProvider) - * @lucene.experimental - */ -public abstract class Similarity { - /** - * Computes the normalization value for a field, given the accumulated - * state of term processing for this field (see {@link FieldInvertState}). - * - *
Implementations should calculate a byte value based on the field - * state and then return that value. - * - *
Matches in longer fields are less precise, so implementations of this
- * method usually return smaller values when state.getLength() is large,
- * and larger values when state.getLength() is small.
- *
- * @lucene.experimental
- *
- * @param state current processing state for this field
- * @return the calculated byte norm
- */
- public abstract byte computeNorm(FieldInvertState state);
-
- /**
- * Compute any collection-level stats (e.g. IDF, average document length, etc) needed for scoring a query.
- */
- public abstract Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException;
-
- /**
- * returns a new {@link Similarity.ExactDocScorer}.
- */
- public abstract ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException;
-
- /**
- * returns a new {@link Similarity.SloppyDocScorer}.
- */
- public abstract SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException;
-
- /**
- * API for scoring exact queries such as {@link TermQuery} and
- * exact {@link PhraseQuery}.
- *
- * Term frequencies are integers (the term or phrase's tf) - */ - public abstract class ExactDocScorer { - /** - * Score a single document - * @param doc document id - * @param freq term frequency - * @return document's score - */ - public abstract float score(int doc, int freq); - - /** - * Explain the score for a single document - * @param doc document id - * @param freq Explanation of how the term frequency was computed - * @return document's score - */ - public Explanation explain(int doc, Explanation freq) { - Explanation result = new Explanation(score(doc, (int)freq.getValue()), - "score(doc=" + doc + ",freq=" + freq.getValue() +"), with freq of:"); - result.addDetail(freq); - return result; - } - } - - /** - * API for scoring "sloppy" queries such as {@link SpanQuery} and - * sloppy {@link PhraseQuery}. - *
- * Term frequencies are floating point values. - */ - public abstract class SloppyDocScorer { - /** - * Score a single document - * @param doc document id - * @param freq sloppy term frequency - * @return document's score - */ - public abstract float score(int doc, float freq); - - /** Computes the amount of a sloppy phrase match, based on an edit distance. */ - public abstract float computeSlopFactor(int distance); - - /** Calculate a scoring factor based on the data in the payload. */ - public abstract float computePayloadFactor(int doc, int start, int end, BytesRef payload); - - /** - * Explain the score for a single document - * @param doc document id - * @param freq Explanation of how the sloppy term frequency was computed - * @return document's score - */ - public Explanation explain(int doc, Explanation freq) { - Explanation result = new Explanation(score(doc, freq.getValue()), - "score(doc=" + doc + ",freq=" + freq.getValue() +"), with freq of:"); - result.addDetail(freq); - return result; - } - } - - /** Stores the statistics for the indexed collection. This abstract - * implementation is empty; descendants of {@code Similarity} should - * subclass {@code Stats} and define the statistics they require in the - * subclass. Examples include idf, average field length, etc. - */ - public static abstract class Stats { - - /** The value for normalization of contained query clauses (e.g. sum of squared weights). - *
- * NOTE: a Similarity implementation might not use any query normalization at all, - * its not required. However, if it wants to participate in query normalization, - * it can return a value here. - */ - public abstract float getValueForNormalization(); - - /** Assigns the query normalization factor and boost from parent queries to this. - *
- * NOTE: a Similarity implementation might not use this normalized value at all, - * its not required. However, its usually a good idea to at least incorporate - * the topLevelBoost (e.g. from an outer BooleanQuery) into its score. - */ - public abstract void normalize(float queryNorm, float topLevelBoost); - } -} diff --git lucene/src/java/org/apache/lucene/search/SimilarityProvider.java lucene/src/java/org/apache/lucene/search/SimilarityProvider.java deleted file mode 100644 index ef9a034..0000000 --- lucene/src/java/org/apache/lucene/search/SimilarityProvider.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Expert: Scoring API. - * - * Provides top-level scoring functions that aren't specific to a field, - * and work across multi-field queries (such as {@link BooleanQuery}). - * - * Field-specific scoring is accomplished through {@link Similarity}. - * - * @lucene.experimental - */ -public interface SimilarityProvider { - - /** Computes a score factor based on the fraction of all query terms that a - * document contains. This value is multiplied into scores. - * - *
The presence of a large portion of the query terms indicates a better - * match with the query, so implementations of this method usually return - * larger values when the ratio between these parameters is large and smaller - * values when the ratio between them is small. - * - * @param overlap the number of query terms matched in the document - * @param maxOverlap the total number of terms in the query - * @return a score factor based on term overlap with the query - */ - public abstract float coord(int overlap, int maxOverlap); - - /** Computes the normalization value for a query given the sum of the squared - * weights of each of the query terms. This value is multiplied into the - * weight of each query term. While the classic query normalization factor is - * computed as 1/sqrt(sumOfSquaredWeights), other implementations might - * completely ignore sumOfSquaredWeights (ie return 1). - * - *
This does not affect ranking, but the default implementation does make scores - * from different queries more comparable than they would be by eliminating the - * magnitude of the Query vector as a factor in the score. - * - * @param sumOfSquaredWeights the sum of the squares of query term weights - * @return a normalization factor for query weights - */ - public abstract float queryNorm(float sumOfSquaredWeights); - - /** Returns a {@link Similarity} for scoring a field - * @param field field name. - * @return a field-specific Similarity. - */ - public abstract Similarity get(String field); -} diff --git lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java index d13f5cb..3197e3c 100644 --- lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java +++ lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java @@ -20,6 +20,8 @@ package org.apache.lucene.search; import java.io.IOException; import java.util.HashSet; +import org.apache.lucene.search.similarities.Similarity; + final class SloppyPhraseScorer extends PhraseScorer { private int slop; private PhrasePositions repeats[]; diff --git lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java deleted file mode 100644 index 3ae85bc..0000000 --- lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java +++ /dev/null @@ -1,870 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import java.io.IOException; - -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.TermContext; -import org.apache.lucene.util.SmallFloat; - - -/** - * Implementation of {@link Similarity} with the Vector Space Model. - *
- * Expert: Scoring API. - *
TFIDFSimilarity defines the components of Lucene scoring. - * Overriding computation of these components is a convenient - * way to alter Lucene scoring. - * - *
Suggested reading: - * - * Introduction To Information Retrieval, Chapter 6. - * - *
The following describes how Lucene scoring evolves from - * underlying information retrieval models to (efficient) implementation. - * We first brief on VSM Score, - * then derive from it Lucene's Conceptual Scoring Formula, - * from which, finally, evolves Lucene's Practical Scoring Function - * (the latter is connected directly with Lucene classes and methods). - * - *
Lucene combines - * - * Boolean model (BM) of Information Retrieval - * with - * - * Vector Space Model (VSM) of Information Retrieval - - * documents "approved" by BM are scored by VSM. - * - *
In VSM, documents and queries are represented as - * weighted vectors in a multi-dimensional space, - * where each distinct index term is a dimension, - * and weights are - * Tf-idf values. - * - *
VSM does not require weights to be Tf-idf values, - * but Tf-idf values are believed to produce search results of high quality, - * and so Lucene is using Tf-idf. - * Tf and Idf are described in more detail below, - * but for now, for completion, let's just say that - * for given term t and document (or query) x, - * Tf(t,x) varies with the number of occurrences of term t in x - * (when one increases so does the other) and - * idf(t) similarly varies with the inverse of the - * number of index documents containing term t. - * - *
VSM score of document d for query q is the
- *
- * Cosine Similarity
- * of the weighted query vectors V(q) and V(d):
- *
- *
- *
- *
| ||||||
|
- * |
Note: the above equation can be viewed as the dot product of - * the normalized weighted vectors, in the sense that dividing - * V(q) by its euclidean norm is normalizing it to a unit vector. - * - *
Lucene refines VSM score for both search quality and usability: - *
Under the simplifying assumption of a single field in the index,
- * we get Lucene's Conceptual scoring formula:
- *
- *
- *
- *
| |||||||
|
- * |
The conceptual formula is a simplification in the sense that (1) terms and documents - * are fielded and (2) boosts are usually per query term rather than per query. - * - *
We now describe how Lucene implements this conceptual scoring formula, and - * derive from it Lucene's Practical Scoring Function. - * - *
For efficient score computation some scoring components - * are computed and aggregated in advance: - * - *
Lucene's Practical Scoring Function is derived from the above. - * The color codes demonstrate how it relates - * to those of the conceptual formula: - * - *
- *
- *
| |||||||
|
- * |
where - *
| - * {@link org.apache.lucene.search.DefaultSimilarity#tf(float) tf(t in d)} = - * | - *- * frequency½ - * | - *
| - * {@link org.apache.lucene.search.DefaultSimilarity#idf(int, int) idf(t)} = - * | - *- * 1 + log ( - * | - *
- *
|
- * - * ) - * | - *
| - * queryNorm(q) = - * {@link org.apache.lucene.search.DefaultSimilarityProvider#queryNorm(float) queryNorm(sumOfSquaredWeights)} - * = - * | - *
- *
|
- *
| - * {@link org.apache.lucene.search.Weight#getValueForNormalization() sumOfSquaredWeights} = - * {@link org.apache.lucene.search.Query#getBoost() q.getBoost()} 2 - * · - * | - *- * ∑ - * | - *- * ( - * idf(t) · - * t.getBoost() - * ) 2 - * | - *
| - * | t in q | - *- * |
- * When a document is added to the index, all the above factors are multiplied.
- * If the document has multiple fields with the same name, all their boosts are multiplied together:
- *
- *
- *
| - * norm(t,d) = - * {@link org.apache.lucene.document.Document#getBoost() doc.getBoost()} - * · - * lengthNorm - * · - * | - *- * ∏ - * | - *- * {@link org.apache.lucene.document.Fieldable#getBoost() f.getBoost}() - * | - *
| - * | field f in d named as t | - *- * |
Terms and phrases repeated in a document indicate the topic of the
- * document, so implementations of this method usually return larger values
- * when freq is large, and smaller values when freq
- * is small.
- *
- *
The default implementation calls {@link #tf(float)}. - * - * @param freq the frequency of a term within a document - * @return a score factor based on a term's within-document frequency - */ - public float tf(int freq) { - return tf((float)freq); - } - - /** Computes a score factor based on a term or phrase's frequency in a - * document. This value is multiplied by the {@link #idf(int, int)} - * factor for each term in the query and these products are then summed to - * form the initial score for a document. - * - *
Terms and phrases repeated in a document indicate the topic of the
- * document, so implementations of this method usually return larger values
- * when freq is large, and smaller values when freq
- * is small.
- *
- * @param freq the frequency of a term within a document
- * @return a score factor based on a term's within-document frequency
- */
- public abstract float tf(float freq);
-
- /**
- * Computes a score factor for a simple term and returns an explanation
- * for that score factor.
- *
- *
- * The default implementation uses: - * - *
- * idf(docFreq, searcher.maxDoc()); - *- * - * Note that {@link IndexSearcher#maxDoc()} is used instead of - * {@link org.apache.lucene.index.IndexReader#numDocs() IndexReader#numDocs()} because also - * {@link IndexSearcher#docFreq(Term)} is used, and when the latter - * is inaccurate, so is {@link IndexSearcher#maxDoc()}, and in the same direction. - * In addition, {@link IndexSearcher#maxDoc()} is more efficient to compute - * - * @param stats statistics of the term in question - * @param searcher the document collection being searched - * @return an Explain object that includes both an idf score factor - and an explanation for the term. - * @throws IOException - */ - public Explanation idfExplain(TermContext stats, final IndexSearcher searcher) throws IOException { - final int df = stats.docFreq(); - final int max = searcher.maxDoc(); - final float idf = idf(df, max); - return new Explanation(idf, "idf(docFreq=" + df + ", maxDocs=" + max + ")"); - } - - /** - * Computes a score factor for a phrase. - * - *
- * The default implementation sums the idf factor for - * each term in the phrase. - * - * @param stats statistics of the terms in the phrase - * @param searcher the document collection being searched - * @return an Explain object that includes both an idf - * score factor for the phrase and an explanation - * for each term. - * @throws IOException - */ - public Explanation idfExplain(final TermContext stats[], IndexSearcher searcher) throws IOException { - final int max = searcher.maxDoc(); - float idf = 0.0f; - final Explanation exp = new Explanation(); - exp.setDescription("idf(), sum of:"); - for (final TermContext stat : stats ) { - final int df = stat.docFreq(); - final float termIdf = idf(df, max); - exp.addDetail(new Explanation(termIdf, "idf(docFreq=" + df + ", maxDocs=" + max + ")")); - idf += termIdf; - } - exp.setValue(idf); - return exp; - } - - /** Computes a score factor based on a term's document frequency (the number - * of documents which contain the term). This value is multiplied by the - * {@link #tf(int)} factor for each term in the query and these products are - * then summed to form the initial score for a document. - * - *
Terms that occur in fewer documents are better indicators of topic, so - * implementations of this method usually return larger values for rare terms, - * and smaller values for common terms. - * - * @param docFreq the number of documents which contain the term - * @param numDocs the total number of documents in the collection - * @return a score factor based on the term's document frequency - */ - public abstract float idf(int docFreq, int numDocs); - - /** Cache of decoded bytes. */ - private static final float[] NORM_TABLE = new float[256]; - - static { - for (int i = 0; i < 256; i++) - NORM_TABLE[i] = SmallFloat.byte315ToFloat((byte)i); - } - - /** Decodes a normalization factor stored in an index. - * @see #encodeNormValue(float) - */ - public float decodeNormValue(byte b) { - return NORM_TABLE[b & 0xFF]; // & 0xFF maps negative bytes to positive above 127 - } - - /** Encodes a normalization factor for storage in an index. - * - *
The encoding uses a three-bit mantissa, a five-bit exponent, and - * the zero-exponent point at 15, thus - * representing values from around 7x10^9 to 2x10^-9 with about one - * significant decimal digit of accuracy. Zero is also represented. - * Negative numbers are rounded up to zero. Values too large to represent - * are rounded down to the largest representable value. Positive values too - * small to represent are rounded up to the smallest positive representable - * value. - * @see org.apache.lucene.document.Field#setBoost(float) - * @see org.apache.lucene.util.SmallFloat - */ - public byte encodeNormValue(float f) { - return SmallFloat.floatToByte315(f); - } - - /** Computes the amount of a sloppy phrase match, based on an edit distance. - * This value is summed for each sloppy phrase match in a document to form - * the frequency to be used in scoring instead of the exact term count. - * - *
A phrase match with a small edit distance to a document passage more
- * closely matches the document, so implementations of this method usually
- * return larger values when the edit distance is small and smaller values
- * when it is large.
- *
- * @see PhraseQuery#setSlop(int)
- * @param distance the edit distance of this sloppy phrase match
- * @return the frequency increment for this match
- */
- public abstract float sloppyFreq(int distance);
-
- /**
- * Calculate a scoring factor based on the data in the payload. Implementations
- * are responsible for interpreting what is in the payload. Lucene makes no assumptions about
- * what is in the byte array.
- *
- * @param doc The docId currently being scored.
- * @param start The start position of the payload
- * @param end The end position of the payload
- * @param payload The payload byte array to be scored
- * @return An implementation dependent float to be used as a scoring factor
- */
- public abstract float scorePayload(int doc, int start, int end, BytesRef payload);
-
- @Override
- public final Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost,
- TermContext... termContexts) throws IOException {
- final Explanation idf = termContexts.length == 1
- ? idfExplain(termContexts[0], searcher)
- : idfExplain(termContexts, searcher);
- return new IDFStats(idf, queryBoost);
- }
-
- @Override
- public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
- return new ExactTFIDFDocScorer((IDFStats)stats, context.reader.norms(fieldName));
- }
-
- @Override
- public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
- return new SloppyTFIDFDocScorer((IDFStats)stats, context.reader.norms(fieldName));
- }
-
- // TODO: we can specialize these for omitNorms up front, but we should test that it doesn't confuse stupid hotspot.
-
- private final class ExactTFIDFDocScorer extends ExactDocScorer {
- private final IDFStats stats;
- private final float weightValue;
- private final byte[] norms;
- private static final int SCORE_CACHE_SIZE = 32;
- private float[] scoreCache = new float[SCORE_CACHE_SIZE];
-
- ExactTFIDFDocScorer(IDFStats stats, byte norms[]) {
- this.stats = stats;
- this.weightValue = stats.value;
- this.norms = norms;
- for (int i = 0; i < SCORE_CACHE_SIZE; i++)
- scoreCache[i] = tf(i) * weightValue;
- }
-
- @Override
- public float score(int doc, int freq) {
- final float raw = // compute tf(f)*weight
- freq < SCORE_CACHE_SIZE // check cache
- ? scoreCache[freq] // cache hit
- : tf(freq)*weightValue; // cache miss
-
- return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field
- }
-
- @Override
- public Explanation explain(int doc, Explanation freq) {
- return explainScore(doc, freq, stats, norms);
- }
- }
-
- private final class SloppyTFIDFDocScorer extends SloppyDocScorer {
- private final IDFStats stats;
- private final float weightValue;
- private final byte[] norms;
-
- SloppyTFIDFDocScorer(IDFStats stats, byte norms[]) {
- this.stats = stats;
- this.weightValue = stats.value;
- this.norms = norms;
- }
-
- @Override
- public float score(int doc, float freq) {
- final float raw = tf(freq) * weightValue; // compute tf(f)*weight
-
- return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field
- }
-
- @Override
- public float computeSlopFactor(int distance) {
- return sloppyFreq(distance);
- }
-
- @Override
- public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
- return scorePayload(doc, start, end, payload);
- }
-
- @Override
- public Explanation explain(int doc, Explanation freq) {
- return explainScore(doc, freq, stats, norms);
- }
- }
-
- /** Collection statistics for the TF-IDF model. The only statistic of interest
- * to this model is idf. */
- private static class IDFStats extends Stats {
- /** The idf and its explanation */
- private final Explanation idf;
- private float queryNorm;
- private float queryWeight;
- private final float queryBoost;
- private float value;
-
- public IDFStats(Explanation idf, float queryBoost) {
- // TODO: Validate?
- this.idf = idf;
- this.queryBoost = queryBoost;
- this.queryWeight = idf.getValue() * queryBoost; // compute query weight
- }
-
- @Override
- public float getValueForNormalization() {
- // TODO: (sorta LUCENE-1907) make non-static class and expose this squaring via a nice method to subclasses?
- return queryWeight * queryWeight; // sum of squared weights
- }
-
- @Override
- public void normalize(float queryNorm, float topLevelBoost) {
- this.queryNorm = queryNorm * topLevelBoost;
- queryWeight *= this.queryNorm; // normalize query weight
- value = queryWeight * idf.getValue(); // idf for document
- }
- }
-
- private Explanation explainScore(int doc, Explanation freq, IDFStats stats, byte[] norms) {
- Explanation result = new Explanation();
- result.setDescription("score(doc="+doc+",freq="+freq+"), product of:");
-
- // explain query weight
- Explanation queryExpl = new Explanation();
- queryExpl.setDescription("queryWeight, product of:");
-
- Explanation boostExpl = new Explanation(stats.queryBoost, "boost");
- if (stats.queryBoost != 1.0f)
- queryExpl.addDetail(boostExpl);
- queryExpl.addDetail(stats.idf);
-
- Explanation queryNormExpl = new Explanation(stats.queryNorm,"queryNorm");
- queryExpl.addDetail(queryNormExpl);
-
- queryExpl.setValue(boostExpl.getValue() *
- stats.idf.getValue() *
- queryNormExpl.getValue());
-
- result.addDetail(queryExpl);
-
- // explain field weight
- Explanation fieldExpl = new Explanation();
- fieldExpl.setDescription("fieldWeight in "+doc+
- ", product of:");
-
- Explanation tfExplanation = new Explanation();
- tfExplanation.setValue(tf(freq.getValue()));
- tfExplanation.setDescription("tf(freq="+freq.getValue()+"), with freq of:");
- tfExplanation.addDetail(freq);
- fieldExpl.addDetail(tfExplanation);
- fieldExpl.addDetail(stats.idf);
-
- Explanation fieldNormExpl = new Explanation();
- float fieldNorm =
- norms!=null ? decodeNormValue(norms[doc]) : 1.0f;
- fieldNormExpl.setValue(fieldNorm);
- fieldNormExpl.setDescription("fieldNorm(doc="+doc+")");
- fieldExpl.addDetail(fieldNormExpl);
-
- fieldExpl.setValue(tfExplanation.getValue() *
- stats.idf.getValue() *
- fieldNormExpl.getValue());
-
- result.addDetail(fieldExpl);
-
- // combine them
- result.setValue(queryExpl.getValue() * fieldExpl.getValue());
-
- if (queryExpl.getValue() == 1.0f)
- return fieldExpl;
-
- return result;
- }
-}
diff --git lucene/src/java/org/apache/lucene/search/TermQuery.java lucene/src/java/org/apache/lucene/search/TermQuery.java
index 936b0bf..92f1336 100644
--- lucene/src/java/org/apache/lucene/search/TermQuery.java
+++ lucene/src/java/org/apache/lucene/search/TermQuery.java
@@ -27,7 +27,8 @@ import org.apache.lucene.index.Terms;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Similarity.ExactDocScorer;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ReaderUtil;
diff --git lucene/src/java/org/apache/lucene/search/TermScorer.java lucene/src/java/org/apache/lucene/search/TermScorer.java
index 3534079..5811815 100644
--- lucene/src/java/org/apache/lucene/search/TermScorer.java
+++ lucene/src/java/org/apache/lucene/search/TermScorer.java
@@ -20,6 +20,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.search.similarities.Similarity;
/** Expert: A Scorer for documents matching a Term.
*/
diff --git lucene/src/java/org/apache/lucene/search/Weight.java lucene/src/java/org/apache/lucene/search/Weight.java
index e99c5a6..94607cb 100644
--- lucene/src/java/org/apache/lucene/search/Weight.java
+++ lucene/src/java/org/apache/lucene/search/Weight.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
+import org.apache.lucene.search.similarities.SimilarityProvider;
/**
* Expert: Calculate query weights and build query scorers.
diff --git lucene/src/java/org/apache/lucene/search/package.html lucene/src/java/org/apache/lucene/search/package.html
index a11d84e..082b5cb 100644
--- lucene/src/java/org/apache/lucene/search/package.html
+++ lucene/src/java/org/apache/lucene/search/package.html
@@ -223,9 +223,9 @@ org.apache.lucene.search.IndexSearcher#search(Query,Filter,int)}.
you'll want to simply subclass
DefaultSimilarity) and then use the new
class by calling
- IndexWriter.setSimilarity
+ IndexWriter.setSimilarity
before indexing and
- Searcher.setSimilarity
+ Searcher.setSimilarity
before searching.
state.getBoost()*lengthNorm(numTerms), where
+ * numTerms is {@link FieldInvertState#getLength()} if {@link
+ * #setDiscountOverlaps} is false, else it's {@link
+ * FieldInvertState#getLength()} - {@link
+ * FieldInvertState#getNumOverlap()}.
+ *
+ * @lucene.experimental */
+ @Override
+ public byte computeNorm(FieldInvertState state) {
+ final int numTerms;
+ if (discountOverlaps)
+ numTerms = state.getLength() - state.getNumOverlap();
+ else
+ numTerms = state.getLength();
+ return encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms))));
+ }
+
+ /** Implemented as sqrt(freq). */
+ @Override
+ public float tf(float freq) {
+ return (float)Math.sqrt(freq);
+ }
+
+ /** Implemented as 1 / (distance + 1). */
+ @Override
+ public float sloppyFreq(int distance) {
+ return 1.0f / (distance + 1);
+ }
+
+ /** The default implementation returns 1 */
+ @Override
+ public float scorePayload(int doc, int start, int end, BytesRef payload) {
+ return 1;
+ }
+
+ /** Implemented as log(numDocs/(docFreq+1)) + 1. */
+ @Override
+ public float idf(int docFreq, int numDocs) {
+ return (float)(Math.log(numDocs/(double)(docFreq+1)) + 1.0);
+ }
+
+ // Default true
+ protected boolean discountOverlaps = true;
+
+ /** Determines whether overlap tokens (Tokens with
+ * 0 position increment) are ignored when computing
+ * norm. By default this is true, meaning overlap
+ * tokens do not count when computing norms.
+ *
+ * @lucene.experimental
+ *
+ * @see #computeNorm
+ */
+ public void setDiscountOverlaps(boolean v) {
+ discountOverlaps = v;
+ }
+
+ /** @see #setDiscountOverlaps */
+ public boolean getDiscountOverlaps() {
+ return discountOverlaps;
+ }
+}
diff --git lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarityProvider.java lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarityProvider.java
new file mode 100644
index 0000000..1ef73ff
--- /dev/null
+++ lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarityProvider.java
@@ -0,0 +1,42 @@
+package org.apache.lucene.search.similarities;
+
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Expert: Default scoring provider.
+ *
+ * Returns {@link DefaultSimilarity} for every field
+ */
+public class DefaultSimilarityProvider implements SimilarityProvider {
+ private static final Similarity impl = new DefaultSimilarity();
+
+ /** Implemented as overlap / maxOverlap. */
+ public float coord(int overlap, int maxOverlap) {
+ return overlap / (float)maxOverlap;
+ }
+
+ /** Implemented as 1/sqrt(sumOfSquaredWeights). */
+ public float queryNorm(float sumOfSquaredWeights) {
+ return (float)(1.0 / Math.sqrt(sumOfSquaredWeights));
+ }
+
+ public Similarity get(String field) {
+ return impl;
+ }
+}
diff --git lucene/src/java/org/apache/lucene/search/similarities/EasySimilarity.java lucene/src/java/org/apache/lucene/search/similarities/EasySimilarity.java
index 4425ba5..870f3fb 100644
--- lucene/src/java/org/apache/lucene/search/similarities/EasySimilarity.java
+++ lucene/src/java/org/apache/lucene/search/similarities/EasySimilarity.java
@@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Similarity;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SmallFloat;
import org.apache.lucene.util.TermContext;
diff --git lucene/src/java/org/apache/lucene/search/similarities/EasyStats.java lucene/src/java/org/apache/lucene/search/similarities/EasyStats.java
index 89b3263..fbf3cc3 100644
--- lucene/src/java/org/apache/lucene/search/similarities/EasyStats.java
+++ lucene/src/java/org/apache/lucene/search/similarities/EasyStats.java
@@ -17,9 +17,7 @@ package org.apache.lucene.search.similarities;
* limitations under the License.
*/
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.search.Similarity;
/**
* Stores all statistics commonly used ranking methods.
diff --git lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
index 32ff804..f002caf 100644
--- lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
+++ lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java
@@ -18,7 +18,6 @@ package org.apache.lucene.search.similarities;
*/
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.similarities.LMSimilarity.LMStats;
/**
* Language model based on the Jelinek-Mercer smoothing method. From Chengxiang
diff --git lucene/src/java/org/apache/lucene/search/similarities/Similarity.java lucene/src/java/org/apache/lucene/search/similarities/Similarity.java
new file mode 100644
index 0000000..cbe2a48
--- /dev/null
+++ lucene/src/java/org/apache/lucene/search/similarities/Similarity.java
@@ -0,0 +1,230 @@
+package org.apache.lucene.search.similarities;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+import java.io.IOException;
+
+import org.apache.lucene.document.IndexDocValuesField; // javadoc
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.index.IndexReader; // javadoc
+import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.Terms; // javadoc
+import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.Explanation;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.Query;
+import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.spans.SpanQuery; // javadoc
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.SmallFloat; // javadoc
+import org.apache.lucene.util.TermContext;
+
+
+/**
+ * Similarity defines the components of Lucene scoring.
+ *
+ * Expert: Scoring API. + *
+ * This is a low-level API, you should only extend this API if you want to implement + * an information retrieval model. If you are instead looking for a convenient way + * to alter Lucene's scoring, consider extending a higher-level implementation + * such as {@link TFIDFSimilarity}, which implements the vector space model with this API, or + * just tweaking the default implementation: {@link DefaultSimilarity}. + *
+ * Similarity determines how Lucene weights terms, and Lucene interacts with + * this class at both index-time and + * query-time. + *
+ * + * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState)}, allowing + * the Similarity implementation to return a per-document byte for the field that will + * be later accessible via {@link IndexReader#norms(String)}. Lucene makes no assumption + * about what is in this byte, but it is most useful for encoding length normalization + * information. + *
+ * Implementations should carefully consider how the normalization byte is encoded: while + * Lucene's classical {@link TFIDFSimilarity} encodes a combination of index-time boost + * and length normalization information with {@link SmallFloat}, this might not be suitable + * for all purposes. + *
+ * Many formulas require the use of average document length, which can be computed via a + * combination of {@link Terms#getSumTotalTermFreq()} and {@link IndexReader#maxDoc()}, + *
+ * Because index-time boost is handled entirely at the application level anyway, + * an application can alternatively store the index-time boost separately using an + * {@link IndexDocValuesField}, and access this at query-time with + * {@link IndexReader#docValues(String)}. + *
+ * Finally, using index-time boosts (either via folding into the normalization byte or + * via IndexDocValues), is an inefficient way to boost the scores of different fields if the + * boost will be the same for every document, instead the Similarity can simply take a constant + * boost parameter C, and the SimilarityProvider can return different instances with + * different boosts depending upon field name. + *
+ * + * At query-time, Queries interact with the Similarity via these steps: + *
+ * + * When {@link IndexSearcher#explain(Query, int)} is called, queries consult the Similarity's DocScorer for an + * explanation of how it computed its score. The query passes in a the document id and an explanation of how the frequency + * was computed. + * + * @see org.apache.lucene.index.IndexWriterConfig#setSimilarityProvider(SimilarityProvider) + * @see IndexSearcher#setSimilarityProvider(SimilarityProvider) + * @lucene.experimental + */ +public abstract class Similarity { + /** + * Computes the normalization value for a field, given the accumulated + * state of term processing for this field (see {@link FieldInvertState}). + * + *
Implementations should calculate a byte value based on the field + * state and then return that value. + * + *
Matches in longer fields are less precise, so implementations of this
+ * method usually return smaller values when state.getLength() is large,
+ * and larger values when state.getLength() is small.
+ *
+ * @lucene.experimental
+ *
+ * @param state current processing state for this field
+ * @return the calculated byte norm
+ */
+ public abstract byte computeNorm(FieldInvertState state);
+
+ /**
+ * Compute any collection-level stats (e.g. IDF, average document length, etc) needed for scoring a query.
+ */
+ public abstract Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException;
+
+ /**
+ * returns a new {@link Similarity.ExactDocScorer}.
+ */
+ public abstract ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException;
+
+ /**
+ * returns a new {@link Similarity.SloppyDocScorer}.
+ */
+ public abstract SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException;
+
+ /**
+ * API for scoring exact queries such as {@link TermQuery} and
+ * exact {@link PhraseQuery}.
+ *
+ * Term frequencies are integers (the term or phrase's tf) + */ + public abstract class ExactDocScorer { + /** + * Score a single document + * @param doc document id + * @param freq term frequency + * @return document's score + */ + public abstract float score(int doc, int freq); + + /** + * Explain the score for a single document + * @param doc document id + * @param freq Explanation of how the term frequency was computed + * @return document's score + */ + public Explanation explain(int doc, Explanation freq) { + Explanation result = new Explanation(score(doc, (int)freq.getValue()), + "score(doc=" + doc + ",freq=" + freq.getValue() +"), with freq of:"); + result.addDetail(freq); + return result; + } + } + + /** + * API for scoring "sloppy" queries such as {@link SpanQuery} and + * sloppy {@link PhraseQuery}. + *
+ * Term frequencies are floating point values. + */ + public abstract class SloppyDocScorer { + /** + * Score a single document + * @param doc document id + * @param freq sloppy term frequency + * @return document's score + */ + public abstract float score(int doc, float freq); + + /** Computes the amount of a sloppy phrase match, based on an edit distance. */ + public abstract float computeSlopFactor(int distance); + + /** Calculate a scoring factor based on the data in the payload. */ + public abstract float computePayloadFactor(int doc, int start, int end, BytesRef payload); + + /** + * Explain the score for a single document + * @param doc document id + * @param freq Explanation of how the sloppy term frequency was computed + * @return document's score + */ + public Explanation explain(int doc, Explanation freq) { + Explanation result = new Explanation(score(doc, freq.getValue()), + "score(doc=" + doc + ",freq=" + freq.getValue() +"), with freq of:"); + result.addDetail(freq); + return result; + } + } + + /** Stores the statistics for the indexed collection. This abstract + * implementation is empty; descendants of {@code Similarity} should + * subclass {@code Stats} and define the statistics they require in the + * subclass. Examples include idf, average field length, etc. + */ + public static abstract class Stats { + + /** The value for normalization of contained query clauses (e.g. sum of squared weights). + *
+ * NOTE: a Similarity implementation might not use any query normalization at all, + * its not required. However, if it wants to participate in query normalization, + * it can return a value here. + */ + public abstract float getValueForNormalization(); + + /** Assigns the query normalization factor and boost from parent queries to this. + *
+ * NOTE: a Similarity implementation might not use this normalized value at all, + * its not required. However, its usually a good idea to at least incorporate + * the topLevelBoost (e.g. from an outer BooleanQuery) into its score. + */ + public abstract void normalize(float queryNorm, float topLevelBoost); + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/SimilarityProvider.java lucene/src/java/org/apache/lucene/search/similarities/SimilarityProvider.java new file mode 100644 index 0000000..e3f6e86 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/SimilarityProvider.java @@ -0,0 +1,68 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.BooleanQuery; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Expert: Scoring API. + * + * Provides top-level scoring functions that aren't specific to a field, + * and work across multi-field queries (such as {@link BooleanQuery}). + * + * Field-specific scoring is accomplished through {@link Similarity}. + * + * @lucene.experimental + */ +public interface SimilarityProvider { + + /** Computes a score factor based on the fraction of all query terms that a + * document contains. This value is multiplied into scores. + * + *
The presence of a large portion of the query terms indicates a better + * match with the query, so implementations of this method usually return + * larger values when the ratio between these parameters is large and smaller + * values when the ratio between them is small. + * + * @param overlap the number of query terms matched in the document + * @param maxOverlap the total number of terms in the query + * @return a score factor based on term overlap with the query + */ + public abstract float coord(int overlap, int maxOverlap); + + /** Computes the normalization value for a query given the sum of the squared + * weights of each of the query terms. This value is multiplied into the + * weight of each query term. While the classic query normalization factor is + * computed as 1/sqrt(sumOfSquaredWeights), other implementations might + * completely ignore sumOfSquaredWeights (ie return 1). + * + *
This does not affect ranking, but the default implementation does make scores + * from different queries more comparable than they would be by eliminating the + * magnitude of the Query vector as a factor in the score. + * + * @param sumOfSquaredWeights the sum of the squares of query term weights + * @return a normalization factor for query weights + */ + public abstract float queryNorm(float sumOfSquaredWeights); + + /** Returns a {@link Similarity} for scoring a field + * @param field field name. + * @return a field-specific Similarity. + */ + public abstract Similarity get(String field); +} diff --git lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java new file mode 100644 index 0000000..bac07dd --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java @@ -0,0 +1,880 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.Weight; +import org.apache.lucene.search.similarities.Similarity.ExactDocScorer; +import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer; +import org.apache.lucene.search.similarities.Similarity.Stats; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.TermContext; +import org.apache.lucene.util.SmallFloat; + + +/** + * Implementation of {@link Similarity} with the Vector Space Model. + *
+ * Expert: Scoring API. + *
TFIDFSimilarity defines the components of Lucene scoring. + * Overriding computation of these components is a convenient + * way to alter Lucene scoring. + * + *
Suggested reading: + * + * Introduction To Information Retrieval, Chapter 6. + * + *
The following describes how Lucene scoring evolves from + * underlying information retrieval models to (efficient) implementation. + * We first brief on VSM Score, + * then derive from it Lucene's Conceptual Scoring Formula, + * from which, finally, evolves Lucene's Practical Scoring Function + * (the latter is connected directly with Lucene classes and methods). + * + *
Lucene combines + * + * Boolean model (BM) of Information Retrieval + * with + * + * Vector Space Model (VSM) of Information Retrieval - + * documents "approved" by BM are scored by VSM. + * + *
In VSM, documents and queries are represented as + * weighted vectors in a multi-dimensional space, + * where each distinct index term is a dimension, + * and weights are + * Tf-idf values. + * + *
VSM does not require weights to be Tf-idf values, + * but Tf-idf values are believed to produce search results of high quality, + * and so Lucene is using Tf-idf. + * Tf and Idf are described in more detail below, + * but for now, for completion, let's just say that + * for given term t and document (or query) x, + * Tf(t,x) varies with the number of occurrences of term t in x + * (when one increases so does the other) and + * idf(t) similarly varies with the inverse of the + * number of index documents containing term t. + * + *
VSM score of document d for query q is the
+ *
+ * Cosine Similarity
+ * of the weighted query vectors V(q) and V(d):
+ *
+ *
+ *
+ *
| ||||||
|
+ * |
Note: the above equation can be viewed as the dot product of + * the normalized weighted vectors, in the sense that dividing + * V(q) by its euclidean norm is normalizing it to a unit vector. + * + *
Lucene refines VSM score for both search quality and usability: + *
Under the simplifying assumption of a single field in the index,
+ * we get Lucene's Conceptual scoring formula:
+ *
+ *
+ *
+ *
| |||||||
|
+ * |
The conceptual formula is a simplification in the sense that (1) terms and documents + * are fielded and (2) boosts are usually per query term rather than per query. + * + *
We now describe how Lucene implements this conceptual scoring formula, and + * derive from it Lucene's Practical Scoring Function. + * + *
For efficient score computation some scoring components + * are computed and aggregated in advance: + * + *
Lucene's Practical Scoring Function is derived from the above. + * The color codes demonstrate how it relates + * to those of the conceptual formula: + * + *
+ *
+ *
| |||||||
|
+ * |
where + *
| + * {@link org.apache.lucene.search.similarities.DefaultSimilarity#tf(float) tf(t in d)} = + * | + *+ * frequency½ + * | + *
| + * {@link org.apache.lucene.search.similarities.DefaultSimilarity#idf(int, int) idf(t)} = + * | + *+ * 1 + log ( + * | + *
+ *
|
+ * + * ) + * | + *
| + * queryNorm(q) = + * {@link org.apache.lucene.search.similarities.DefaultSimilarityProvider#queryNorm(float) queryNorm(sumOfSquaredWeights)} + * = + * | + *
+ *
|
+ *
| + * {@link org.apache.lucene.search.Weight#getValueForNormalization() sumOfSquaredWeights} = + * {@link org.apache.lucene.search.Query#getBoost() q.getBoost()} 2 + * · + * | + *+ * ∑ + * | + *+ * ( + * idf(t) · + * t.getBoost() + * ) 2 + * | + *
| + * | t in q | + *+ * |
+ * When a document is added to the index, all the above factors are multiplied.
+ * If the document has multiple fields with the same name, all their boosts are multiplied together:
+ *
+ *
+ *
| + * norm(t,d) = + * {@link org.apache.lucene.document.Document#getBoost() doc.getBoost()} + * · + * lengthNorm + * · + * | + *+ * ∏ + * | + *+ * {@link org.apache.lucene.document.Fieldable#getBoost() f.getBoost}() + * | + *
| + * | field f in d named as t | + *+ * |
Terms and phrases repeated in a document indicate the topic of the
+ * document, so implementations of this method usually return larger values
+ * when freq is large, and smaller values when freq
+ * is small.
+ *
+ *
The default implementation calls {@link #tf(float)}. + * + * @param freq the frequency of a term within a document + * @return a score factor based on a term's within-document frequency + */ + public float tf(int freq) { + return tf((float)freq); + } + + /** Computes a score factor based on a term or phrase's frequency in a + * document. This value is multiplied by the {@link #idf(int, int)} + * factor for each term in the query and these products are then summed to + * form the initial score for a document. + * + *
Terms and phrases repeated in a document indicate the topic of the
+ * document, so implementations of this method usually return larger values
+ * when freq is large, and smaller values when freq
+ * is small.
+ *
+ * @param freq the frequency of a term within a document
+ * @return a score factor based on a term's within-document frequency
+ */
+ public abstract float tf(float freq);
+
+ /**
+ * Computes a score factor for a simple term and returns an explanation
+ * for that score factor.
+ *
+ *
+ * The default implementation uses: + * + *
+ * idf(docFreq, searcher.maxDoc()); + *+ * + * Note that {@link IndexSearcher#maxDoc()} is used instead of + * {@link org.apache.lucene.index.IndexReader#numDocs() IndexReader#numDocs()} because also + * {@link IndexSearcher#docFreq(Term)} is used, and when the latter + * is inaccurate, so is {@link IndexSearcher#maxDoc()}, and in the same direction. + * In addition, {@link IndexSearcher#maxDoc()} is more efficient to compute + * + * @param stats statistics of the term in question + * @param searcher the document collection being searched + * @return an Explain object that includes both an idf score factor + and an explanation for the term. + * @throws IOException + */ + public Explanation idfExplain(TermContext stats, final IndexSearcher searcher) throws IOException { + final int df = stats.docFreq(); + final int max = searcher.maxDoc(); + final float idf = idf(df, max); + return new Explanation(idf, "idf(docFreq=" + df + ", maxDocs=" + max + ")"); + } + + /** + * Computes a score factor for a phrase. + * + *
+ * The default implementation sums the idf factor for + * each term in the phrase. + * + * @param stats statistics of the terms in the phrase + * @param searcher the document collection being searched + * @return an Explain object that includes both an idf + * score factor for the phrase and an explanation + * for each term. + * @throws IOException + */ + public Explanation idfExplain(final TermContext stats[], IndexSearcher searcher) throws IOException { + final int max = searcher.maxDoc(); + float idf = 0.0f; + final Explanation exp = new Explanation(); + exp.setDescription("idf(), sum of:"); + for (final TermContext stat : stats ) { + final int df = stat.docFreq(); + final float termIdf = idf(df, max); + exp.addDetail(new Explanation(termIdf, "idf(docFreq=" + df + ", maxDocs=" + max + ")")); + idf += termIdf; + } + exp.setValue(idf); + return exp; + } + + /** Computes a score factor based on a term's document frequency (the number + * of documents which contain the term). This value is multiplied by the + * {@link #tf(int)} factor for each term in the query and these products are + * then summed to form the initial score for a document. + * + *
Terms that occur in fewer documents are better indicators of topic, so + * implementations of this method usually return larger values for rare terms, + * and smaller values for common terms. + * + * @param docFreq the number of documents which contain the term + * @param numDocs the total number of documents in the collection + * @return a score factor based on the term's document frequency + */ + public abstract float idf(int docFreq, int numDocs); + + /** Cache of decoded bytes. */ + private static final float[] NORM_TABLE = new float[256]; + + static { + for (int i = 0; i < 256; i++) + NORM_TABLE[i] = SmallFloat.byte315ToFloat((byte)i); + } + + /** Decodes a normalization factor stored in an index. + * @see #encodeNormValue(float) + */ + public float decodeNormValue(byte b) { + return NORM_TABLE[b & 0xFF]; // & 0xFF maps negative bytes to positive above 127 + } + + /** Encodes a normalization factor for storage in an index. + * + *
The encoding uses a three-bit mantissa, a five-bit exponent, and + * the zero-exponent point at 15, thus + * representing values from around 7x10^9 to 2x10^-9 with about one + * significant decimal digit of accuracy. Zero is also represented. + * Negative numbers are rounded up to zero. Values too large to represent + * are rounded down to the largest representable value. Positive values too + * small to represent are rounded up to the smallest positive representable + * value. + * @see org.apache.lucene.document.Field#setBoost(float) + * @see org.apache.lucene.util.SmallFloat + */ + public byte encodeNormValue(float f) { + return SmallFloat.floatToByte315(f); + } + + /** Computes the amount of a sloppy phrase match, based on an edit distance. + * This value is summed for each sloppy phrase match in a document to form + * the frequency to be used in scoring instead of the exact term count. + * + *
A phrase match with a small edit distance to a document passage more + * closely matches the document, so implementations of this method usually + * return larger values when the edit distance is small and smaller values + * when it is large. + * + * @see PhraseQuery#setSlop(int) + * @param distance the edit distance of this sloppy phrase match + * @return the frequency increment for this match + */ + public abstract float sloppyFreq(int distance); + + /** + * Calculate a scoring factor based on the data in the payload. Implementations + * are responsible for interpreting what is in the payload. Lucene makes no assumptions about + * what is in the byte array. + * + * @param doc The docId currently being scored. + * @param start The start position of the payload + * @param end The end position of the payload + * @param payload The payload byte array to be scored + * @return An implementation dependent float to be used as a scoring factor + */ + public abstract float scorePayload(int doc, int start, int end, BytesRef payload); + + @Override + public final Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, + TermContext... termContexts) throws IOException { + final Explanation idf = termContexts.length == 1 + ? idfExplain(termContexts[0], searcher) + : idfExplain(termContexts, searcher); + return new IDFStats(idf, queryBoost); + } + + @Override + public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { + return new ExactTFIDFDocScorer((IDFStats)stats, context.reader.norms(fieldName)); + } + + @Override + public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { + return new SloppyTFIDFDocScorer((IDFStats)stats, context.reader.norms(fieldName)); + } + + // TODO: we can specialize these for omitNorms up front, but we should test that it doesn't confuse stupid hotspot. + + private final class ExactTFIDFDocScorer extends ExactDocScorer { + private final IDFStats stats; + private final float weightValue; + private final byte[] norms; + private static final int SCORE_CACHE_SIZE = 32; + private float[] scoreCache = new float[SCORE_CACHE_SIZE]; + + ExactTFIDFDocScorer(IDFStats stats, byte norms[]) { + this.stats = stats; + this.weightValue = stats.value; + this.norms = norms; + for (int i = 0; i < SCORE_CACHE_SIZE; i++) + scoreCache[i] = tf(i) * weightValue; + } + + @Override + public float score(int doc, int freq) { + final float raw = // compute tf(f)*weight + freq < SCORE_CACHE_SIZE // check cache + ? scoreCache[freq] // cache hit + : tf(freq)*weightValue; // cache miss + + return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field + } + + @Override + public Explanation explain(int doc, Explanation freq) { + return explainScore(doc, freq, stats, norms); + } + } + + private final class SloppyTFIDFDocScorer extends SloppyDocScorer { + private final IDFStats stats; + private final float weightValue; + private final byte[] norms; + + SloppyTFIDFDocScorer(IDFStats stats, byte norms[]) { + this.stats = stats; + this.weightValue = stats.value; + this.norms = norms; + } + + @Override + public float score(int doc, float freq) { + final float raw = tf(freq) * weightValue; // compute tf(f)*weight + + return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field + } + + @Override + public float computeSlopFactor(int distance) { + return sloppyFreq(distance); + } + + @Override + public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { + return scorePayload(doc, start, end, payload); + } + + @Override + public Explanation explain(int doc, Explanation freq) { + return explainScore(doc, freq, stats, norms); + } + } + + /** Collection statistics for the TF-IDF model. The only statistic of interest + * to this model is idf. */ + private static class IDFStats extends Stats { + /** The idf and its explanation */ + private final Explanation idf; + private float queryNorm; + private float queryWeight; + private final float queryBoost; + private float value; + + public IDFStats(Explanation idf, float queryBoost) { + // TODO: Validate? + this.idf = idf; + this.queryBoost = queryBoost; + this.queryWeight = idf.getValue() * queryBoost; // compute query weight + } + + @Override + public float getValueForNormalization() { + // TODO: (sorta LUCENE-1907) make non-static class and expose this squaring via a nice method to subclasses? + return queryWeight * queryWeight; // sum of squared weights + } + + @Override + public void normalize(float queryNorm, float topLevelBoost) { + this.queryNorm = queryNorm * topLevelBoost; + queryWeight *= this.queryNorm; // normalize query weight + value = queryWeight * idf.getValue(); // idf for document + } + } + + private Explanation explainScore(int doc, Explanation freq, IDFStats stats, byte[] norms) { + Explanation result = new Explanation(); + result.setDescription("score(doc="+doc+",freq="+freq+"), product of:"); + + // explain query weight + Explanation queryExpl = new Explanation(); + queryExpl.setDescription("queryWeight, product of:"); + + Explanation boostExpl = new Explanation(stats.queryBoost, "boost"); + if (stats.queryBoost != 1.0f) + queryExpl.addDetail(boostExpl); + queryExpl.addDetail(stats.idf); + + Explanation queryNormExpl = new Explanation(stats.queryNorm,"queryNorm"); + queryExpl.addDetail(queryNormExpl); + + queryExpl.setValue(boostExpl.getValue() * + stats.idf.getValue() * + queryNormExpl.getValue()); + + result.addDetail(queryExpl); + + // explain field weight + Explanation fieldExpl = new Explanation(); + fieldExpl.setDescription("fieldWeight in "+doc+ + ", product of:"); + + Explanation tfExplanation = new Explanation(); + tfExplanation.setValue(tf(freq.getValue())); + tfExplanation.setDescription("tf(freq="+freq.getValue()+"), with freq of:"); + tfExplanation.addDetail(freq); + fieldExpl.addDetail(tfExplanation); + fieldExpl.addDetail(stats.idf); + + Explanation fieldNormExpl = new Explanation(); + float fieldNorm = + norms!=null ? decodeNormValue(norms[doc]) : 1.0f; + fieldNormExpl.setValue(fieldNorm); + fieldNormExpl.setDescription("fieldNorm(doc="+doc+")"); + fieldExpl.addDetail(fieldNormExpl); + + fieldExpl.setValue(tfExplanation.getValue() * + stats.idf.getValue() * + fieldNormExpl.getValue()); + + result.addDetail(fieldExpl); + + // combine them + result.setValue(queryExpl.getValue() * fieldExpl.getValue()); + + if (queryExpl.getValue() == 1.0f) + return fieldExpl; + + return result; + } +} diff --git lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java index 573c91b..5456012 100644 --- lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java +++ lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java @@ -20,10 +20,10 @@ package org.apache.lucene.search.spans; import java.io.IOException; import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.TFIDFSimilarity; import org.apache.lucene.search.Weight; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.TFIDFSimilarity; /** * Public for extension only. diff --git lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java index 14dba75..e741547 100644 --- lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java +++ lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java @@ -21,7 +21,8 @@ import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexReader.ReaderContext; import org.apache.lucene.index.Term; import org.apache.lucene.search.*; -import org.apache.lucene.search.Similarity.SloppyDocScorer; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer; import org.apache.lucene.util.TermContext; import java.io.IOException; diff --git lucene/src/test-framework/org/apache/lucene/index/DocHelper.java lucene/src/test-framework/org/apache/lucene/index/DocHelper.java index e0cc6ee..3dc5482 100644 --- lucene/src/test-framework/org/apache/lucene/index/DocHelper.java +++ lucene/src/test-framework/org/apache/lucene/index/DocHelper.java @@ -30,7 +30,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Fieldable; import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.search.SimilarityProvider; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; diff --git lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index 165aee9..1ce6503 100644 --- lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -33,13 +33,13 @@ import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.NumericField; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.NumericRangeQuery; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; diff --git lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java index 9c4994e..89f3e85 100644 --- lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java +++ lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java @@ -27,11 +27,11 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; diff --git lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java index a87bcb9..b38b7dc 100644 --- lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java +++ lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java @@ -18,8 +18,8 @@ package org.apache.lucene.index; */ import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.search.DefaultSimilarity; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; diff --git lucene/src/test/org/apache/lucene/index/TestIndexReader.java lucene/src/test/org/apache/lucene/index/TestIndexReader.java index 7965406..39e7a48 100644 --- lucene/src/test/org/apache/lucene/index/TestIndexReader.java +++ lucene/src/test/org/apache/lucene/index/TestIndexReader.java @@ -39,13 +39,13 @@ import org.apache.lucene.document.SetBasedFieldSelector; import org.apache.lucene.index.IndexReader.FieldOption; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldCache; -import org.apache.lucene.search.Similarity; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; diff --git lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java index 9b6c4d2..0968e9c 100644 --- lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java @@ -18,8 +18,8 @@ package org.apache.lucene.index; */ import org.apache.lucene.index.SegmentNorms; -import org.apache.lucene.search.DefaultSimilarity; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; diff --git lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java index 228d033..854524a 100644 --- lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java @@ -30,10 +30,10 @@ import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.SegmentNorms; -import org.apache.lucene.search.DefaultSimilarity; -import org.apache.lucene.search.DefaultSimilarityProvider; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.search.SimilarityProvider; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; diff --git lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java index 058939e..e3715a4 100644 --- lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java @@ -22,11 +22,11 @@ import java.io.IOException; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Similarity; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; diff --git lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java index 3200204..2d13327 100644 --- lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java @@ -35,13 +35,13 @@ import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BitVector; diff --git lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java index d501694..4a04c08 100644 --- lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java @@ -26,8 +26,8 @@ import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.search.DefaultSimilarityProvider; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.util.LuceneTestCase; import org.junit.Test; diff --git lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java index 9744008..96a3ec1 100644 --- lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java +++ lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java @@ -26,9 +26,9 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.search.DefaultSimilarity; -import org.apache.lucene.search.DefaultSimilarityProvider; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; diff --git lucene/src/test/org/apache/lucene/index/TestNorms.java lucene/src/test/org/apache/lucene/index/TestNorms.java index 372ae2e..6b5a1a8 100755 --- lucene/src/test/org/apache/lucene/index/TestNorms.java +++ lucene/src/test/org/apache/lucene/index/TestNorms.java @@ -28,10 +28,10 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.search.DefaultSimilarity; -import org.apache.lucene.search.DefaultSimilarityProvider; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.search.SimilarityProvider; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; diff --git lucene/src/test/org/apache/lucene/index/TestOmitTf.java lucene/src/test/org/apache/lucene/index/TestOmitTf.java index d9bae53..9d38fd3 100644 --- lucene/src/test/org/apache/lucene/index/TestOmitTf.java +++ lucene/src/test/org/apache/lucene/index/TestOmitTf.java @@ -31,6 +31,9 @@ import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.search.*; import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; +import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.store.Directory; diff --git lucene/src/test/org/apache/lucene/index/TestParallelReader.java lucene/src/test/org/apache/lucene/index/TestParallelReader.java index b2d0b3c..c352037 100644 --- lucene/src/test/org/apache/lucene/index/TestParallelReader.java +++ lucene/src/test/org/apache/lucene/index/TestParallelReader.java @@ -27,6 +27,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.MapFieldSelector; import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; diff --git lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java index 3fd64fa..3b7b609 100644 --- lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java +++ lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java @@ -24,9 +24,9 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.search.DefaultSimilarity; -import org.apache.lucene.search.DefaultSimilarityProvider; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; diff --git lucene/src/test/org/apache/lucene/search/JustCompileSearch.java lucene/src/test/org/apache/lucene/search/JustCompileSearch.java index a76dbdc..f1f49bc 100644 --- lucene/src/test/org/apache/lucene/search/JustCompileSearch.java +++ lucene/src/test/org/apache/lucene/search/JustCompileSearch.java @@ -20,9 +20,11 @@ package org.apache.lucene.search; import java.io.IOException; import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.search.Similarity.ExactDocScorer; -import org.apache.lucene.search.Similarity.SloppyDocScorer; -import org.apache.lucene.search.Similarity.Stats; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; +import org.apache.lucene.search.similarities.Similarity.ExactDocScorer; +import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer; +import org.apache.lucene.search.similarities.Similarity.Stats; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.TermContext; import org.apache.lucene.index.FieldInvertState; diff --git lucene/src/test/org/apache/lucene/search/TestBoolean2.java lucene/src/test/org/apache/lucene/search/TestBoolean2.java index 2a460a5..7365d28 100644 --- lucene/src/test/org/apache/lucene/search/TestBoolean2.java +++ lucene/src/test/org/apache/lucene/search/TestBoolean2.java @@ -26,6 +26,8 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.MockDirectoryWrapper; diff --git lucene/src/test/org/apache/lucene/search/TestComplexExplanations.java lucene/src/test/org/apache/lucene/search/TestComplexExplanations.java index eeaa4ff..4e0411f 100644 --- lucene/src/test/org/apache/lucene/search/TestComplexExplanations.java +++ lucene/src/test/org/apache/lucene/search/TestComplexExplanations.java @@ -19,6 +19,7 @@ package org.apache.lucene.search; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.search.spans.*; /** diff --git lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java index 35b388e..a8343a2 100644 --- lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java +++ lucene/src/test/org/apache/lucene/search/TestConstantScoreQuery.java @@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; diff --git lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java index 71b96a4..6e8eba0 100644 --- lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java +++ lucene/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java @@ -28,6 +28,10 @@ import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.search.Weight.ScorerContext; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import java.text.DecimalFormat; diff --git lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java index fe642d1..5ddee5e 100644 --- lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java +++ lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java @@ -29,6 +29,8 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.values.IndexDocValues.Source; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; diff --git lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java index b5ef9fd..b696572 100644 --- lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java +++ lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java @@ -35,6 +35,9 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.Similarity; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; diff --git lucene/src/test/org/apache/lucene/search/TestSetNorm.java lucene/src/test/org/apache/lucene/search/TestSetNorm.java index 72245e1..64eba46 100644 --- lucene/src/test/org/apache/lucene/search/TestSetNorm.java +++ lucene/src/test/org/apache/lucene/search/TestSetNorm.java @@ -26,6 +26,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.store.Directory; /** Document boost unit test. diff --git lucene/src/test/org/apache/lucene/search/TestSimilarity.java lucene/src/test/org/apache/lucene/search/TestSimilarity.java index 55c6224..e89e699 100644 --- lucene/src/test/org/apache/lucene/search/TestSimilarity.java +++ lucene/src/test/org/apache/lucene/search/TestSimilarity.java @@ -27,6 +27,9 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; diff --git lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java index 42f5046..d92d547 100644 --- lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java +++ lucene/src/test/org/apache/lucene/search/TestSimilarityProvider.java @@ -26,6 +26,9 @@ import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.MultiNorms; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; +import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; diff --git lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java index 501dd4d..a3e3a5a 100644 --- lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java +++ lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java @@ -28,7 +28,7 @@ import org.apache.lucene.util.English; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.index.IndexReader; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.SimilarityProvider; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.RAMDirectory; diff --git lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java index 2e1da3a..edfeb21 100644 --- lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java +++ lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java @@ -30,14 +30,14 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Payload; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.QueryUtils; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.search.spans.SpanQuery; import org.apache.lucene.search.spans.SpanNearQuery; import org.apache.lucene.search.spans.SpanTermQuery; diff --git lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java index 0a4ad06..ec9ba98 100644 --- lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java +++ lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java @@ -19,17 +19,17 @@ package org.apache.lucene.search.payloads; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.English; -import org.apache.lucene.search.DefaultSimilarityProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.QueryUtils; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.CheckHits; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.search.spans.MultiSpansWrapper; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.search.spans.Spans; diff --git lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java index 0f0dcfd..02d7dd1 100644 --- lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java +++ lucene/src/test/org/apache/lucene/search/similarities/TestEasySimilarity.java @@ -31,7 +31,6 @@ import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; -import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; diff --git lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java index 9b2ee4b..8611056 100644 --- lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java +++ lucene/src/test/org/apache/lucene/search/spans/JustCompileSearchSpans.java @@ -22,7 +22,7 @@ import java.util.Collection; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.search.Weight; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.similarities.Similarity; /** * Holds all implementations of classes in the o.a.l.s.spans package as a diff --git lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java index 1e1d2ca..2d8ec2b 100644 --- lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java +++ lucene/src/test/org/apache/lucene/search/spans/TestPayloadSpans.java @@ -37,13 +37,13 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Payload; import org.apache.lucene.index.Term; -import org.apache.lucene.search.DefaultSimilarityProvider; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.payloads.PayloadHelper; import org.apache.lucene.search.payloads.PayloadSpanUtil; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.LuceneTestCase; diff --git lucene/src/test/org/apache/lucene/search/spans/TestSpans.java lucene/src/test/org/apache/lucene/search/spans/TestSpans.java index 393eb0e..4544936 100644 --- lucene/src/test/org/apache/lucene/search/spans/TestSpans.java +++ lucene/src/test/org/apache/lucene/search/spans/TestSpans.java @@ -17,17 +17,17 @@ package org.apache.lucene.search.spans; * limitations under the License. */ -import org.apache.lucene.search.DefaultSimilarityProvider; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.Query; import org.apache.lucene.search.CheckHits; -import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.Scorer; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Weight.ScorerContext; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.index.IndexReader.ReaderContext; diff --git modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java index b6a5341..259ddaa 100755 --- modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java +++ modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java @@ -21,8 +21,8 @@ import org.apache.lucene.index.*; import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.queries.function.DocValues; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.search.TFIDFSimilarity; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.util.BytesRef; import java.io.IOException; diff --git modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java index f2b5436..81fa80e 100755 --- modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java +++ modules/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java @@ -22,8 +22,8 @@ import org.apache.lucene.queries.function.DocValues; import org.apache.lucene.queries.function.ValueSource; import org.apache.lucene.queries.function.docvalues.FloatDocValues; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.search.TFIDFSimilarity; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.TFIDFSimilarity; import java.io.IOException; import java.util.Map; diff --git modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java index 90b605b..42bfb1c 100755 --- modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java +++ modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java @@ -23,8 +23,8 @@ import org.apache.lucene.queries.function.DocValues; import org.apache.lucene.queries.function.docvalues.FloatDocValues; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.search.TFIDFSimilarity; +import org.apache.lucene.search.similarities.Similarity; +import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.util.BytesRef; import java.io.IOException; diff --git modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java index 3134826..322de1d 100644 --- modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java +++ modules/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java @@ -23,6 +23,8 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermFreqVector; import org.apache.lucene.search.*; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRef; diff --git modules/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardBooleanQueryNodeBuilder.java modules/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardBooleanQueryNodeBuilder.java index 04616be..5a4b09e 100644 --- modules/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardBooleanQueryNodeBuilder.java +++ modules/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/builders/StandardBooleanQueryNodeBuilder.java @@ -31,8 +31,8 @@ import org.apache.lucene.queryparser.flexible.standard.parser.EscapeQuerySyntaxI import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.Query; -import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.search.BooleanQuery.TooManyClauses; +import org.apache.lucene.search.similarities.SimilarityProvider; /** * This builder does the same as the {@link BooleanQueryNodeBuilder}, but this diff --git modules/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/StandardBooleanQueryNode.java modules/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/StandardBooleanQueryNode.java index b72dd04..aceb281 100644 --- modules/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/StandardBooleanQueryNode.java +++ modules/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/nodes/StandardBooleanQueryNode.java @@ -22,7 +22,7 @@ import java.util.List; import org.apache.lucene.queryparser.flexible.core.nodes.BooleanQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.SimilarityProvider; +import org.apache.lucene.search.similarities.SimilarityProvider; /** * A {@link StandardBooleanQueryNode} has the same behavior as diff --git solr/client/ruby/solr-ruby/solr/conf/schema.xml solr/client/ruby/solr-ruby/solr/conf/schema.xml index bbd360a..daac616 100755 --- solr/client/ruby/solr-ruby/solr/conf/schema.xml +++ solr/client/ruby/solr-ruby/solr/conf/schema.xml @@ -217,6 +217,6 @@ - + diff --git solr/client/ruby/solr-ruby/test/conf/schema.xml solr/client/ruby/solr-ruby/test/conf/schema.xml index fafe6c1..a284b29 100755 --- solr/client/ruby/solr-ruby/test/conf/schema.xml +++ solr/client/ruby/solr-ruby/test/conf/schema.xml @@ -235,6 +235,6 @@ - + diff --git solr/contrib/clustering/src/test-files/solr-clustering/conf/schema.xml solr/contrib/clustering/src/test-files/solr-clustering/conf/schema.xml index 1a00116..ca35ffe 100644 --- solr/contrib/clustering/src/test-files/solr-clustering/conf/schema.xml +++ solr/contrib/clustering/src/test-files/solr-clustering/conf/schema.xml @@ -335,7 +335,7 @@ - + - + diff --git solr/core/src/test-files/solr/conf/schema-required-fields.xml solr/core/src/test-files/solr/conf/schema-required-fields.xml index f179484..c5dec55 100644 --- solr/core/src/test-files/solr/conf/schema-required-fields.xml +++ solr/core/src/test-files/solr/conf/schema-required-fields.xml @@ -434,6 +434,6 @@ A custom similarity may be specified here, but the default is fine for most applications. --> - + diff --git solr/core/src/test/org/apache/solr/schema/CustomSimilarityFactory.java solr/core/src/test/org/apache/solr/schema/CustomSimilarityFactory.java index 98267cc..782bf6b 100644 --- solr/core/src/test/org/apache/solr/schema/CustomSimilarityFactory.java +++ solr/core/src/test/org/apache/solr/schema/CustomSimilarityFactory.java @@ -16,7 +16,7 @@ */ package org.apache.solr.schema; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.similarities.Similarity; public class CustomSimilarityFactory extends SimilarityFactory { @Override diff --git solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java index 737cb90..5a0a9f0 100644 --- solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java +++ solr/core/src/test/org/apache/solr/schema/IndexSchemaTest.java @@ -27,7 +27,7 @@ import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.core.SolrCore; import org.apache.solr.request.LocalSolrQueryRequest; import org.apache.solr.request.SolrQueryRequest; -import org.apache.lucene.search.SimilarityProvider; +import org.apache.lucene.search.similarities.SimilarityProvider; import org.junit.BeforeClass; import org.junit.Test; diff --git solr/core/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java solr/core/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java index 36021fa..74394c4 100644 --- solr/core/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java +++ solr/core/src/test/org/apache/solr/schema/MockConfigurableSimilarity.java @@ -16,7 +16,7 @@ */ package org.apache.solr.schema; -import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarity; public class MockConfigurableSimilarity extends DefaultSimilarity { private String passthrough; diff --git solr/core/src/test/org/apache/solr/schema/TestPerFieldSimilarity.java solr/core/src/test/org/apache/solr/schema/TestPerFieldSimilarity.java index 3aa81c3..1ab5025 100644 --- solr/core/src/test/org/apache/solr/schema/TestPerFieldSimilarity.java +++ solr/core/src/test/org/apache/solr/schema/TestPerFieldSimilarity.java @@ -18,8 +18,8 @@ package org.apache.solr.schema; */ import org.apache.lucene.misc.SweetSpotSimilarity; -import org.apache.lucene.search.DefaultSimilarity; -import org.apache.lucene.search.Similarity; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.Similarity; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.core.SolrCore; import org.apache.solr.search.SolrIndexSearcher; diff --git solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java index 38ab99e..c66c69f 100755 --- solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java +++ solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java @@ -19,9 +19,9 @@ package org.apache.solr.search.function; import org.apache.lucene.index.FieldInvertState; import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.search.DefaultSimilarity; import org.apache.lucene.search.FieldCache; -import org.apache.lucene.search.TFIDFSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.TFIDFSimilarity; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.params.SolrParams; import org.apache.solr.common.util.NamedList; diff --git solr/example/example-DIH/solr/db/conf/schema.xml solr/example/example-DIH/solr/db/conf/schema.xml index 24718d3..f110064 100644 --- solr/example/example-DIH/solr/db/conf/schema.xml +++ solr/example/example-DIH/solr/db/conf/schema.xml @@ -354,6 +354,6 @@ - + diff --git solr/example/solr/conf/schema.xml solr/example/solr/conf/schema.xml index dc5ae9a..b17500a 100755 --- solr/example/solr/conf/schema.xml +++ solr/example/solr/conf/schema.xml @@ -615,7 +615,7 @@ - +