The boost is used to compute the norm factor for the field. By
- * default, in the {@link org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)} method,
+ * default, in the {@link org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState)} method,
* the boost value is multiplied by the length normalization factor and then
- * rounded by {@link org.apache.lucene.search.DefaultSimilarity#encodeNormValue(float)} before it is stored in the
+ * rounded by {@link org.apache.lucene.search.similarities.DefaultSimilarity#encodeNormValue(float)} before it is stored in the
* index. One should attempt to ensure that this product does not overflow
* the range of that encoding.
*
- * @see org.apache.lucene.search.Similarity#computeNorm(FieldInvertState)
- * @see org.apache.lucene.search.DefaultSimilarity#encodeNormValue(float)
+ * @see org.apache.lucene.search.similarities.Similarity#computeNorm(FieldInvertState)
+ * @see org.apache.lucene.search.similarities.DefaultSimilarity#encodeNormValue(float)
*/
public void setBoost(float boost) {
this.boost = boost;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java lucene-flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java 2011-08-29 12:22:46.861459551 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java 2011-08-29 13:45:06.071459544 -0400
@@ -32,7 +32,7 @@
import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState;
import org.apache.lucene.index.FieldInfos.FieldNumberBiMap;
import org.apache.lucene.search.Query;
-import org.apache.lucene.search.SimilarityProvider;
+import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java lucene-flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java 2011-09-08 17:01:39.526036271 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java 2011-09-08 12:56:47.666030849 -0400
@@ -26,7 +26,7 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice;
-import org.apache.lucene.search.SimilarityProvider;
+import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FlushInfo;
import org.apache.lucene.store.IOContext;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java lucene-flexscoring/lucene/src/java/org/apache/lucene/index/IndexReader.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java 2011-09-08 17:01:39.526036271 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/index/IndexReader.java 2011-09-08 12:56:57.076030851 -0400
@@ -32,7 +32,7 @@
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.values.IndexDocValues;
import org.apache.lucene.search.FieldCache; // javadocs
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.*;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
@@ -1012,7 +1012,7 @@
*
* @see #norms(String)
* @see Similarity#computeNorm(FieldInvertState)
- * @see org.apache.lucene.search.DefaultSimilarity#decodeNormValue(byte)
+ * @see org.apache.lucene.search.similarities.DefaultSimilarity#decodeNormValue(byte)
* @throws StaleReaderException if the index has changed
* since this reader was opened
* @throws CorruptIndexException if the index is corrupt
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java lucene-flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java 2011-07-27 00:16:39.302328617 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java 2011-08-23 09:45:29.771460290 -0400
@@ -22,7 +22,7 @@
import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.SimilarityProvider;
+import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.util.Version;
/**
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java lucene-flexscoring/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java 2011-07-08 01:31:28.421820268 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/index/NormsWriterPerField.java 2011-08-23 08:43:39.621460295 -0400
@@ -17,7 +17,7 @@
* limitations under the License.
*/
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.ArrayUtil;
/** Taps into DocInverter, as an InvertedDocEndConsumer,
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/BooleanQuery.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/BooleanQuery.java 2011-07-24 19:00:25.772328885 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/BooleanQuery.java 2011-08-23 09:52:56.061460289 -0400
@@ -24,7 +24,8 @@
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.ConjunctionTermScorer.DocsAndFreqs;
-import org.apache.lucene.search.Similarity.ExactDocScorer;
+import org.apache.lucene.search.similarities.SimilarityProvider;
+import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
import org.apache.lucene.search.TermQuery.TermWeight;
import java.io.IOException;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java 2011-08-17 23:39:13.791460945 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java 2011-08-23 09:53:45.661460289 -0400
@@ -24,6 +24,7 @@
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery.BooleanWeight;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.Scorer.ChildScorer;
/* See the description in BooleanScorer.java, comparing
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java 2011-07-24 19:00:25.772328885 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java 2011-08-23 09:53:19.801460289 -0400
@@ -18,7 +18,7 @@
*/
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.search.Similarity.ExactDocScorer;
+import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
import org.apache.lucene.util.ArrayUtil;
import java.io.IOException;
import java.util.Comparator;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java 2011-07-17 06:27:33.748948135 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/DefaultSimilarity.java 1969-12-31 19:00:00.000000000 -0500
@@ -1,88 +0,0 @@
-package org.apache.lucene.search;
-
-import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.util.BytesRef;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/** Expert: Default scoring implementation. */
-public class DefaultSimilarity extends TFIDFSimilarity {
-
- /** Implemented as
- * state.getBoost()*lengthNorm(numTerms), where
- * numTerms is {@link FieldInvertState#getLength()} if {@link
- * #setDiscountOverlaps} is false, else it's {@link
- * FieldInvertState#getLength()} - {@link
- * FieldInvertState#getNumOverlap()}.
- *
- * @lucene.experimental */
- @Override
- public byte computeNorm(FieldInvertState state) {
- final int numTerms;
- if (discountOverlaps)
- numTerms = state.getLength() - state.getNumOverlap();
- else
- numTerms = state.getLength();
- return encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms))));
- }
-
- /** Implemented as sqrt(freq). */
- @Override
- public float tf(float freq) {
- return (float)Math.sqrt(freq);
- }
-
- /** Implemented as 1 / (distance + 1). */
- @Override
- public float sloppyFreq(int distance) {
- return 1.0f / (distance + 1);
- }
-
- /** The default implementation returns 1 */
- @Override
- public float scorePayload(int doc, int start, int end, BytesRef payload) {
- return 1;
- }
-
- /** Implemented as log(numDocs/(docFreq+1)) + 1. */
- @Override
- public float idf(int docFreq, int numDocs) {
- return (float)(Math.log(numDocs/(double)(docFreq+1)) + 1.0);
- }
-
- // Default true
- protected boolean discountOverlaps = true;
-
- /** Determines whether overlap tokens (Tokens with
- * 0 position increment) are ignored when computing
- * norm. By default this is true, meaning overlap
- * tokens do not count when computing norms.
- *
- * @lucene.experimental
- *
- * @see #computeNorm
- */
- public void setDiscountOverlaps(boolean v) {
- discountOverlaps = v;
- }
-
- /** @see #setDiscountOverlaps */
- public boolean getDiscountOverlaps() {
- return discountOverlaps;
- }
-}
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/DefaultSimilarityProvider.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/DefaultSimilarityProvider.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/DefaultSimilarityProvider.java 2011-05-08 10:52:19.350078710 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/DefaultSimilarityProvider.java 1969-12-31 19:00:00.000000000 -0500
@@ -1,41 +0,0 @@
-package org.apache.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Expert: Default scoring provider.
- *
- * Returns {@link DefaultSimilarity} for every field
- */
-public class DefaultSimilarityProvider implements SimilarityProvider {
- private static final Similarity impl = new DefaultSimilarity();
-
- /** Implemented as overlap / maxOverlap. */
- public float coord(int overlap, int maxOverlap) {
- return overlap / (float)maxOverlap;
- }
-
- /** Implemented as 1/sqrt(sumOfSquaredWeights). */
- public float queryNorm(float sumOfSquaredWeights) {
- return (float)(1.0 / Math.sqrt(sumOfSquaredWeights));
- }
-
- public Similarity get(String field) {
- return impl;
- }
-}
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java 2011-07-08 01:31:28.421820268 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/ExactPhraseScorer.java 2011-08-23 08:43:39.661460295 -0400
@@ -21,6 +21,7 @@
import java.util.Arrays;
import org.apache.lucene.index.*;
+import org.apache.lucene.search.similarities.Similarity;
final class ExactPhraseScorer extends Scorer {
private final int endMinus1;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/IndexSearcher.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/IndexSearcher.java 2011-08-29 12:22:46.711459551 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/IndexSearcher.java 2011-08-29 13:44:26.011459544 -0400
@@ -38,6 +38,8 @@
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Weight.ScorerContext;
+import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
+import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory; // javadoc
import org.apache.lucene.util.ReaderUtil;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java 2011-07-20 07:05:53.828132695 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java 2011-08-30 00:42:46.261459489 -0400
@@ -26,7 +26,8 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.DocsAndPositionsEnum;
-import org.apache.lucene.search.Similarity.SloppyDocScorer;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
@@ -164,8 +165,7 @@
@Override
public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- if (termArrays.size() == 0) // optimize zero-term case
- return null;
+ assert !termArrays.isEmpty();
final IndexReader reader = context.reader;
final Bits liveDocs = reader.getLiveDocs();
@@ -249,7 +249,11 @@
@Override
public Query rewrite(IndexReader reader) {
- if (termArrays.size() == 1) { // optimize one-term case
+ if (termArrays.isEmpty()) {
+ BooleanQuery bq = new BooleanQuery();
+ bq.setBoost(getBoost());
+ return bq;
+ } else if (termArrays.size() == 1) { // optimize one-term case
Term[] terms = termArrays.get(0);
BooleanQuery boq = new BooleanQuery(true);
for (int i=0; i
+ * WARNING: for terms that do not meet the expected random distribution + * (e.g. stopwords), this model may give poor performance, such as + * abnormally high scores for low tf values. + * @lucene.experimental + */ +public class BasicModelD extends BasicModel { + @Override + public final float score(BasicStats stats, float tfn) { + // we have to ensure phi is always < 1 for tiny TTF values, otherwise nphi can go negative, + // resulting in NaN. cleanest way is to unconditionally always add tfn to totalTermFreq + // to create a 'normalized' F. + double F = stats.getTotalTermFreq() + tfn; + double phi = (double)tfn / F; + double nphi = 1 - phi; + double p = 1.0 / (stats.getNumberOfDocuments() + 1); + double D = phi * log2(phi / p) + nphi * log2(nphi / (1 - p)); + return (float)(D * F + 0.5 * log2(1 + 2 * Math.PI * tfn * nphi)); + } + + @Override + public String toString() { + return "D"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModelG.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModelG.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModelG.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModelG.java 2011-09-07 00:13:13.635982055 -0400 @@ -0,0 +1,41 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.SimilarityBase.log2; + +/** + * Geometric as limiting form of the Bose-Einstein model. The formula used in Lucene differs + * slightly from the one in the original paper: {@code F} is increased by {@code tfn} + * and {@code N} is increased by {@code F}. + * @lucene.experimental + */ +public class BasicModelG extends BasicModel { + @Override + public final float score(BasicStats stats, float tfn) { + // just like in BE, approximation only holds true when F << N, so we use lambda = F / (N + F) + double lambda = stats.getTotalTermFreq() / (double) (stats.getNumberOfDocuments() + stats.getTotalTermFreq()); + // -log(1 / (lambda + 1)) -> log(lambda + 1) + return (float)(log2(lambda + 1) + tfn * log2((1 + lambda) / lambda)); + } + + @Override + public String toString() { + return "G"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIF.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIF.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIF.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIF.java 2011-08-23 09:16:32.701460292 -0400 @@ -0,0 +1,38 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.SimilarityBase.log2; + +/** + * An approximation of the I(ne) model. + * @lucene.experimental + */ +public class BasicModelIF extends BasicModel { + @Override + public final float score(BasicStats stats, float tfn) { + int N = stats.getNumberOfDocuments(); + long F = stats.getTotalTermFreq(); + return tfn * (float)(log2(1 + (N + 1) / (F + 0.5))); + } + + @Override + public String toString() { + return "I(F)"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIne.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIne.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIne.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIne.java 2011-08-23 09:16:32.701460292 -0400 @@ -0,0 +1,40 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.SimilarityBase.log2; + +/** + * Tf-idf model of randomness, based on a mixture of Poisson and inverse + * document frequency. + * @lucene.experimental + */ +public class BasicModelIne extends BasicModel { + @Override + public final float score(BasicStats stats, float tfn) { + int N = stats.getNumberOfDocuments(); + long F = stats.getTotalTermFreq(); + double ne = N * (1 - Math.pow((N - 1) / (double)N, F)); + return tfn * (float)(log2((N + 1) / (ne + 0.5))); + } + + @Override + public String toString() { + return "I(ne)"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIn.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIn.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIn.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModelIn.java 2011-08-23 09:16:32.701460292 -0400 @@ -0,0 +1,52 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; +import static org.apache.lucene.search.similarities.SimilarityBase.log2; + +/** + * The basic tf-idf model of randomness. + * @lucene.experimental + */ +public class BasicModelIn extends BasicModel { + @Override + public final float score(BasicStats stats, float tfn) { + int N = stats.getNumberOfDocuments(); + int n = stats.getDocFreq(); + return tfn * (float)(log2((N + 1) / (n + 0.5))); + } + + @Override + public final Explanation explain(BasicStats stats, float tfn) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(score(stats, tfn)); + result.addDetail(new Explanation(tfn, "tfn")); + result.addDetail( + new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); + result.addDetail( + new Explanation(stats.getDocFreq(), "docFreq")); + return result; + } + + @Override + public String toString() { + return "I(n)"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModel.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModel.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModel.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModel.java 2011-08-23 09:16:32.701460292 -0400 @@ -0,0 +1,60 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * This class acts as the base class for the specific basic model + * implementations in the DFR framework. Basic models compute the + * informative content Inf1 = -log2Prob1 + * . + * + * @see DFRSimilarity + * @lucene.experimental + */ +public abstract class BasicModel { + /** Returns the informative content score. */ + public abstract float score(BasicStats stats, float tfn); + + /** + * Returns an explanation for the score. + *
Most basic models use the number of documents and the total term + * frequency to compute Inf1. This method provides a generic + * explanation for such models. Subclasses that use other statistics must + * override this method.
+ */ + public Explanation explain(BasicStats stats, float tfn) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(score(stats, tfn)); + result.addDetail(new Explanation(tfn, "tfn")); + result.addDetail( + new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); + result.addDetail( + new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); + return result; + } + + /** + * Subclasses must override this method to return the code of the + * basic model formula. Refer to the original paper for the list. + */ + @Override + public abstract String toString(); +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModelP.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModelP.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicModelP.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicModelP.java 2011-09-07 00:10:39.015981997 -0400 @@ -0,0 +1,46 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.SimilarityBase.log2; + +/** + * Implements the Poisson approximation for the binomial model for DFR. + * @lucene.experimental + *+ * WARNING: for terms that do not meet the expected random distribution + * (e.g. stopwords), this model may give poor performance, such as + * abnormally high scores for low tf values. + */ +public class BasicModelP extends BasicModel { + /** {@code log2(Math.E)}, precomputed. */ + protected static double LOG2_E = log2(Math.E); + + @Override + public final float score(BasicStats stats, float tfn) { + float lambda = (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); + return (float)(tfn * log2(tfn / lambda) + + (lambda + 1 / (12 * tfn) - tfn) * LOG2_E + + 0.5 * log2(2 * Math.PI * tfn)); + } + + @Override + public String toString() { + return "P"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicSimilarityProvider.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicSimilarityProvider.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicSimilarityProvider.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicSimilarityProvider.java 2011-08-30 09:11:26.321459446 -0400 @@ -0,0 +1,53 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * A simple {@link Similarity} provider that returns in + * {@code get(String field)} the object passed to its constructor. This class + * is aimed at non-VSM models, and therefore both the {@link #coord} and + * {@link #queryNorm} methods return {@code 1}. Use + * {@link DefaultSimilarityProvider} for {@link DefaultSimilarity}. + */ +public class BasicSimilarityProvider implements SimilarityProvider { + private final Similarity sim; + + public BasicSimilarityProvider(Similarity sim) { + this.sim = sim; + } + + @Override + public float coord(int overlap, int maxOverlap) { + return 1f; + } + + @Override + public float queryNorm(float sumOfSquaredWeights) { + return 1f; + } + + @Override + public Similarity get(String field) { + return sim; + } + + @Override + public String toString() { + return "BasicSimilarityProvider(" + sim + ")"; + } +} \ No newline at end of file diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicStats.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicStats.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BasicStats.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BasicStats.java 2011-08-23 09:16:32.711460292 -0400 @@ -0,0 +1,144 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.Terms; + +/** + * Stores all statistics commonly used ranking methods. + * @lucene.experimental + */ +public class BasicStats extends Similarity.Stats { + /** The number of documents. */ + protected int numberOfDocuments; + /** The total number of tokens in the field. */ + protected long numberOfFieldTokens; + /** The average field length. */ + protected float avgFieldLength; + /** The document frequency. */ + protected int docFreq; + /** The total number of occurrences of this term across all documents. */ + protected long totalTermFreq; + + // -------------------------- Boost-related stuff -------------------------- + + /** Query's inner boost. */ + protected final float queryBoost; + /** Any outer query's boost. */ + protected float topLevelBoost; + /** For most Similarities, the immediate and the top level query boosts are + * not handled differently. Hence, this field is just the product of the + * other two. */ + protected float totalBoost; + + /** Constructor. Sets the query boost. */ + public BasicStats(float queryBoost) { + this.queryBoost = queryBoost; + this.totalBoost = queryBoost; + } + + // ------------------------- Getter/setter methods ------------------------- + + /** Returns the number of documents. */ + public int getNumberOfDocuments() { + return numberOfDocuments; + } + + /** Sets the number of documents. */ + public void setNumberOfDocuments(int numberOfDocuments) { + this.numberOfDocuments = numberOfDocuments; + } + + /** + * Returns the total number of tokens in the field. + * @see Terms#getSumTotalTermFreq() + */ + public long getNumberOfFieldTokens() { + return numberOfFieldTokens; + } + + /** + * Sets the total number of tokens in the field. + * @see Terms#getSumTotalTermFreq() + */ + public void setNumberOfFieldTokens(long numberOfFieldTokens) { + this.numberOfFieldTokens = numberOfFieldTokens; + } + + /** Returns the average field length. */ + public float getAvgFieldLength() { + return avgFieldLength; + } + + /** Sets the average field length. */ + public void setAvgFieldLength(float avgFieldLength) { + this.avgFieldLength = avgFieldLength; + } + + /** Returns the document frequency. */ + public int getDocFreq() { + return docFreq; + } + + /** Sets the document frequency. */ + public void setDocFreq(int docFreq) { + this.docFreq = docFreq; + } + + /** Returns the total number of occurrences of this term across all documents. */ + public long getTotalTermFreq() { + return totalTermFreq; + } + + /** Sets the total number of occurrences of this term across all documents. */ + public void setTotalTermFreq(long totalTermFreq) { + this.totalTermFreq = totalTermFreq; + } + + // -------------------------- Boost-related stuff -------------------------- + + /** The square of the raw normalization value. + * @see #rawNormalizationValue() */ + @Override + public float getValueForNormalization() { + float rawValue = rawNormalizationValue(); + return rawValue * rawValue; + } + + /** Computes the raw normalization value. This basic implementation returns + * the query boost. Subclasses may override this method to include other + * factors (such as idf), or to save the value for inclusion in + * {@link #normalize(float, float)}, etc. + */ + protected float rawNormalizationValue() { + return queryBoost; + } + + /** No normalization is done. {@code topLevelBoost} is saved in the object, + * however. */ + @Override + public void normalize(float queryNorm, float topLevelBoost) { + this.topLevelBoost = topLevelBoost; + totalBoost = queryBoost * topLevelBoost; + } + + /** Returns the total boost. */ + public float getTotalBoost() { + return totalBoost; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java 2011-09-09 11:38:26.186060999 -0400 @@ -0,0 +1,338 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SmallFloat; +import org.apache.lucene.util.TermContext; + +/** + * BM25 Similarity. Introduced in Stephen E. Robertson, Steve Walker, + * Susan Jones, Micheline Hancock-Beaulieu, and Mike Gatford. Okapi at TREC-3. + * In Proceedings of the Third Text REtrieval Conference (TREC 1994). + * Gaithersburg, USA, November 1994. + */ +public class BM25Similarity extends Similarity { + private final float k1; + private final float b; + // TODO: should we add a delta like sifaka.cs.uiuc.edu/~ylv2/pub/sigir11-bm25l.pdf ? + + public BM25Similarity(float k1, float b) { + this.k1 = k1; + this.b = b; + } + + /** BM25 with these default values: + *
log(1 + (numDocs - docFreq + 0.5)/(docFreq + 0.5)). */
+ protected float idf(int docFreq, int numDocs) {
+ return (float) Math.log(1 + (numDocs - docFreq + 0.5D)/(docFreq + 0.5D));
+ }
+
+ /** Implemented as 1 / (distance + 1). */
+ protected float sloppyFreq(int distance) {
+ return 1.0f / (distance + 1);
+ }
+
+ /** The default implementation returns 1 */
+ protected float scorePayload(int doc, int start, int end, BytesRef payload) {
+ return 1;
+ }
+
+ /** The default implementation computes the average as sumTotalTermFreq / maxDoc,
+ * or returns 1 if the index does not store sumTotalTermFreq (Lucene 3.x indexes
+ * or any field that omits frequency information). */
+ protected float avgFieldLength(IndexSearcher searcher, String field) throws IOException {
+ Terms terms = MultiFields.getTerms(searcher.getIndexReader(), field);
+ if (terms == null) {
+ // field does not exist;
+ return 1f;
+ }
+ long sumTotalTermFreq = terms.getSumTotalTermFreq();
+ long maxdoc = searcher.maxDoc();
+ return sumTotalTermFreq == -1 ? 1f : (float) (sumTotalTermFreq / (double) maxdoc);
+ }
+
+ /** The default implementation encodes boost / sqrt(length)
+ * with {@link SmallFloat#floatToByte315(float)}. This is compatible with
+ * Lucene's default implementation. If you change this, then you should
+ * change {@link #decodeNormValue(byte)} to match. */
+ protected byte encodeNormValue(float boost, int fieldLength) {
+ return SmallFloat.floatToByte315(boost / (float) Math.sqrt(fieldLength));
+ }
+
+ /** The default implementation returns 1 / f2
+ * where f is {@link SmallFloat#byte315ToFloat(byte)}. */
+ protected float decodeNormValue(byte b) {
+ return NORM_TABLE[b & 0xFF];
+ }
+
+ // Default true
+ protected boolean discountOverlaps = true;
+
+ /** Determines whether overlap tokens (Tokens with 0 position increment) are
+ * ignored when computing norm. By default this is true, meaning overlap
+ * tokens do not count when computing norms. */
+ public void setDiscountOverlaps(boolean v) {
+ discountOverlaps = v;
+ }
+
+ /** @see #setDiscountOverlaps */
+ public boolean getDiscountOverlaps() {
+ return discountOverlaps;
+ }
+
+ /** Cache of decoded bytes. */
+ private static final float[] NORM_TABLE = new float[256];
+
+ static {
+ for (int i = 0; i < 256; i++) {
+ float f = SmallFloat.byte315ToFloat((byte)i);
+ NORM_TABLE[i] = 1.0f / (f*f);
+ }
+ }
+
+ @Override
+ public final byte computeNorm(FieldInvertState state) {
+ final int numTerms = discountOverlaps ? state.getLength() - state.getNumOverlap() : state.getLength();
+ return encodeNormValue(state.getBoost(), numTerms);
+ }
+
+ public Explanation idfExplain(TermContext stats, final IndexSearcher searcher) throws IOException {
+ final int df = stats.docFreq();
+ final int max = searcher.maxDoc();
+ final float idf = idf(df, max);
+ return new Explanation(idf, "idf(docFreq=" + df + ", maxDocs=" + max + ")");
+ }
+
+ public Explanation idfExplain(final TermContext stats[], IndexSearcher searcher) throws IOException {
+ final int max = searcher.maxDoc();
+ float idf = 0.0f;
+ final Explanation exp = new Explanation();
+ exp.setDescription("idf(), sum of:");
+ for (final TermContext stat : stats ) {
+ final int df = stat.docFreq();
+ final float termIdf = idf(df, max);
+ exp.addDetail(new Explanation(termIdf, "idf(docFreq=" + df + ", maxDocs=" + max + ")"));
+ idf += termIdf;
+ }
+ exp.setValue(idf);
+ return exp;
+ }
+
+ @Override
+ public final Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException {
+ Explanation idf = termStats.length == 1 ? idfExplain(termStats[0], searcher) : idfExplain(termStats, searcher);
+
+ float avgdl = avgFieldLength(searcher, fieldName);
+
+ // compute freq-independent part of bm25 equation across all norm values
+ float cache[] = new float[256];
+ for (int i = 0; i < cache.length; i++) {
+ cache[i] = k1 * ((1 - b) + b * decodeNormValue((byte)i) / avgdl);
+ }
+ return new BM25Stats(idf, queryBoost, avgdl, cache);
+ }
+
+ @Override
+ public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ final byte[] norms = context.reader.norms(fieldName);
+ return norms == null
+ ? new ExactBM25DocScorerNoNorms((BM25Stats)stats)
+ : new ExactBM25DocScorer((BM25Stats)stats, norms);
+ }
+
+ @Override
+ public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new SloppyBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName));
+ }
+
+ private class ExactBM25DocScorer extends ExactDocScorer {
+ private final BM25Stats stats;
+ private final float weightValue;
+ private final byte[] norms;
+ private final float[] cache;
+
+ ExactBM25DocScorer(BM25Stats stats, byte norms[]) {
+ assert norms != null;
+ this.stats = stats;
+ this.weightValue = stats.weight * (k1 + 1); // boost * idf * (k1 + 1)
+ this.cache = stats.cache;
+ this.norms = norms;
+ }
+
+ @Override
+ public float score(int doc, int freq) {
+ return weightValue * freq / (freq + cache[norms[doc] & 0xFF]);
+ }
+
+ @Override
+ public Explanation explain(int doc, Explanation freq) {
+ return explainScore(doc, freq, stats, norms);
+ }
+ }
+
+ /** there are no norms, we act as if b=0 */
+ private class ExactBM25DocScorerNoNorms extends ExactDocScorer {
+ private final BM25Stats stats;
+ private final float weightValue;
+ private static final int SCORE_CACHE_SIZE = 32;
+ private float[] scoreCache = new float[SCORE_CACHE_SIZE];
+
+ ExactBM25DocScorerNoNorms(BM25Stats stats) {
+ this.stats = stats;
+ this.weightValue = stats.weight * (k1 + 1); // boost * idf * (k1 + 1)
+ for (int i = 0; i < SCORE_CACHE_SIZE; i++)
+ scoreCache[i] = weightValue * i / (i + k1);
+ }
+
+ @Override
+ public float score(int doc, int freq) {
+ // TODO: maybe score cache is more trouble than its worth?
+ return freq < SCORE_CACHE_SIZE // check cache
+ ? scoreCache[freq] // cache hit
+ : weightValue * freq / (freq + k1); // cache miss
+ }
+
+ @Override
+ public Explanation explain(int doc, Explanation freq) {
+ return explainScore(doc, freq, stats, null);
+ }
+ }
+
+ private class SloppyBM25DocScorer extends SloppyDocScorer {
+ private final BM25Stats stats;
+ private final float weightValue; // boost * idf * (k1 + 1)
+ private final byte[] norms;
+ private final float[] cache;
+
+ SloppyBM25DocScorer(BM25Stats stats, byte norms[]) {
+ this.stats = stats;
+ this.weightValue = stats.weight * (k1 + 1);
+ this.cache = stats.cache;
+ this.norms = norms;
+ }
+
+ @Override
+ public float score(int doc, float freq) {
+ // if there are no norms, we act as if b=0
+ float norm = norms == null ? k1 : cache[norms[doc] & 0xFF];
+ return weightValue * freq / (freq + norm);
+ }
+
+ @Override
+ public Explanation explain(int doc, Explanation freq) {
+ return explainScore(doc, freq, stats, norms);
+ }
+
+ @Override
+ public float computeSlopFactor(int distance) {
+ return sloppyFreq(distance);
+ }
+
+ @Override
+ public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
+ return scorePayload(doc, start, end, payload);
+ }
+ }
+
+ /** Collection statistics for the BM25 model. */
+ private static class BM25Stats extends Stats {
+ /** BM25's idf */
+ private final Explanation idf;
+ /** The average document length. */
+ private final float avgdl;
+ /** query's inner boost */
+ private final float queryBoost;
+ /** weight (idf * boost) */
+ private float weight;
+ /** precomputed norm[256] with k1 * ((1 - b) + b * dl / avgdl) */
+ private final float cache[];
+
+ BM25Stats(Explanation idf, float queryBoost, float avgdl, float cache[]) {
+ this.idf = idf;
+ this.queryBoost = queryBoost;
+ this.avgdl = avgdl;
+ this.cache = cache;
+ }
+
+ @Override
+ public float getValueForNormalization() {
+ // we return a TF-IDF like normalization to be nice, but we don't actually normalize ourselves.
+ final float queryWeight = idf.getValue() * queryBoost;
+ return queryWeight * queryWeight;
+ }
+
+ @Override
+ public void normalize(float queryNorm, float topLevelBoost) {
+ // we don't normalize with queryNorm at all, we just capture the top-level boost
+ this.weight = idf.getValue() * queryBoost * topLevelBoost;
+ }
+ }
+
+ private Explanation explainScore(int doc, Explanation freq, BM25Stats stats, byte[] norms) {
+ Explanation result = new Explanation();
+ result.setDescription("score(doc="+doc+",freq="+freq+"), product of:");
+
+ Explanation boostExpl = new Explanation(stats.queryBoost, "boost");
+ if (stats.queryBoost != 1.0f)
+ result.addDetail(boostExpl);
+
+ result.addDetail(stats.idf);
+
+ Explanation tfNormExpl = new Explanation();
+ tfNormExpl.setDescription("tfNorm, computed from:");
+ tfNormExpl.addDetail(freq);
+ tfNormExpl.addDetail(new Explanation(k1, "parameter k1"));
+ if (norms == null) {
+ tfNormExpl.addDetail(new Explanation(0, "parameter b (norms omitted for field)"));
+ tfNormExpl.setValue((freq.getValue() * (k1 + 1)) / (freq.getValue() + k1));
+ } else {
+ float doclen = decodeNormValue(norms[doc]);
+ tfNormExpl.addDetail(new Explanation(b, "parameter b"));
+ tfNormExpl.addDetail(new Explanation(stats.avgdl, "avgFieldLength"));
+ tfNormExpl.addDetail(new Explanation(doclen, "fieldLength"));
+ tfNormExpl.setValue((freq.getValue() * (k1 + 1)) / (freq.getValue() + k1 * (1 - b + b * doclen/stats.avgdl)));
+ }
+ result.addDetail(tfNormExpl);
+ result.setValue(boostExpl.getValue() * stats.idf.getValue() * tfNormExpl.getValue());
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return "BM25(k1=" + k1 + ",b=" + b + ")";
+ }
+}
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java 1969-12-31 19:00:00.000000000 -0500
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarity.java 2011-08-30 01:01:01.231459487 -0400
@@ -0,0 +1,93 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.index.FieldInvertState;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** Expert: Default scoring implementation. */
+public class DefaultSimilarity extends TFIDFSimilarity {
+
+ /** Implemented as
+ * state.getBoost()*lengthNorm(numTerms), where
+ * numTerms is {@link FieldInvertState#getLength()} if {@link
+ * #setDiscountOverlaps} is false, else it's {@link
+ * FieldInvertState#getLength()} - {@link
+ * FieldInvertState#getNumOverlap()}.
+ *
+ * @lucene.experimental */
+ @Override
+ public byte computeNorm(FieldInvertState state) {
+ final int numTerms;
+ if (discountOverlaps)
+ numTerms = state.getLength() - state.getNumOverlap();
+ else
+ numTerms = state.getLength();
+ return encodeNormValue(state.getBoost() * ((float) (1.0 / Math.sqrt(numTerms))));
+ }
+
+ /** Implemented as sqrt(freq). */
+ @Override
+ public float tf(float freq) {
+ return (float)Math.sqrt(freq);
+ }
+
+ /** Implemented as 1 / (distance + 1). */
+ @Override
+ public float sloppyFreq(int distance) {
+ return 1.0f / (distance + 1);
+ }
+
+ /** The default implementation returns 1 */
+ @Override
+ public float scorePayload(int doc, int start, int end, BytesRef payload) {
+ return 1;
+ }
+
+ /** Implemented as log(numDocs/(docFreq+1)) + 1. */
+ @Override
+ public float idf(int docFreq, int numDocs) {
+ return (float)(Math.log(numDocs/(double)(docFreq+1)) + 1.0);
+ }
+
+ // Default true
+ protected boolean discountOverlaps = true;
+
+ /** Determines whether overlap tokens (Tokens with
+ * 0 position increment) are ignored when computing
+ * norm. By default this is true, meaning overlap
+ * tokens do not count when computing norms.
+ *
+ * @lucene.experimental
+ *
+ * @see #computeNorm
+ */
+ public void setDiscountOverlaps(boolean v) {
+ discountOverlaps = v;
+ }
+
+ /** @see #setDiscountOverlaps */
+ public boolean getDiscountOverlaps() {
+ return discountOverlaps;
+ }
+
+ @Override
+ public String toString() {
+ return "DefaultSimilarity";
+ }
+}
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarityProvider.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarityProvider.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarityProvider.java 1969-12-31 19:00:00.000000000 -0500
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/DefaultSimilarityProvider.java 2011-08-23 08:43:39.731460295 -0400
@@ -0,0 +1,42 @@
+package org.apache.lucene.search.similarities;
+
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Expert: Default scoring provider.
+ *
+ * Returns {@link DefaultSimilarity} for every field
+ */
+public class DefaultSimilarityProvider implements SimilarityProvider {
+ private static final Similarity impl = new DefaultSimilarity();
+
+ /** Implemented as overlap / maxOverlap. */
+ public float coord(int overlap, int maxOverlap) {
+ return overlap / (float)maxOverlap;
+ }
+
+ /** Implemented as 1/sqrt(sumOfSquaredWeights). */
+ public float queryNorm(float sumOfSquaredWeights) {
+ return (float)(1.0 / Math.sqrt(sumOfSquaredWeights));
+ }
+
+ public Similarity get(String field) {
+ return impl;
+ }
+}
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java 1969-12-31 19:00:00.000000000 -0500
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java 2011-09-01 14:52:10.165810203 -0400
@@ -0,0 +1,86 @@
+package org.apache.lucene.search.similarities;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.search.Explanation;
+
+/**
+ * Implements the divergence from randomness (DFR) framework
+ * introduced in Gianni Amati and Cornelis Joost Van Rijsbergen. 2002.
+ * Probabilistic models of information retrieval based on measuring the
+ * divergence from randomness. ACM Trans. Inf. Syst. 20, 4 (October 2002),
+ * 357-389.
+ *
The DFR scoring formula is composed of three separate components: the + * basic model, the aftereffect and an additional + * normalization component, represented by the classes + * {@code BasicModel}, {@code AfterEffect} and {@code Normalization}, + * respectively. The names of these classes were chosen to match the names of + * their counterparts in the Terrier IR engine.
+ *Note that qtf, the multiplicity of term-occurrence in the query, + * is not handled by this implementation.
+ * @see BasicModel + * @see AfterEffect + * @see Normalization + * @lucene.experimental + */ +public class DFRSimilarity extends SimilarityBase { + /** The basic model for information content. */ + protected final BasicModel basicModel; + /** The first normalization of the information content. */ + protected final AfterEffect afterEffect; + /** The term frequency normalization. */ + protected final Normalization normalization; + + public DFRSimilarity(BasicModel basicModel, + AfterEffect afterEffect, + Normalization normalization) { + if (basicModel == null || afterEffect == null || normalization == null) { + throw new NullPointerException("null parameters not allowed."); + } + this.basicModel = basicModel; + this.afterEffect = afterEffect; + this.normalization = normalization; + } + + @Override + protected float score(BasicStats stats, float freq, float docLen) { + float tfn = normalization.tfn(stats, freq, docLen); + return stats.getTotalBoost() * + basicModel.score(stats, tfn) * afterEffect.score(stats, tfn); + } + + @Override + protected void explain(Explanation expl, + BasicStats stats, int doc, float freq, float docLen) { + if (stats.getTotalBoost() != 1.0f) { + expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); + } + + Explanation normExpl = normalization.explain(stats, freq, docLen); + float tfn = normExpl.getValue(); + expl.addDetail(normExpl); + expl.addDetail(basicModel.explain(stats, tfn)); + expl.addDetail(afterEffect.explain(stats, tfn)); + } + + @Override + public String toString() { + return "DFR " + basicModel.toString() + afterEffect.toString() + + normalization.toString(); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/Distribution.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/Distribution.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/Distribution.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/Distribution.java 2011-08-23 09:16:32.711460292 -0400 @@ -0,0 +1,45 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * The probabilistic distribution used to model term occurrence + * in information-based models. + * @see IBSimilarity + * @lucene.experimental + */ +public abstract class Distribution { + /** Computes the score. */ + public abstract float score(BasicStats stats, float tfn, float lambda); + + /** Explains the score. Returns the name of the model only, since + * both {@code tfn} and {@code lambda} are explained elsewhere. */ + public Explanation explain(BasicStats stats, float tfn, float lambda) { + return new Explanation( + score(stats, tfn, lambda), getClass().getSimpleName()); + } + + /** + * Subclasses must override this method to return the name of the + * distribution. + */ + @Override + public abstract String toString(); +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/DistributionLL.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/DistributionLL.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/DistributionLL.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/DistributionLL.java 2011-08-23 09:16:32.711460292 -0400 @@ -0,0 +1,37 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Log-logistic distribution. + *Unlike for DFR, the natural logarithm is used, as + * it is faster to compute and the original paper does not express any + * preference to a specific base.
+ * @lucene.experimental + */ +public class DistributionLL extends Distribution { + @Override + public final float score(BasicStats stats, float tfn, float lambda) { + return (float)-Math.log(lambda / (tfn + lambda)); + } + + @Override + public String toString() { + return "LL"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/DistributionSPL.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/DistributionSPL.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/DistributionSPL.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/DistributionSPL.java 2011-08-23 09:16:32.711460292 -0400 @@ -0,0 +1,42 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The smoothed power-law (SPL) distribution for the information-based framework + * that is described in the original paper. + *Unlike for DFR, the natural logarithm is used, as + * it is faster to compute and the original paper does not express any + * preference to a specific base.
+ * @lucene.experimental + */ +public class DistributionSPL extends Distribution { + @Override + public final float score(BasicStats stats, float tfn, float lambda) { + if (lambda == 1f) { + lambda = 0.99f; + } + return (float)-Math.log( + (Math.pow(lambda, (tfn / (tfn + 1))) - lambda) / (1 - lambda)); + } + + @Override + public String toString() { + return "SPL"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/IBSimilarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/IBSimilarity.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/IBSimilarity.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/IBSimilarity.java 2011-08-30 10:20:35.671459440 -0400 @@ -0,0 +1,94 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * Provides a framework for the family of information-based models, as described + * in Stéphane Clinchant and Eric Gaussier. 2010. Information-based + * models for ad hoc IR. In Proceeding of the 33rd international ACM SIGIR + * conference on Research and development in information retrieval (SIGIR '10). + * ACM, New York, NY, USA, 234-241. + *The retrieval function is of the form RSV(q, d) = ∑ + * -xqw log Prob(Xw ≥ + * tdw | λw), where + *
The framework described in the paper has many similarities to the DFR + * framework (see {@link DFRSimilarity}). It is possible that the two + * Similarities will be merged at one point.
+ * @lucene.experimental + */ +public class IBSimilarity extends SimilarityBase { + /** The probabilistic distribution used to model term occurrence. */ + protected final Distribution distribution; + /** The lambda (λw) parameter. */ + protected final Lambda lambda; + /** The term frequency normalization. */ + protected final Normalization normalization; + + public IBSimilarity(Distribution distribution, + Lambda lambda, + Normalization normalization) { + this.distribution = distribution; + this.lambda = lambda; + this.normalization = normalization; + } + + @Override + protected float score(BasicStats stats, float freq, float docLen) { + return stats.getTotalBoost() * + distribution.score( + stats, + normalization.tfn(stats, freq, docLen), + lambda.lambda(stats)); + } + + @Override + protected void explain( + Explanation expl, BasicStats stats, int doc, float freq, float docLen) { + if (stats.getTotalBoost() != 1.0f) { + expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); + } + Explanation normExpl = normalization.explain(stats, freq, docLen); + Explanation lambdaExpl = lambda.explain(stats); + expl.addDetail(normExpl); + expl.addDetail(lambdaExpl); + expl.addDetail(distribution.explain( + stats, normExpl.getValue(), lambdaExpl.getValue())); + } + + /** + * The name of IB methods follow the pattern + * {@code IB+ * The formula as defined the paper assigns a negative score to documents that + * contain the term, but with fewer occurrences than predicted by the collection + * language model. The Lucene implementation returns {@code 0} for such + * documents. + *
+ * + * @lucene.experimental + */ +public class LMDirichletSimilarity extends LMSimilarity { + /** The μ parameter. */ + private final float mu; + + /** @param mu the μ parameter. */ + public LMDirichletSimilarity(CollectionModel collectionModel, float mu) { + super(collectionModel); + this.mu = mu; + } + + /** @param mu the μ parameter. */ + public LMDirichletSimilarity(float mu) { + this.mu = mu; + } + + /** Instantiates the similarity with the default μ value of 2000. */ + public LMDirichletSimilarity(CollectionModel collectionModel) { + this(collectionModel, 2000); + } + + /** Instantiates the similarity with the default μ value of 2000. */ + public LMDirichletSimilarity() { + this(2000); + } + + @Override + protected float score(BasicStats stats, float freq, float docLen) { + float score = stats.getTotalBoost() * (float)(Math.log(1 + freq / + (mu * ((LMStats)stats).getCollectionProbability())) + + Math.log(mu / (docLen + mu))); + return score > 0.0f ? score : 0.0f; + } + + @Override + protected void explain(Explanation expl, BasicStats stats, int doc, + float freq, float docLen) { + if (stats.getTotalBoost() != 1.0f) { + expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); + } + + expl.addDetail(new Explanation(mu, "mu")); + Explanation weightExpl = new Explanation(); + weightExpl.setValue((float)Math.log(1 + freq / + (mu * ((LMStats)stats).getCollectionProbability()))); + weightExpl.setDescription("term weight"); + expl.addDetail(weightExpl); + expl.addDetail(new Explanation( + (float)Math.log(mu / (docLen + mu)), "document norm")); + super.explain(expl, stats, doc, freq, docLen); + } + + /** Returns the μ parameter. */ + public float getMu() { + return mu; + } + + @Override + public String getName() { + return String.format("Dirichlet(%f)", getMu()); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java 2011-08-23 09:16:32.741460292 -0400 @@ -0,0 +1,77 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * Language model based on the Jelinek-Mercer smoothing method. From Chengxiang + * Zhai and John Lafferty. 2001. A study of smoothing methods for language + * models applied to Ad Hoc information retrieval. In Proceedings of the 24th + * annual international ACM SIGIR conference on Research and development in + * information retrieval (SIGIR '01). ACM, New York, NY, USA, 334-342. + *The model has a single parameter, λ. According to said paper, the + * optimal value depends on both the collection and the query. The optimal value + * is around {@code 0.1} for title queries and {@code 0.7} for long queries.
+ * + * @lucene.experimental + */ +public class LMJelinekMercerSimilarity extends LMSimilarity { + /** The λ parameter. */ + private final float lambda; + + /** @param lambda the λ parameter. */ + public LMJelinekMercerSimilarity( + CollectionModel collectionModel, float lambda) { + super(collectionModel); + this.lambda = lambda; + } + + /** @param lambda the λ parameter. */ + public LMJelinekMercerSimilarity(float lambda) { + this.lambda = lambda; + } + + @Override + protected float score(BasicStats stats, float freq, float docLen) { + return stats.getTotalBoost() * + (float)Math.log(1 + + ((1 - lambda) * freq / docLen) / + (lambda * ((LMStats)stats).getCollectionProbability())); + } + + @Override + protected void explain(Explanation expl, BasicStats stats, int doc, + float freq, float docLen) { + if (stats.getTotalBoost() != 1.0f) { + expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); + } + expl.addDetail(new Explanation(lambda, "lambda")); + super.explain(expl, stats, doc, freq, docLen); + } + + /** Returns the λ parameter. */ + public float getLambda() { + return lambda; + } + + @Override + public String getName() { + return String.format("Jelinek-Mercer(%f)", getLambda()); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java 2011-09-07 00:50:54.785982890 -0400 @@ -0,0 +1,155 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.TermContext; + +/** + * Abstract superclass for language modeling Similarities. The following inner + * types are introduced: + *Used in {@link #toString()}
. + */ + public abstract String getName(); + + /** + * Returns the name of the LM method. If a custom collection model strategy is + * used, its name is included as well. + * @see #getName() + * @see CollectionModel#getName() + * @see DefaultCollectionModel + */ + @Override + public String toString() { + String coll = collectionModel.getName(); + if (coll != null) { + return String.format("LM %s - %s", getName(), coll); + } else { + return String.format("LM %s", getName()); + } + } + + /** Stores the collection distribution of the current term. */ + public static class LMStats extends BasicStats { + /** The probability that the current term is generated by the collection. */ + private float collectionProbability; + + public LMStats(float queryBoost) { + super(queryBoost); + } + + /** + * Returns the probability that the current term is generated by the + * collection. + */ + public final float getCollectionProbability() { + return collectionProbability; + } + + /** + * Sets the probability that the current term is generated by the + * collection. + */ + public final void setCollectionProbability(float collectionProbability) { + this.collectionProbability = collectionProbability; + } + } + + /** A strategy for computing the collection language model. */ + public static interface CollectionModel { + /** + * Computes the probability {@code p(w|C)} according to the language model + * strategy for the current term. + */ + public float computeProbability(BasicStats stats); + + /** The name of the collection model strategy. */ + public String getName(); + } + + /** + * Models {@code p(w|C)} as the number of occurrences of the term in the + * collection, divided by the total number of tokens {@code + 1}. + */ + public static class DefaultCollectionModel implements CollectionModel { + @Override + public float computeProbability(BasicStats stats) { + return (float)stats.getTotalTermFreq() / (stats.getNumberOfFieldTokens() +1); + } + + @Override + public String getName() { + return null; + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java 2011-09-07 00:42:53.735982712 -0400 @@ -0,0 +1,159 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.TermContext; + +/** + * Implements the CombSUM method for combining evidence from multiple + * similarity values described in: Joseph A. Shaw, Edward A. Fox. + * In Text REtrieval Conference (1993), pp. 243-252 + * @lucene.experimental + */ +public class MultiSimilarity extends Similarity { + protected final Similarity sims[]; + + public MultiSimilarity(Similarity sims[]) { + this.sims = sims; + } + + @Override + public byte computeNorm(FieldInvertState state) { + return sims[0].computeNorm(state); + } + + @Override + public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException { + Stats subStats[] = new Stats[sims.length]; + for (int i = 0; i < subStats.length; i++) { + subStats[i] = sims[i].computeStats(searcher, fieldName, queryBoost, termContexts); + } + return new MultiStats(subStats); + } + + @Override + public ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { + ExactDocScorer subScorers[] = new ExactDocScorer[sims.length]; + for (int i = 0; i < subScorers.length; i++) { + subScorers[i] = sims[i].exactDocScorer(((MultiStats)stats).subStats[i], fieldName, context); + } + return new MultiExactDocScorer(subScorers); + } + + @Override + public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { + SloppyDocScorer subScorers[] = new SloppyDocScorer[sims.length]; + for (int i = 0; i < subScorers.length; i++) { + subScorers[i] = sims[i].sloppyDocScorer(((MultiStats)stats).subStats[i], fieldName, context); + } + return new MultiSloppyDocScorer(subScorers); + } + + public static class MultiExactDocScorer extends ExactDocScorer { + private final ExactDocScorer subScorers[]; + + MultiExactDocScorer(ExactDocScorer subScorers[]) { + this.subScorers = subScorers; + } + + @Override + public float score(int doc, int freq) { + float sum = 0.0f; + for (ExactDocScorer subScorer : subScorers) { + sum += subScorer.score(doc, freq); + } + return sum; + } + + @Override + public Explanation explain(int doc, Explanation freq) { + Explanation expl = new Explanation(score(doc, (int)freq.getValue()), "sum of:"); + for (ExactDocScorer subScorer : subScorers) { + expl.addDetail(subScorer.explain(doc, freq)); + } + return expl; + } + } + + public static class MultiSloppyDocScorer extends SloppyDocScorer { + private final SloppyDocScorer subScorers[]; + + MultiSloppyDocScorer(SloppyDocScorer subScorers[]) { + this.subScorers = subScorers; + } + + @Override + public float score(int doc, float freq) { + float sum = 0.0f; + for (SloppyDocScorer subScorer : subScorers) { + sum += subScorer.score(doc, freq); + } + return sum; + } + + @Override + public Explanation explain(int doc, Explanation freq) { + Explanation expl = new Explanation(score(doc, freq.getValue()), "sum of:"); + for (SloppyDocScorer subScorer : subScorers) { + expl.addDetail(subScorer.explain(doc, freq)); + } + return expl; + } + + @Override + public float computeSlopFactor(int distance) { + return subScorers[0].computeSlopFactor(distance); + } + + @Override + public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { + return subScorers[0].computePayloadFactor(doc, start, end, payload); + } + } + + public static class MultiStats extends Stats { + final Stats subStats[]; + + MultiStats(Stats subStats[]) { + this.subStats = subStats; + } + + @Override + public float getValueForNormalization() { + float sum = 0.0f; + for (Stats stat : subStats) { + sum += stat.getValueForNormalization(); + } + return sum / subStats.length; + } + + @Override + public void normalize(float queryNorm, float topLevelBoost) { + for (Stats stat : subStats) { + stat.normalize(queryNorm, topLevelBoost); + } + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java 2011-08-30 11:57:25.601459432 -0400 @@ -0,0 +1,33 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Normalization model that assumes a uniform distribution of the term frequency. + */ +public class NormalizationH1 extends Normalization { + @Override + public final float tfn(BasicStats stats, float tf, float len) { + return tf * stats.getAvgFieldLength() / len; + } + + @Override + public String toString() { + return "1"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java 2011-08-23 09:16:32.741460292 -0400 @@ -0,0 +1,36 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.SimilarityBase.log2; + +/** + * Normalization model in which the term frequency is inversely related to the + * length. + */ +public class NormalizationH2 extends Normalization { + @Override + public final float tfn(BasicStats stats, float tf, float len) { + return (float)(tf * log2(1 + stats.getAvgFieldLength() / len)); + } + + @Override + public String toString() { + return "2"; + } +} \ No newline at end of file diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH3.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH3.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH3.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/NormalizationH3.java 2011-09-07 01:06:06.755983226 -0400 @@ -0,0 +1,43 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Dirichlet Priors normalization + */ +public class NormalizationH3 extends Normalization { + private final float mu; + + public NormalizationH3() { + this(800F); + } + + public NormalizationH3(float mu) { + this.mu = mu; + } + + @Override + public float tfn(BasicStats stats, float tf, float len) { + return (tf + mu * (stats.getTotalTermFreq() / (float)stats.getNumberOfFieldTokens())) / (len + mu) * mu; + } + + @Override + public String toString() { + return "3(" + mu + ")"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/Normalization.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/Normalization.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/Normalization.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/Normalization.java 2011-08-23 09:16:32.741460292 -0400 @@ -0,0 +1,75 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * This class acts as the base class for the implementations of the term + * frequency normalization methods in the DFR framework. + * + * @see DFRSimilarity + * @lucene.experimental + */ +public abstract class Normalization { + /** Returns the normalized term frequency. + * @param len the field length. */ + public abstract float tfn(BasicStats stats, float tf, float len); + + /** Returns an explanation for the normalized term frequency. + *The default normalization methods use the field length of the document + * and the average field length to compute the normalized term frequency. + * This method provides a generic explanation for such methods. + * Subclasses that use other statistics must override this method.
+ */ + public Explanation explain(BasicStats stats, float tf, float len) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(tfn(stats, tf, len)); + result.addDetail(new Explanation(tf, "tf")); + result.addDetail( + new Explanation(stats.getAvgFieldLength(), "avgFieldLength")); + result.addDetail(new Explanation(len, "len")); + return result; + } + + /** Implementation used when there is no normalization. */ + public static final class NoNormalization extends Normalization { + @Override + public final float tfn(BasicStats stats, float tf, float len) { + return tf; + } + + @Override + public final Explanation explain(BasicStats stats, float tf, float len) { + return new Explanation(1, "no normalization"); + } + + @Override + public String toString() { + return ""; + } + } + + /** + * Subclasses must override this method to return the code of the + * normalization formula. Refer to the original paper for the list. + */ + @Override + public abstract String toString(); +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/NormalizationZ.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/NormalizationZ.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/NormalizationZ.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/NormalizationZ.java 2011-09-06 15:40:57.955970712 -0400 @@ -0,0 +1,43 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Pareto-Zipf Normalization + */ +public class NormalizationZ extends Normalization { + final float z; + + public NormalizationZ() { + this(0.30F); + } + + public NormalizationZ(float z) { + this.z = z; + } + + @Override + public float tfn(BasicStats stats, float tf, float len) { + return (float)(tf * Math.pow(stats.avgFieldLength / len, z)); + } + + @Override + public String toString() { + return "Z(" + z + ")"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java 2011-09-09 12:08:45.116061670 -0400 @@ -0,0 +1,345 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SmallFloat; +import org.apache.lucene.util.TermContext; + +/** + * A subclass of {@code Similarity} that provides a simplified API for its + * descendants. Subclasses are only required to implement the {@link #score} + * and {@link #toString()} methods. Implementing + * {@link #explain(Explanation, BasicStats, int, float, float)} is optional, + * inasmuch as SimilarityBase already provides a basic explanation of the score + * and the term frequency. However, implementers of a subclass are encouraged to + * include as much detail about the scoring method as possible. + *+ * Note: multi-word queries such as phrase queries are scored in a different way + * than Lucene's default ranking algorithm: whereas it "fakes" an IDF value for + * the phrase as a whole (since it does not know it), this class instead scores + * phrases as a summation of the individual term scores. + * @lucene.experimental + */ +public abstract class SimilarityBase extends Similarity { + /** For {@link #log2(double)}. Precomputed for efficiency reasons. */ + private static final double LOG_2 = Math.log(2); + + /** @see #setDiscountOverlaps */ + protected boolean discountOverlaps = true; + + /** Determines whether overlap tokens (Tokens with + * 0 position increment) are ignored when computing + * norm. By default this is true, meaning overlap + * tokens do not count when computing norms. + * + * @lucene.experimental + * + * @see #computeNorm + */ + public void setDiscountOverlaps(boolean v) { + discountOverlaps = v; + } + + /** @see #setDiscountOverlaps */ + public boolean getDiscountOverlaps() { + return discountOverlaps; + } + + @Override + public final Stats computeStats(IndexSearcher searcher, String fieldName, + float queryBoost, TermContext... termContexts) throws IOException { + BasicStats stats[] = new BasicStats[termContexts.length]; + for (int i = 0; i < termContexts.length; i++) { + stats[i] = newStats(queryBoost); + fillBasicStats(stats[i], searcher, fieldName, termContexts[i]); + } + return stats.length == 1 ? stats[0] : new MultiSimilarity.MultiStats(stats); + } + + /** Factory method to return a custom stats object */ + protected BasicStats newStats(float queryBoost) { + return new BasicStats(queryBoost); + } + + /** Fills all member fields defined in {@code BasicStats} in {@code stats}. + * Subclasses can override this method to fill additional stats. */ + protected void fillBasicStats(BasicStats stats, IndexSearcher searcher, + String fieldName, TermContext termContext) throws IOException { + IndexReader reader = searcher.getIndexReader(); + int numberOfDocuments = reader.maxDoc(); + + int docFreq = termContext.docFreq(); + long totalTermFreq = termContext.totalTermFreq(); + + // codec does not supply totalTermFreq: substitute docFreq + if (totalTermFreq == -1) { + totalTermFreq = docFreq; + } + + final long numberOfFieldTokens; + final float avgFieldLength; + + Terms terms = MultiFields.getTerms(searcher.getIndexReader(), fieldName); + if (terms == null) { + // field does not exist; + numberOfFieldTokens = 0; + avgFieldLength = 1; + } else { + long sumTotalTermFreq = terms.getSumTotalTermFreq(); + + // We have to provide something if codec doesnt supply these measures, + // or if someone omitted frequencies for the field... negative values cause + // NaN/Inf for some scorers. + if (sumTotalTermFreq == -1) { + numberOfFieldTokens = docFreq; + avgFieldLength = 1; + } else { + numberOfFieldTokens = sumTotalTermFreq; + avgFieldLength = (float)numberOfFieldTokens / numberOfDocuments; + } + } + + // TODO: add sumDocFreq for field (numberOfFieldPostings) + stats.setNumberOfDocuments(numberOfDocuments); + stats.setNumberOfFieldTokens(numberOfFieldTokens); + stats.setAvgFieldLength(avgFieldLength); + stats.setDocFreq(docFreq); + stats.setTotalTermFreq(totalTermFreq); + } + + /** + * Scores the document {@code doc}. + *
Subclasses must apply their scoring formula in this class.
+ * @param stats the corpus level statistics. + * @param freq the term frequency. + * @param docLen the document length. + * @return the score. + */ + protected abstract float score(BasicStats stats, float freq, float docLen); + + /** + * Subclasses should implement this method to explain the score. {@code expl} + * already contains the score, the name of the class and the doc id, as well + * as the term frequency and its explanation; subclasses can add additional + * clauses to explain details of their scoring formulae. + *The default implementation does nothing.
+ * + * @param expl the explanation to extend with details. + * @param stats the corpus level statistics. + * @param doc the document id. + * @param freq the term frequency. + * @param docLen the document length. + */ + protected void explain( + Explanation expl, BasicStats stats, int doc, float freq, float docLen) {} + + /** + * Explains the score. The implementation here provides a basic explanation + * in the format score(name-of-similarity, doc=doc-id, + * freq=term-frequency), computed from:, and + * attaches the score (computed via the {@link #score(BasicStats, float, float)} + * method) and the explanation for the term frequency. Subclasses content with + * this format may add additional details in + * {@link #explain(Explanation, BasicStats, int, float, float)}. + * + * @param stats the corpus level statistics. + * @param doc the document id. + * @param freq the term frequency and its explanation. + * @param docLen the document length. + * @return the explanation. + */ + protected Explanation explain( + BasicStats stats, int doc, Explanation freq, float docLen) { + Explanation result = new Explanation(); + result.setValue(score(stats, freq.getValue(), docLen)); + result.setDescription("score(" + getClass().getSimpleName() + + ", doc=" + doc + ", freq=" + freq.getValue() +"), computed from:"); + result.addDetail(freq); + + explain(result, stats, doc, freq.getValue(), docLen); + + return result; + } + + @Override + public ExactDocScorer exactDocScorer(Stats stats, String fieldName, + AtomicReaderContext context) throws IOException { + byte norms[] = context.reader.norms(fieldName); + + if (stats instanceof MultiSimilarity.MultiStats) { + // a multi term query (e.g. phrase). return the summation, + // scoring almost as if it were boolean query + Stats subStats[] = ((MultiSimilarity.MultiStats) stats).subStats; + ExactDocScorer subScorers[] = new ExactDocScorer[subStats.length]; + for (int i = 0; i < subScorers.length; i++) { + subScorers[i] = new BasicExactDocScorer((BasicStats)subStats[i], norms); + } + return new MultiSimilarity.MultiExactDocScorer(subScorers); + } else { + return new BasicExactDocScorer((BasicStats) stats, norms); + } + } + + @Override + public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, + AtomicReaderContext context) throws IOException { + byte norms[] = context.reader.norms(fieldName); + + if (stats instanceof MultiSimilarity.MultiStats) { + // a multi term query (e.g. phrase). return the summation, + // scoring almost as if it were boolean query + Stats subStats[] = ((MultiSimilarity.MultiStats) stats).subStats; + SloppyDocScorer subScorers[] = new SloppyDocScorer[subStats.length]; + for (int i = 0; i < subScorers.length; i++) { + subScorers[i] = new BasicSloppyDocScorer((BasicStats)subStats[i], norms); + } + return new MultiSimilarity.MultiSloppyDocScorer(subScorers); + } else { + return new BasicSloppyDocScorer((BasicStats) stats, norms); + } + } + + /** + * Subclasses must override this method to return the name of the Similarity + * and preferably the values of parameters (if any) as well. + */ + @Override + public abstract String toString(); + + // ------------------------------ Norm handling ------------------------------ + + /** Norm -> document length map. */ + private static final float[] NORM_TABLE = new float[256]; + + static { + for (int i = 0; i < 256; i++) { + float floatNorm = SmallFloat.byte315ToFloat((byte)i); + NORM_TABLE[i] = 1.0f / (floatNorm * floatNorm); + } + } + + /** Encodes the document length in the same way as {@link TFIDFSimilarity}. */ + @Override + public byte computeNorm(FieldInvertState state) { + final float numTerms; + if (discountOverlaps) + numTerms = state.getLength() - state.getNumOverlap(); + else + numTerms = state.getLength() / state.getBoost(); + return encodeNormValue(state.getBoost(), numTerms); + } + + /** Decodes a normalization factor (document length) stored in an index. + * @see #encodeNormValue(float,float) + */ + protected float decodeNormValue(byte norm) { + return NORM_TABLE[norm & 0xFF]; // & 0xFF maps negative bytes to positive above 127 + } + + /** Encodes the length to a byte via SmallFloat. */ + protected byte encodeNormValue(float boost, float length) { + return SmallFloat.floatToByte315((boost / (float) Math.sqrt(length))); + } + + // ----------------------------- Static methods ------------------------------ + + /** Returns the base two logarithm of {@code x}. */ + public static double log2(double x) { + // Put this to a 'util' class if we need more of these. + return Math.log(x) / LOG_2; + } + + // --------------------------------- Classes --------------------------------- + + /** Delegates the {@link #score(int, int)} and + * {@link #explain(int, Explanation)} methods to + * {@link SimilarityBase#score(BasicStats, float, int)} and + * {@link SimilarityBase#explain(BasicStats, int, Explanation, int)}, + * respectively. + */ + private class BasicExactDocScorer extends ExactDocScorer { + private final BasicStats stats; + private final byte[] norms; + + BasicExactDocScorer(BasicStats stats, byte norms[]) { + this.stats = stats; + this.norms = norms; + } + + @Override + public float score(int doc, int freq) { + // We have to supply something in case norms are omitted + return SimilarityBase.this.score(stats, freq, + norms == null ? 1F : decodeNormValue(norms[doc])); + } + + @Override + public Explanation explain(int doc, Explanation freq) { + return SimilarityBase.this.explain(stats, doc, freq, + norms == null ? 1F : decodeNormValue(norms[doc])); + } + } + + /** Delegates the {@link #score(int, int)} and + * {@link #explain(int, Explanation)} methods to + * {@link SimilarityBase#score(BasicStats, float, int)} and + * {@link SimilarityBase#explain(BasicStats, int, Explanation, int)}, + * respectively. + */ + private class BasicSloppyDocScorer extends SloppyDocScorer { + private final BasicStats stats; + private final byte[] norms; + + BasicSloppyDocScorer(BasicStats stats, byte norms[]) { + this.stats = stats; + this.norms = norms; + } + + @Override + public float score(int doc, float freq) { + // We have to supply something in case norms are omitted + return SimilarityBase.this.score(stats, freq, + norms == null ? 1F : decodeNormValue(norms[doc])); + } + @Override + public Explanation explain(int doc, Explanation freq) { + return SimilarityBase.this.explain(stats, doc, freq, + norms == null ? 1F : decodeNormValue(norms[doc])); + } + + @Override + public float computeSlopFactor(int distance) { + return 1.0f / (distance + 1); + } + + @Override + public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { + return 1f; + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java 2011-09-07 00:16:40.515982131 -0400 @@ -0,0 +1,230 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.io.IOException; + +import org.apache.lucene.document.IndexDocValuesField; // javadoc +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.IndexReader; // javadoc +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.Terms; // javadoc +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.spans.SpanQuery; // javadoc +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SmallFloat; // javadoc +import org.apache.lucene.util.TermContext; + + +/** + * Similarity defines the components of Lucene scoring. + *+ * Expert: Scoring API. + *
+ * This is a low-level API, you should only extend this API if you want to implement + * an information retrieval model. If you are instead looking for a convenient way + * to alter Lucene's scoring, consider extending a higher-level implementation + * such as {@link TFIDFSimilarity}, which implements the vector space model with this API, or + * just tweaking the default implementation: {@link DefaultSimilarity}. + *
+ * Similarity determines how Lucene weights terms, and Lucene interacts with + * this class at both index-time and + * query-time. + *
+ * + * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState)}, allowing + * the Similarity implementation to return a per-document byte for the field that will + * be later accessible via {@link IndexReader#norms(String)}. Lucene makes no assumption + * about what is in this byte, but it is most useful for encoding length normalization + * information. + *
+ * Implementations should carefully consider how the normalization byte is encoded: while + * Lucene's classical {@link TFIDFSimilarity} encodes a combination of index-time boost + * and length normalization information with {@link SmallFloat}, this might not be suitable + * for all purposes. + *
+ * Many formulas require the use of average document length, which can be computed via a + * combination of {@link Terms#getSumTotalTermFreq()} and {@link IndexReader#maxDoc()}, + *
+ * Because index-time boost is handled entirely at the application level anyway, + * an application can alternatively store the index-time boost separately using an + * {@link IndexDocValuesField}, and access this at query-time with + * {@link IndexReader#docValues(String)}. + *
+ * Finally, using index-time boosts (either via folding into the normalization byte or + * via IndexDocValues), is an inefficient way to boost the scores of different fields if the + * boost will be the same for every document, instead the Similarity can simply take a constant + * boost parameter C, and the SimilarityProvider can return different instances with + * different boosts depending upon field name. + *
+ * + * At query-time, Queries interact with the Similarity via these steps: + *
+ * + * When {@link IndexSearcher#explain(Query, int)} is called, queries consult the Similarity's DocScorer for an + * explanation of how it computed its score. The query passes in a the document id and an explanation of how the frequency + * was computed. + * + * @see org.apache.lucene.index.IndexWriterConfig#setSimilarityProvider(SimilarityProvider) + * @see IndexSearcher#setSimilarityProvider(SimilarityProvider) + * @lucene.experimental + */ +public abstract class Similarity { + /** + * Computes the normalization value for a field, given the accumulated + * state of term processing for this field (see {@link FieldInvertState}). + * + *
Implementations should calculate a byte value based on the field + * state and then return that value. + * + *
Matches in longer fields are less precise, so implementations of this
+ * method usually return smaller values when state.getLength() is large,
+ * and larger values when state.getLength() is small.
+ *
+ * @lucene.experimental
+ *
+ * @param state current processing state for this field
+ * @return the calculated byte norm
+ */
+ public abstract byte computeNorm(FieldInvertState state);
+
+ /**
+ * Compute any collection-level stats (e.g. IDF, average document length, etc) needed for scoring a query.
+ */
+ public abstract Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException;
+
+ /**
+ * returns a new {@link Similarity.ExactDocScorer}.
+ */
+ public abstract ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException;
+
+ /**
+ * returns a new {@link Similarity.SloppyDocScorer}.
+ */
+ public abstract SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException;
+
+ /**
+ * API for scoring exact queries such as {@link TermQuery} and
+ * exact {@link PhraseQuery}.
+ *
+ * Term frequencies are integers (the term or phrase's tf) + */ + public static abstract class ExactDocScorer { + /** + * Score a single document + * @param doc document id + * @param freq term frequency + * @return document's score + */ + public abstract float score(int doc, int freq); + + /** + * Explain the score for a single document + * @param doc document id + * @param freq Explanation of how the term frequency was computed + * @return document's score + */ + public Explanation explain(int doc, Explanation freq) { + Explanation result = new Explanation(score(doc, (int)freq.getValue()), + "score(doc=" + doc + ",freq=" + freq.getValue() +"), with freq of:"); + result.addDetail(freq); + return result; + } + } + + /** + * API for scoring "sloppy" queries such as {@link SpanQuery} and + * sloppy {@link PhraseQuery}. + *
+ * Term frequencies are floating point values. + */ + public static abstract class SloppyDocScorer { + /** + * Score a single document + * @param doc document id + * @param freq sloppy term frequency + * @return document's score + */ + public abstract float score(int doc, float freq); + + /** Computes the amount of a sloppy phrase match, based on an edit distance. */ + public abstract float computeSlopFactor(int distance); + + /** Calculate a scoring factor based on the data in the payload. */ + public abstract float computePayloadFactor(int doc, int start, int end, BytesRef payload); + + /** + * Explain the score for a single document + * @param doc document id + * @param freq Explanation of how the sloppy term frequency was computed + * @return document's score + */ + public Explanation explain(int doc, Explanation freq) { + Explanation result = new Explanation(score(doc, freq.getValue()), + "score(doc=" + doc + ",freq=" + freq.getValue() +"), with freq of:"); + result.addDetail(freq); + return result; + } + } + + /** Stores the statistics for the indexed collection. This abstract + * implementation is empty; descendants of {@code Similarity} should + * subclass {@code Stats} and define the statistics they require in the + * subclass. Examples include idf, average field length, etc. + */ + public static abstract class Stats { + + /** The value for normalization of contained query clauses (e.g. sum of squared weights). + *
+ * NOTE: a Similarity implementation might not use any query normalization at all, + * its not required. However, if it wants to participate in query normalization, + * it can return a value here. + */ + public abstract float getValueForNormalization(); + + /** Assigns the query normalization factor and boost from parent queries to this. + *
+ * NOTE: a Similarity implementation might not use this normalized value at all, + * its not required. However, its usually a good idea to at least incorporate + * the topLevelBoost (e.g. from an outer BooleanQuery) into its score. + */ + public abstract void normalize(float queryNorm, float topLevelBoost); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityProvider.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/SimilarityProvider.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/SimilarityProvider.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/SimilarityProvider.java 2011-08-23 08:43:39.731460295 -0400 @@ -0,0 +1,68 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.BooleanQuery; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Expert: Scoring API. + * + * Provides top-level scoring functions that aren't specific to a field, + * and work across multi-field queries (such as {@link BooleanQuery}). + * + * Field-specific scoring is accomplished through {@link Similarity}. + * + * @lucene.experimental + */ +public interface SimilarityProvider { + + /** Computes a score factor based on the fraction of all query terms that a + * document contains. This value is multiplied into scores. + * + *
The presence of a large portion of the query terms indicates a better + * match with the query, so implementations of this method usually return + * larger values when the ratio between these parameters is large and smaller + * values when the ratio between them is small. + * + * @param overlap the number of query terms matched in the document + * @param maxOverlap the total number of terms in the query + * @return a score factor based on term overlap with the query + */ + public abstract float coord(int overlap, int maxOverlap); + + /** Computes the normalization value for a query given the sum of the squared + * weights of each of the query terms. This value is multiplied into the + * weight of each query term. While the classic query normalization factor is + * computed as 1/sqrt(sumOfSquaredWeights), other implementations might + * completely ignore sumOfSquaredWeights (ie return 1). + * + *
This does not affect ranking, but the default implementation does make scores + * from different queries more comparable than they would be by eliminating the + * magnitude of the Query vector as a factor in the score. + * + * @param sumOfSquaredWeights the sum of the squares of query term weights + * @return a normalization factor for query weights + */ + public abstract float queryNorm(float sumOfSquaredWeights); + + /** Returns a {@link Similarity} for scoring a field + * @param field field name. + * @return a field-specific Similarity. + */ + public abstract Similarity get(String field); +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java 2011-08-29 18:59:31.331459518 -0400 @@ -0,0 +1,867 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.io.IOException; + +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.TermContext; +import org.apache.lucene.util.SmallFloat; + + +/** + * Implementation of {@link Similarity} with the Vector Space Model. + *
+ * Expert: Scoring API. + *
TFIDFSimilarity defines the components of Lucene scoring. + * Overriding computation of these components is a convenient + * way to alter Lucene scoring. + * + *
Suggested reading: + * + * Introduction To Information Retrieval, Chapter 6. + * + *
The following describes how Lucene scoring evolves from + * underlying information retrieval models to (efficient) implementation. + * We first brief on VSM Score, + * then derive from it Lucene's Conceptual Scoring Formula, + * from which, finally, evolves Lucene's Practical Scoring Function + * (the latter is connected directly with Lucene classes and methods). + * + *
Lucene combines + * + * Boolean model (BM) of Information Retrieval + * with + * + * Vector Space Model (VSM) of Information Retrieval - + * documents "approved" by BM are scored by VSM. + * + *
In VSM, documents and queries are represented as + * weighted vectors in a multi-dimensional space, + * where each distinct index term is a dimension, + * and weights are + * Tf-idf values. + * + *
VSM does not require weights to be Tf-idf values, + * but Tf-idf values are believed to produce search results of high quality, + * and so Lucene is using Tf-idf. + * Tf and Idf are described in more detail below, + * but for now, for completion, let's just say that + * for given term t and document (or query) x, + * Tf(t,x) varies with the number of occurrences of term t in x + * (when one increases so does the other) and + * idf(t) similarly varies with the inverse of the + * number of index documents containing term t. + * + *
VSM score of document d for query q is the
+ *
+ * Cosine Similarity
+ * of the weighted query vectors V(q) and V(d):
+ *
+ *
+ *
+ *
| ||||||
|
+ * |
Note: the above equation can be viewed as the dot product of + * the normalized weighted vectors, in the sense that dividing + * V(q) by its euclidean norm is normalizing it to a unit vector. + * + *
Lucene refines VSM score for both search quality and usability: + *
Under the simplifying assumption of a single field in the index,
+ * we get Lucene's Conceptual scoring formula:
+ *
+ *
+ *
+ *
| |||||||
|
+ * |
The conceptual formula is a simplification in the sense that (1) terms and documents + * are fielded and (2) boosts are usually per query term rather than per query. + * + *
We now describe how Lucene implements this conceptual scoring formula, and + * derive from it Lucene's Practical Scoring Function. + * + *
For efficient score computation some scoring components + * are computed and aggregated in advance: + * + *
Lucene's Practical Scoring Function is derived from the above. + * The color codes demonstrate how it relates + * to those of the conceptual formula: + * + *
+ *
+ *
| |||||||
|
+ * |
where + *
| + * {@link org.apache.lucene.search.similarities.DefaultSimilarity#tf(float) tf(t in d)} = + * | + *+ * frequency½ + * | + *
| + * {@link org.apache.lucene.search.similarities.DefaultSimilarity#idf(int, int) idf(t)} = + * | + *+ * 1 + log ( + * | + *
+ *
|
+ * + * ) + * | + *
| + * queryNorm(q) = + * {@link org.apache.lucene.search.similarities.DefaultSimilarityProvider#queryNorm(float) queryNorm(sumOfSquaredWeights)} + * = + * | + *
+ *
|
+ *
| + * {@link org.apache.lucene.search.Weight#getValueForNormalization() sumOfSquaredWeights} = + * {@link org.apache.lucene.search.Query#getBoost() q.getBoost()} 2 + * · + * | + *+ * ∑ + * | + *+ * ( + * idf(t) · + * t.getBoost() + * ) 2 + * | + *
| + * | t in q | + *+ * |
+ * When a document is added to the index, all the above factors are multiplied.
+ * If the document has multiple fields with the same name, all their boosts are multiplied together:
+ *
+ *
+ *
| + * norm(t,d) = + * lengthNorm + * · + * | + *+ * ∏ + * | + *+ * {@link org.apache.lucene.index.IndexableField#boost() f.boost}() + * | + *
| + * | field f in d named as t | + *+ * |
Terms and phrases repeated in a document indicate the topic of the
+ * document, so implementations of this method usually return larger values
+ * when freq is large, and smaller values when freq
+ * is small.
+ *
+ *
The default implementation calls {@link #tf(float)}. + * + * @param freq the frequency of a term within a document + * @return a score factor based on a term's within-document frequency + */ + public float tf(int freq) { + return tf((float)freq); + } + + /** Computes a score factor based on a term or phrase's frequency in a + * document. This value is multiplied by the {@link #idf(int, int)} + * factor for each term in the query and these products are then summed to + * form the initial score for a document. + * + *
Terms and phrases repeated in a document indicate the topic of the
+ * document, so implementations of this method usually return larger values
+ * when freq is large, and smaller values when freq
+ * is small.
+ *
+ * @param freq the frequency of a term within a document
+ * @return a score factor based on a term's within-document frequency
+ */
+ public abstract float tf(float freq);
+
+ /**
+ * Computes a score factor for a simple term and returns an explanation
+ * for that score factor.
+ *
+ *
+ * The default implementation uses: + * + *
+ * idf(docFreq, searcher.maxDoc()); + *+ * + * Note that {@link IndexSearcher#maxDoc()} is used instead of + * {@link org.apache.lucene.index.IndexReader#numDocs() IndexReader#numDocs()} because also + * {@link IndexSearcher#docFreq(Term)} is used, and when the latter + * is inaccurate, so is {@link IndexSearcher#maxDoc()}, and in the same direction. + * In addition, {@link IndexSearcher#maxDoc()} is more efficient to compute + * + * @param stats statistics of the term in question + * @param searcher the document collection being searched + * @return an Explain object that includes both an idf score factor + and an explanation for the term. + * @throws IOException + */ + public Explanation idfExplain(TermContext stats, final IndexSearcher searcher) throws IOException { + final int df = stats.docFreq(); + final int max = searcher.maxDoc(); + final float idf = idf(df, max); + return new Explanation(idf, "idf(docFreq=" + df + ", maxDocs=" + max + ")"); + } + + /** + * Computes a score factor for a phrase. + * + *
+ * The default implementation sums the idf factor for + * each term in the phrase. + * + * @param stats statistics of the terms in the phrase + * @param searcher the document collection being searched + * @return an Explain object that includes both an idf + * score factor for the phrase and an explanation + * for each term. + * @throws IOException + */ + public Explanation idfExplain(final TermContext stats[], IndexSearcher searcher) throws IOException { + final int max = searcher.maxDoc(); + float idf = 0.0f; + final Explanation exp = new Explanation(); + exp.setDescription("idf(), sum of:"); + for (final TermContext stat : stats ) { + final int df = stat.docFreq(); + final float termIdf = idf(df, max); + exp.addDetail(new Explanation(termIdf, "idf(docFreq=" + df + ", maxDocs=" + max + ")")); + idf += termIdf; + } + exp.setValue(idf); + return exp; + } + + /** Computes a score factor based on a term's document frequency (the number + * of documents which contain the term). This value is multiplied by the + * {@link #tf(int)} factor for each term in the query and these products are + * then summed to form the initial score for a document. + * + *
Terms that occur in fewer documents are better indicators of topic, so + * implementations of this method usually return larger values for rare terms, + * and smaller values for common terms. + * + * @param docFreq the number of documents which contain the term + * @param numDocs the total number of documents in the collection + * @return a score factor based on the term's document frequency + */ + public abstract float idf(int docFreq, int numDocs); + + /** Cache of decoded bytes. */ + private static final float[] NORM_TABLE = new float[256]; + + static { + for (int i = 0; i < 256; i++) + NORM_TABLE[i] = SmallFloat.byte315ToFloat((byte)i); + } + + /** Decodes a normalization factor stored in an index. + * @see #encodeNormValue(float) + */ + public float decodeNormValue(byte b) { + return NORM_TABLE[b & 0xFF]; // & 0xFF maps negative bytes to positive above 127 + } + + /** Encodes a normalization factor for storage in an index. + * + *
The encoding uses a three-bit mantissa, a five-bit exponent, and + * the zero-exponent point at 15, thus + * representing values from around 7x10^9 to 2x10^-9 with about one + * significant decimal digit of accuracy. Zero is also represented. + * Negative numbers are rounded up to zero. Values too large to represent + * are rounded down to the largest representable value. Positive values too + * small to represent are rounded up to the smallest positive representable + * value. + * @see org.apache.lucene.document.Field#setBoost(float) + * @see org.apache.lucene.util.SmallFloat + */ + public byte encodeNormValue(float f) { + return SmallFloat.floatToByte315(f); + } + + /** Computes the amount of a sloppy phrase match, based on an edit distance. + * This value is summed for each sloppy phrase match in a document to form + * the frequency to be used in scoring instead of the exact term count. + * + *
A phrase match with a small edit distance to a document passage more + * closely matches the document, so implementations of this method usually + * return larger values when the edit distance is small and smaller values + * when it is large. + * + * @see PhraseQuery#setSlop(int) + * @param distance the edit distance of this sloppy phrase match + * @return the frequency increment for this match + */ + public abstract float sloppyFreq(int distance); + + /** + * Calculate a scoring factor based on the data in the payload. Implementations + * are responsible for interpreting what is in the payload. Lucene makes no assumptions about + * what is in the byte array. + * + * @param doc The docId currently being scored. + * @param start The start position of the payload + * @param end The end position of the payload + * @param payload The payload byte array to be scored + * @return An implementation dependent float to be used as a scoring factor + */ + public abstract float scorePayload(int doc, int start, int end, BytesRef payload); + + @Override + public final Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, + TermContext... termContexts) throws IOException { + final Explanation idf = termContexts.length == 1 + ? idfExplain(termContexts[0], searcher) + : idfExplain(termContexts, searcher); + return new IDFStats(idf, queryBoost); + } + + @Override + public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { + return new ExactTFIDFDocScorer((IDFStats)stats, context.reader.norms(fieldName)); + } + + @Override + public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { + return new SloppyTFIDFDocScorer((IDFStats)stats, context.reader.norms(fieldName)); + } + + // TODO: we can specialize these for omitNorms up front, but we should test that it doesn't confuse stupid hotspot. + + private final class ExactTFIDFDocScorer extends ExactDocScorer { + private final IDFStats stats; + private final float weightValue; + private final byte[] norms; + private static final int SCORE_CACHE_SIZE = 32; + private float[] scoreCache = new float[SCORE_CACHE_SIZE]; + + ExactTFIDFDocScorer(IDFStats stats, byte norms[]) { + this.stats = stats; + this.weightValue = stats.value; + this.norms = norms; + for (int i = 0; i < SCORE_CACHE_SIZE; i++) + scoreCache[i] = tf(i) * weightValue; + } + + @Override + public float score(int doc, int freq) { + final float raw = // compute tf(f)*weight + freq < SCORE_CACHE_SIZE // check cache + ? scoreCache[freq] // cache hit + : tf(freq)*weightValue; // cache miss + + return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field + } + + @Override + public Explanation explain(int doc, Explanation freq) { + return explainScore(doc, freq, stats, norms); + } + } + + private final class SloppyTFIDFDocScorer extends SloppyDocScorer { + private final IDFStats stats; + private final float weightValue; + private final byte[] norms; + + SloppyTFIDFDocScorer(IDFStats stats, byte norms[]) { + this.stats = stats; + this.weightValue = stats.value; + this.norms = norms; + } + + @Override + public float score(int doc, float freq) { + final float raw = tf(freq) * weightValue; // compute tf(f)*weight + + return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field + } + + @Override + public float computeSlopFactor(int distance) { + return sloppyFreq(distance); + } + + @Override + public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { + return scorePayload(doc, start, end, payload); + } + + @Override + public Explanation explain(int doc, Explanation freq) { + return explainScore(doc, freq, stats, norms); + } + } + + /** Collection statistics for the TF-IDF model. The only statistic of interest + * to this model is idf. */ + private static class IDFStats extends Stats { + /** The idf and its explanation */ + private final Explanation idf; + private float queryNorm; + private float queryWeight; + private final float queryBoost; + private float value; + + public IDFStats(Explanation idf, float queryBoost) { + // TODO: Validate? + this.idf = idf; + this.queryBoost = queryBoost; + this.queryWeight = idf.getValue() * queryBoost; // compute query weight + } + + @Override + public float getValueForNormalization() { + // TODO: (sorta LUCENE-1907) make non-static class and expose this squaring via a nice method to subclasses? + return queryWeight * queryWeight; // sum of squared weights + } + + @Override + public void normalize(float queryNorm, float topLevelBoost) { + this.queryNorm = queryNorm * topLevelBoost; + queryWeight *= this.queryNorm; // normalize query weight + value = queryWeight * idf.getValue(); // idf for document + } + } + + private Explanation explainScore(int doc, Explanation freq, IDFStats stats, byte[] norms) { + Explanation result = new Explanation(); + result.setDescription("score(doc="+doc+",freq="+freq+"), product of:"); + + // explain query weight + Explanation queryExpl = new Explanation(); + queryExpl.setDescription("queryWeight, product of:"); + + Explanation boostExpl = new Explanation(stats.queryBoost, "boost"); + if (stats.queryBoost != 1.0f) + queryExpl.addDetail(boostExpl); + queryExpl.addDetail(stats.idf); + + Explanation queryNormExpl = new Explanation(stats.queryNorm,"queryNorm"); + queryExpl.addDetail(queryNormExpl); + + queryExpl.setValue(boostExpl.getValue() * + stats.idf.getValue() * + queryNormExpl.getValue()); + + result.addDetail(queryExpl); + + // explain field weight + Explanation fieldExpl = new Explanation(); + fieldExpl.setDescription("fieldWeight in "+doc+ + ", product of:"); + + Explanation tfExplanation = new Explanation(); + tfExplanation.setValue(tf(freq.getValue())); + tfExplanation.setDescription("tf(freq="+freq.getValue()+"), with freq of:"); + tfExplanation.addDetail(freq); + fieldExpl.addDetail(tfExplanation); + fieldExpl.addDetail(stats.idf); + + Explanation fieldNormExpl = new Explanation(); + float fieldNorm = + norms!=null ? decodeNormValue(norms[doc]) : 1.0f; + fieldNormExpl.setValue(fieldNorm); + fieldNormExpl.setDescription("fieldNorm(doc="+doc+")"); + fieldExpl.addDetail(fieldNormExpl); + + fieldExpl.setValue(tfExplanation.getValue() * + stats.idf.getValue() * + fieldNormExpl.getValue()); + + result.addDetail(fieldExpl); + + // combine them + result.setValue(queryExpl.getValue() * fieldExpl.getValue()); + + if (queryExpl.getValue() == 1.0f) + return fieldExpl; + + return result; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/Similarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/Similarity.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/Similarity.java 2011-07-17 06:27:33.748948135 -0400 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/Similarity.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,224 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import java.io.IOException; - -import org.apache.lucene.document.IndexDocValuesField; // javadoc -import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.IndexReader; // javadoc -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.Terms; // javadoc -import org.apache.lucene.search.spans.SpanQuery; // javadoc -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.SmallFloat; // javadoc -import org.apache.lucene.util.TermContext; - - -/** - * Similarity defines the components of Lucene scoring. - *
- * Expert: Scoring API. - *
- * This is a low-level API, you should only extend this API if you want to implement - * an information retrieval model. If you are instead looking for a convenient way - * to alter Lucene's scoring, consider extending a higher-level implementation - * such as {@link TFIDFSimilarity}, which implements the vector space model with this API, or - * just tweaking the default implementation: {@link DefaultSimilarity}. - *
- * Similarity determines how Lucene weights terms, and Lucene interacts with - * this class at both index-time and - * query-time. - *
- * - * At indexing time, the indexer calls {@link #computeNorm(FieldInvertState)}, allowing - * the Similarity implementation to return a per-document byte for the field that will - * be later accessible via {@link IndexReader#norms(String)}. Lucene makes no assumption - * about what is in this byte, but it is most useful for encoding length normalization - * information. - *
- * Implementations should carefully consider how the normalization byte is encoded: while - * Lucene's classical {@link TFIDFSimilarity} encodes a combination of index-time boost - * and length normalization information with {@link SmallFloat}, this might not be suitable - * for all purposes. - *
- * Many formulas require the use of average document length, which can be computed via a - * combination of {@link Terms#getSumTotalTermFreq()} and {@link IndexReader#maxDoc()}, - *
- * Because index-time boost is handled entirely at the application level anyway, - * an application can alternatively store the index-time boost separately using an - * {@link IndexDocValuesField}, and access this at query-time with - * {@link IndexReader#docValues(String)}. - *
- * Finally, using index-time boosts (either via folding into the normalization byte or - * via IndexDocValues), is an inefficient way to boost the scores of different fields if the - * boost will be the same for every document, instead the Similarity can simply take a constant - * boost parameter C, and the SimilarityProvider can return different instances with - * different boosts depending upon field name. - *
- * - * At query-time, Queries interact with the Similarity via these steps: - *
- * - * When {@link IndexSearcher#explain(Query, int)} is called, queries consult the Similarity's DocScorer for an - * explanation of how it computed its score. The query passes in a the document id and an explanation of how the frequency - * was computed. - * - * @see org.apache.lucene.index.IndexWriterConfig#setSimilarityProvider(SimilarityProvider) - * @see IndexSearcher#setSimilarityProvider(SimilarityProvider) - * @lucene.experimental - */ -public abstract class Similarity { - /** - * Computes the normalization value for a field, given the accumulated - * state of term processing for this field (see {@link FieldInvertState}). - * - *
Implementations should calculate a byte value based on the field - * state and then return that value. - * - *
Matches in longer fields are less precise, so implementations of this
- * method usually return smaller values when state.getLength() is large,
- * and larger values when state.getLength() is small.
- *
- * @lucene.experimental
- *
- * @param state current processing state for this field
- * @return the calculated byte norm
- */
- public abstract byte computeNorm(FieldInvertState state);
-
- /**
- * Compute any collection-level stats (e.g. IDF, average document length, etc) needed for scoring a query.
- */
- public abstract Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException;
-
- /**
- * returns a new {@link Similarity.ExactDocScorer}.
- */
- public abstract ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException;
-
- /**
- * returns a new {@link Similarity.SloppyDocScorer}.
- */
- public abstract SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException;
-
- /**
- * API for scoring exact queries such as {@link TermQuery} and
- * exact {@link PhraseQuery}.
- *
- * Term frequencies are integers (the term or phrase's tf) - */ - public abstract class ExactDocScorer { - /** - * Score a single document - * @param doc document id - * @param freq term frequency - * @return document's score - */ - public abstract float score(int doc, int freq); - - /** - * Explain the score for a single document - * @param doc document id - * @param freq Explanation of how the term frequency was computed - * @return document's score - */ - public Explanation explain(int doc, Explanation freq) { - Explanation result = new Explanation(score(doc, (int)freq.getValue()), - "score(doc=" + doc + ",freq=" + freq.getValue() +"), with freq of:"); - result.addDetail(freq); - return result; - } - } - - /** - * API for scoring "sloppy" queries such as {@link SpanQuery} and - * sloppy {@link PhraseQuery}. - *
- * Term frequencies are floating point values. - */ - public abstract class SloppyDocScorer { - /** - * Score a single document - * @param doc document id - * @param freq sloppy term frequency - * @return document's score - */ - public abstract float score(int doc, float freq); - - /** Computes the amount of a sloppy phrase match, based on an edit distance. */ - public abstract float computeSlopFactor(int distance); - - /** Calculate a scoring factor based on the data in the payload. */ - public abstract float computePayloadFactor(int doc, int start, int end, BytesRef payload); - - /** - * Explain the score for a single document - * @param doc document id - * @param freq Explanation of how the sloppy term frequency was computed - * @return document's score - */ - public Explanation explain(int doc, Explanation freq) { - Explanation result = new Explanation(score(doc, freq.getValue()), - "score(doc=" + doc + ",freq=" + freq.getValue() +"), with freq of:"); - result.addDetail(freq); - return result; - } - } - - /** Stores the statistics for the indexed collection. This abstract - * implementation is empty; descendants of {@code Similarity} should - * subclass {@code Stats} and define the statistics they require in the - * subclass. Examples include idf, average field length, etc. - */ - public static abstract class Stats { - - /** The value for normalization of contained query clauses (e.g. sum of squared weights). - *
- * NOTE: a Similarity implementation might not use any query normalization at all, - * its not required. However, if it wants to participate in query normalization, - * it can return a value here. - */ - public abstract float getValueForNormalization(); - - /** Assigns the query normalization factor and boost from parent queries to this. - *
- * NOTE: a Similarity implementation might not use this normalized value at all, - * its not required. However, its usually a good idea to at least incorporate - * the topLevelBoost (e.g. from an outer BooleanQuery) into its score. - */ - public abstract void normalize(float queryNorm, float topLevelBoost); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/SimilarityProvider.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/SimilarityProvider.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/SimilarityProvider.java 2011-05-08 10:52:19.360078710 -0400 +++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/SimilarityProvider.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,66 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Expert: Scoring API. - * - * Provides top-level scoring functions that aren't specific to a field, - * and work across multi-field queries (such as {@link BooleanQuery}). - * - * Field-specific scoring is accomplished through {@link Similarity}. - * - * @lucene.experimental - */ -public interface SimilarityProvider { - - /** Computes a score factor based on the fraction of all query terms that a - * document contains. This value is multiplied into scores. - * - *
The presence of a large portion of the query terms indicates a better - * match with the query, so implementations of this method usually return - * larger values when the ratio between these parameters is large and smaller - * values when the ratio between them is small. - * - * @param overlap the number of query terms matched in the document - * @param maxOverlap the total number of terms in the query - * @return a score factor based on term overlap with the query - */ - public abstract float coord(int overlap, int maxOverlap); - - /** Computes the normalization value for a query given the sum of the squared - * weights of each of the query terms. This value is multiplied into the - * weight of each query term. While the classic query normalization factor is - * computed as 1/sqrt(sumOfSquaredWeights), other implementations might - * completely ignore sumOfSquaredWeights (ie return 1). - * - *
This does not affect ranking, but the default implementation does make scores
- * from different queries more comparable than they would be by eliminating the
- * magnitude of the Query vector as a factor in the score.
- *
- * @param sumOfSquaredWeights the sum of the squares of query term weights
- * @return a normalization factor for query weights
- */
- public abstract float queryNorm(float sumOfSquaredWeights);
-
- /** Returns a {@link Similarity} for scoring a field
- * @param field field name.
- * @return a field-specific Similarity.
- */
- public abstract Similarity get(String field);
-}
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java 2011-09-08 17:01:39.466036272 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/SloppyPhraseScorer.java 2011-09-08 12:56:46.716030847 -0400
@@ -20,6 +20,8 @@
import java.io.IOException;
import java.util.LinkedHashSet;
+import org.apache.lucene.search.similarities.Similarity;
+
final class SloppyPhraseScorer extends PhraseScorer {
private int slop;
private PhrasePositions repeats[];
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java 2011-07-17 06:27:33.738948135 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/spans/SpanScorer.java 2011-08-23 08:43:39.731460295 -0400
@@ -20,10 +20,10 @@
import java.io.IOException;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.TFIDFSimilarity;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.Scorer;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.TFIDFSimilarity;
/**
* Public for extension only.
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java 2011-07-17 06:27:33.738948135 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java 2011-08-23 08:43:39.731460295 -0400
@@ -21,7 +21,8 @@
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
-import org.apache.lucene.search.Similarity.SloppyDocScorer;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer;
import org.apache.lucene.util.TermContext;
import java.io.IOException;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/TermQuery.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/TermQuery.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/TermQuery.java 2011-08-20 13:35:07.911460633 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/TermQuery.java 2011-08-23 09:44:48.421460290 -0400
@@ -28,7 +28,8 @@
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.Similarity.ExactDocScorer;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ReaderUtil;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/TermScorer.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/TermScorer.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/TermScorer.java 2011-08-20 13:35:07.931460633 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/TermScorer.java 2011-08-23 09:44:50.111460290 -0400
@@ -20,6 +20,7 @@
import java.io.IOException;
import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.search.similarities.Similarity;
/** Expert: A Scorer for documents matching a Term.
*/
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java 2011-08-29 12:22:46.711459551 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/TFIDFSimilarity.java 1969-12-31 19:00:00.000000000 -0500
@@ -1,863 +0,0 @@
-package org.apache.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-import java.io.IOException;
-
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.TermContext;
-import org.apache.lucene.util.SmallFloat;
-
-
-/**
- * Implementation of {@link Similarity} with the Vector Space Model.
- *
- * Expert: Scoring API. - *
TFIDFSimilarity defines the components of Lucene scoring. - * Overriding computation of these components is a convenient - * way to alter Lucene scoring. - * - *
Suggested reading: - * - * Introduction To Information Retrieval, Chapter 6. - * - *
The following describes how Lucene scoring evolves from - * underlying information retrieval models to (efficient) implementation. - * We first brief on VSM Score, - * then derive from it Lucene's Conceptual Scoring Formula, - * from which, finally, evolves Lucene's Practical Scoring Function - * (the latter is connected directly with Lucene classes and methods). - * - *
Lucene combines - * - * Boolean model (BM) of Information Retrieval - * with - * - * Vector Space Model (VSM) of Information Retrieval - - * documents "approved" by BM are scored by VSM. - * - *
In VSM, documents and queries are represented as - * weighted vectors in a multi-dimensional space, - * where each distinct index term is a dimension, - * and weights are - * Tf-idf values. - * - *
VSM does not require weights to be Tf-idf values, - * but Tf-idf values are believed to produce search results of high quality, - * and so Lucene is using Tf-idf. - * Tf and Idf are described in more detail below, - * but for now, for completion, let's just say that - * for given term t and document (or query) x, - * Tf(t,x) varies with the number of occurrences of term t in x - * (when one increases so does the other) and - * idf(t) similarly varies with the inverse of the - * number of index documents containing term t. - * - *
VSM score of document d for query q is the
- *
- * Cosine Similarity
- * of the weighted query vectors V(q) and V(d):
- *
- *
- *
- *
| ||||||
|
- * |
Note: the above equation can be viewed as the dot product of - * the normalized weighted vectors, in the sense that dividing - * V(q) by its euclidean norm is normalizing it to a unit vector. - * - *
Lucene refines VSM score for both search quality and usability: - *
Under the simplifying assumption of a single field in the index,
- * we get Lucene's Conceptual scoring formula:
- *
- *
- *
- *
| |||||||
|
- * |
The conceptual formula is a simplification in the sense that (1) terms and documents - * are fielded and (2) boosts are usually per query term rather than per query. - * - *
We now describe how Lucene implements this conceptual scoring formula, and - * derive from it Lucene's Practical Scoring Function. - * - *
For efficient score computation some scoring components - * are computed and aggregated in advance: - * - *
Lucene's Practical Scoring Function is derived from the above. - * The color codes demonstrate how it relates - * to those of the conceptual formula: - * - *
- *
- *
| |||||||
|
- * |
where - *
| - * {@link org.apache.lucene.search.DefaultSimilarity#tf(float) tf(t in d)} = - * | - *- * frequency½ - * | - *
| - * {@link org.apache.lucene.search.DefaultSimilarity#idf(int, int) idf(t)} = - * | - *- * 1 + log ( - * | - *
- *
|
- * - * ) - * | - *
| - * queryNorm(q) = - * {@link org.apache.lucene.search.DefaultSimilarityProvider#queryNorm(float) queryNorm(sumOfSquaredWeights)} - * = - * | - *
- *
|
- *
| - * {@link org.apache.lucene.search.Weight#getValueForNormalization() sumOfSquaredWeights} = - * {@link org.apache.lucene.search.Query#getBoost() q.getBoost()} 2 - * · - * | - *- * ∑ - * | - *- * ( - * idf(t) · - * t.getBoost() - * ) 2 - * | - *
| - * | t in q | - *- * |
- * When a document is added to the index, all the above factors are multiplied.
- * If the document has multiple fields with the same name, all their boosts are multiplied together:
- *
- *
- *
| - * norm(t,d) = - * lengthNorm - * · - * | - *- * ∏ - * | - *- * {@link org.apache.lucene.index.IndexableField#boost() f.boost}() - * | - *
| - * | field f in d named as t | - *- * |
Terms and phrases repeated in a document indicate the topic of the
- * document, so implementations of this method usually return larger values
- * when freq is large, and smaller values when freq
- * is small.
- *
- *
The default implementation calls {@link #tf(float)}. - * - * @param freq the frequency of a term within a document - * @return a score factor based on a term's within-document frequency - */ - public float tf(int freq) { - return tf((float)freq); - } - - /** Computes a score factor based on a term or phrase's frequency in a - * document. This value is multiplied by the {@link #idf(int, int)} - * factor for each term in the query and these products are then summed to - * form the initial score for a document. - * - *
Terms and phrases repeated in a document indicate the topic of the
- * document, so implementations of this method usually return larger values
- * when freq is large, and smaller values when freq
- * is small.
- *
- * @param freq the frequency of a term within a document
- * @return a score factor based on a term's within-document frequency
- */
- public abstract float tf(float freq);
-
- /**
- * Computes a score factor for a simple term and returns an explanation
- * for that score factor.
- *
- *
- * The default implementation uses: - * - *
- * idf(docFreq, searcher.maxDoc()); - *- * - * Note that {@link IndexSearcher#maxDoc()} is used instead of - * {@link org.apache.lucene.index.IndexReader#numDocs() IndexReader#numDocs()} because also - * {@link IndexSearcher#docFreq(Term)} is used, and when the latter - * is inaccurate, so is {@link IndexSearcher#maxDoc()}, and in the same direction. - * In addition, {@link IndexSearcher#maxDoc()} is more efficient to compute - * - * @param stats statistics of the term in question - * @param searcher the document collection being searched - * @return an Explain object that includes both an idf score factor - and an explanation for the term. - * @throws IOException - */ - public Explanation idfExplain(TermContext stats, final IndexSearcher searcher) throws IOException { - final int df = stats.docFreq(); - final int max = searcher.maxDoc(); - final float idf = idf(df, max); - return new Explanation(idf, "idf(docFreq=" + df + ", maxDocs=" + max + ")"); - } - - /** - * Computes a score factor for a phrase. - * - *
- * The default implementation sums the idf factor for - * each term in the phrase. - * - * @param stats statistics of the terms in the phrase - * @param searcher the document collection being searched - * @return an Explain object that includes both an idf - * score factor for the phrase and an explanation - * for each term. - * @throws IOException - */ - public Explanation idfExplain(final TermContext stats[], IndexSearcher searcher) throws IOException { - final int max = searcher.maxDoc(); - float idf = 0.0f; - final Explanation exp = new Explanation(); - exp.setDescription("idf(), sum of:"); - for (final TermContext stat : stats ) { - final int df = stat.docFreq(); - final float termIdf = idf(df, max); - exp.addDetail(new Explanation(termIdf, "idf(docFreq=" + df + ", maxDocs=" + max + ")")); - idf += termIdf; - } - exp.setValue(idf); - return exp; - } - - /** Computes a score factor based on a term's document frequency (the number - * of documents which contain the term). This value is multiplied by the - * {@link #tf(int)} factor for each term in the query and these products are - * then summed to form the initial score for a document. - * - *
Terms that occur in fewer documents are better indicators of topic, so - * implementations of this method usually return larger values for rare terms, - * and smaller values for common terms. - * - * @param docFreq the number of documents which contain the term - * @param numDocs the total number of documents in the collection - * @return a score factor based on the term's document frequency - */ - public abstract float idf(int docFreq, int numDocs); - - /** Cache of decoded bytes. */ - private static final float[] NORM_TABLE = new float[256]; - - static { - for (int i = 0; i < 256; i++) - NORM_TABLE[i] = SmallFloat.byte315ToFloat((byte)i); - } - - /** Decodes a normalization factor stored in an index. - * @see #encodeNormValue(float) - */ - public float decodeNormValue(byte b) { - return NORM_TABLE[b & 0xFF]; // & 0xFF maps negative bytes to positive above 127 - } - - /** Encodes a normalization factor for storage in an index. - * - *
The encoding uses a three-bit mantissa, a five-bit exponent, and - * the zero-exponent point at 15, thus - * representing values from around 7x10^9 to 2x10^-9 with about one - * significant decimal digit of accuracy. Zero is also represented. - * Negative numbers are rounded up to zero. Values too large to represent - * are rounded down to the largest representable value. Positive values too - * small to represent are rounded up to the smallest positive representable - * value. - * @see org.apache.lucene.document.Field#setBoost(float) - * @see org.apache.lucene.util.SmallFloat - */ - public byte encodeNormValue(float f) { - return SmallFloat.floatToByte315(f); - } - - /** Computes the amount of a sloppy phrase match, based on an edit distance. - * This value is summed for each sloppy phrase match in a document to form - * the frequency to be used in scoring instead of the exact term count. - * - *
A phrase match with a small edit distance to a document passage more
- * closely matches the document, so implementations of this method usually
- * return larger values when the edit distance is small and smaller values
- * when it is large.
- *
- * @see PhraseQuery#setSlop(int)
- * @param distance the edit distance of this sloppy phrase match
- * @return the frequency increment for this match
- */
- public abstract float sloppyFreq(int distance);
-
- /**
- * Calculate a scoring factor based on the data in the payload. Implementations
- * are responsible for interpreting what is in the payload. Lucene makes no assumptions about
- * what is in the byte array.
- *
- * @param doc The docId currently being scored.
- * @param start The start position of the payload
- * @param end The end position of the payload
- * @param payload The payload byte array to be scored
- * @return An implementation dependent float to be used as a scoring factor
- */
- public abstract float scorePayload(int doc, int start, int end, BytesRef payload);
-
- @Override
- public final Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost,
- TermContext... termContexts) throws IOException {
- final Explanation idf = termContexts.length == 1
- ? idfExplain(termContexts[0], searcher)
- : idfExplain(termContexts, searcher);
- return new IDFStats(idf, queryBoost);
- }
-
- @Override
- public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
- return new ExactTFIDFDocScorer((IDFStats)stats, context.reader.norms(fieldName));
- }
-
- @Override
- public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
- return new SloppyTFIDFDocScorer((IDFStats)stats, context.reader.norms(fieldName));
- }
-
- // TODO: we can specialize these for omitNorms up front, but we should test that it doesn't confuse stupid hotspot.
-
- private final class ExactTFIDFDocScorer extends ExactDocScorer {
- private final IDFStats stats;
- private final float weightValue;
- private final byte[] norms;
- private static final int SCORE_CACHE_SIZE = 32;
- private float[] scoreCache = new float[SCORE_CACHE_SIZE];
-
- ExactTFIDFDocScorer(IDFStats stats, byte norms[]) {
- this.stats = stats;
- this.weightValue = stats.value;
- this.norms = norms;
- for (int i = 0; i < SCORE_CACHE_SIZE; i++)
- scoreCache[i] = tf(i) * weightValue;
- }
-
- @Override
- public float score(int doc, int freq) {
- final float raw = // compute tf(f)*weight
- freq < SCORE_CACHE_SIZE // check cache
- ? scoreCache[freq] // cache hit
- : tf(freq)*weightValue; // cache miss
-
- return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field
- }
-
- @Override
- public Explanation explain(int doc, Explanation freq) {
- return explainScore(doc, freq, stats, norms);
- }
- }
-
- private final class SloppyTFIDFDocScorer extends SloppyDocScorer {
- private final IDFStats stats;
- private final float weightValue;
- private final byte[] norms;
-
- SloppyTFIDFDocScorer(IDFStats stats, byte norms[]) {
- this.stats = stats;
- this.weightValue = stats.value;
- this.norms = norms;
- }
-
- @Override
- public float score(int doc, float freq) {
- final float raw = tf(freq) * weightValue; // compute tf(f)*weight
-
- return norms == null ? raw : raw * decodeNormValue(norms[doc]); // normalize for field
- }
-
- @Override
- public float computeSlopFactor(int distance) {
- return sloppyFreq(distance);
- }
-
- @Override
- public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
- return scorePayload(doc, start, end, payload);
- }
-
- @Override
- public Explanation explain(int doc, Explanation freq) {
- return explainScore(doc, freq, stats, norms);
- }
- }
-
- /** Collection statistics for the TF-IDF model. The only statistic of interest
- * to this model is idf. */
- private static class IDFStats extends Stats {
- /** The idf and its explanation */
- private final Explanation idf;
- private float queryNorm;
- private float queryWeight;
- private final float queryBoost;
- private float value;
-
- public IDFStats(Explanation idf, float queryBoost) {
- // TODO: Validate?
- this.idf = idf;
- this.queryBoost = queryBoost;
- this.queryWeight = idf.getValue() * queryBoost; // compute query weight
- }
-
- @Override
- public float getValueForNormalization() {
- // TODO: (sorta LUCENE-1907) make non-static class and expose this squaring via a nice method to subclasses?
- return queryWeight * queryWeight; // sum of squared weights
- }
-
- @Override
- public void normalize(float queryNorm, float topLevelBoost) {
- this.queryNorm = queryNorm * topLevelBoost;
- queryWeight *= this.queryNorm; // normalize query weight
- value = queryWeight * idf.getValue(); // idf for document
- }
- }
-
- private Explanation explainScore(int doc, Explanation freq, IDFStats stats, byte[] norms) {
- Explanation result = new Explanation();
- result.setDescription("score(doc="+doc+",freq="+freq+"), product of:");
-
- // explain query weight
- Explanation queryExpl = new Explanation();
- queryExpl.setDescription("queryWeight, product of:");
-
- Explanation boostExpl = new Explanation(stats.queryBoost, "boost");
- if (stats.queryBoost != 1.0f)
- queryExpl.addDetail(boostExpl);
- queryExpl.addDetail(stats.idf);
-
- Explanation queryNormExpl = new Explanation(stats.queryNorm,"queryNorm");
- queryExpl.addDetail(queryNormExpl);
-
- queryExpl.setValue(boostExpl.getValue() *
- stats.idf.getValue() *
- queryNormExpl.getValue());
-
- result.addDetail(queryExpl);
-
- // explain field weight
- Explanation fieldExpl = new Explanation();
- fieldExpl.setDescription("fieldWeight in "+doc+
- ", product of:");
-
- Explanation tfExplanation = new Explanation();
- tfExplanation.setValue(tf(freq.getValue()));
- tfExplanation.setDescription("tf(freq="+freq.getValue()+"), with freq of:");
- tfExplanation.addDetail(freq);
- fieldExpl.addDetail(tfExplanation);
- fieldExpl.addDetail(stats.idf);
-
- Explanation fieldNormExpl = new Explanation();
- float fieldNorm =
- norms!=null ? decodeNormValue(norms[doc]) : 1.0f;
- fieldNormExpl.setValue(fieldNorm);
- fieldNormExpl.setDescription("fieldNorm(doc="+doc+")");
- fieldExpl.addDetail(fieldNormExpl);
-
- fieldExpl.setValue(tfExplanation.getValue() *
- stats.idf.getValue() *
- fieldNormExpl.getValue());
-
- result.addDetail(fieldExpl);
-
- // combine them
- result.setValue(queryExpl.getValue() * fieldExpl.getValue());
-
- if (queryExpl.getValue() == 1.0f)
- return fieldExpl;
-
- return result;
- }
-}
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/Weight.java lucene-flexscoring/lucene/src/java/org/apache/lucene/search/Weight.java
--- lucene-clean-trunk/lucene/src/java/org/apache/lucene/search/Weight.java 2011-07-08 01:31:28.421820268 -0400
+++ lucene-flexscoring/lucene/src/java/org/apache/lucene/search/Weight.java 2011-08-23 08:43:39.721460295 -0400
@@ -22,6 +22,7 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
+import org.apache.lucene.search.similarities.SimilarityProvider;
/**
* Expert: Calculate query weights and build query scorers.
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java 2011-08-29 12:22:46.661459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java 2011-08-29 13:43:37.421459544 -0400
@@ -35,13 +35,13 @@
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.NumericRangeQuery;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java 2011-08-29 12:22:46.671459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java 2011-08-29 13:44:09.991459544 -0400
@@ -27,11 +27,11 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java 2011-08-29 12:22:46.671459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java 2011-08-29 13:43:54.351459544 -0400
@@ -25,7 +25,7 @@
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.store.CompoundFileDirectory;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IndexInput;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java 2011-08-29 12:22:46.661459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java 2011-08-29 13:47:38.761459544 -0400
@@ -17,7 +17,7 @@
* limitations under the License.
*/
-import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java 2011-08-29 12:22:46.651459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java 2011-08-29 13:42:58.831459544 -0400
@@ -29,10 +29,10 @@
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.DefaultSimilarity;
-import org.apache.lucene.search.DefaultSimilarityProvider;
-import org.apache.lucene.search.Similarity;
-import org.apache.lucene.search.SimilarityProvider;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java 2011-09-08 17:01:39.456036272 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReader.java 2011-09-08 12:56:42.776030847 -0400
@@ -40,9 +40,9 @@
import org.apache.lucene.index.IndexReader.FieldOption;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java 2011-08-29 12:22:46.661459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java 2011-08-29 13:43:46.771459544 -0400
@@ -23,10 +23,11 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.LuceneTestCase;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java 2011-08-29 12:22:46.671459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexReaderReopen.java 2011-08-29 13:44:03.351459544 -0400
@@ -35,11 +35,13 @@
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.FieldCache;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BitVector;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java 2011-07-27 00:16:39.282328617 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java 2011-08-23 09:44:16.451460290 -0400
@@ -26,8 +26,8 @@
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.DefaultSimilarityProvider;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java 2011-08-29 12:22:46.651459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestMaxTermFrequency.java 2011-08-29 13:48:08.921459544 -0400
@@ -27,9 +27,9 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.DefaultSimilarity;
-import org.apache.lucene.search.DefaultSimilarityProvider;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestNorms.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestNorms.java 2011-08-29 12:22:46.661459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestNorms.java 2011-08-29 13:43:38.631459544 -0400
@@ -27,10 +27,10 @@
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.search.DefaultSimilarity;
-import org.apache.lucene.search.DefaultSimilarityProvider;
-import org.apache.lucene.search.Similarity;
-import org.apache.lucene.search.SimilarityProvider;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestOmitTf.java 2011-08-30 09:57:44.641459442 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestOmitTf.java 2011-08-30 10:04:15.201459442 -0400
@@ -32,6 +32,9 @@
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.*;
import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.SimilarityProvider;
+import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.store.Directory;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestParallelReader.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestParallelReader.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestParallelReader.java 2011-08-29 12:22:46.671459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestParallelReader.java 2011-08-29 22:25:37.861459500 -0400
@@ -25,6 +25,7 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.*;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -234,7 +235,8 @@
w.addDocument(d2);
w.close();
- return new IndexSearcher(dir, false);
+ IndexReader ir = IndexReader.open(dir, false);
+ return newSearcher(ir);
}
// Fields 1 & 2 in one index, 3 & 4 in other, with ParallelReader:
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java 2011-08-29 12:22:46.671459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/index/TestUniqueTermCount.java 2011-08-29 13:49:52.791459544 -0400
@@ -25,9 +25,9 @@
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.search.DefaultSimilarity;
-import org.apache.lucene.search.DefaultSimilarityProvider;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java lucene-flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java 2011-07-17 06:27:33.558948135 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java 2011-08-23 08:43:39.751460295 -0400
@@ -20,9 +20,11 @@
import java.io.IOException;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.search.Similarity.ExactDocScorer;
-import org.apache.lucene.search.Similarity.SloppyDocScorer;
-import org.apache.lucene.search.Similarity.Stats;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.SimilarityProvider;
+import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
+import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer;
+import org.apache.lucene.search.similarities.Similarity.Stats;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.index.FieldInvertState;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java lucene-flexscoring/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java 2011-08-29 12:22:46.381459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/search/payloads/PayloadHelper.java 2011-08-29 13:42:11.991459544 -0400
@@ -29,7 +29,7 @@
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.SimilarityProvider;
+import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadExplanations.java lucene-flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadExplanations.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadExplanations.java 2011-09-08 17:01:39.436036270 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadExplanations.java 2011-09-08 12:58:43.406030891 -0400
@@ -18,9 +18,9 @@
*/
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DefaultSimilarity;
-import org.apache.lucene.search.DefaultSimilarityProvider;
-import org.apache.lucene.search.Similarity;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.TestExplanations;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.util.BytesRef;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java lucene-flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java 2011-08-29 12:22:46.381459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java 2011-08-29 13:42:10.801459544 -0400
@@ -27,14 +27,14 @@
import org.apache.lucene.index.Payload;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.Similarity;
-import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java lucene-flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java 2011-08-29 12:22:46.381459551 -0400
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadTermQuery.java 2011-08-29 13:42:11.401459544 -0400
@@ -20,17 +20,17 @@
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.English;
-import org.apache.lucene.search.DefaultSimilarityProvider;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryUtils;
-import org.apache.lucene.search.Similarity;
-import org.apache.lucene.search.SimilarityProvider;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.CheckHits;
import org.apache.lucene.search.BooleanClause;
import org.apache.lucene.search.BooleanQuery;
-import org.apache.lucene.search.DefaultSimilarity;
+import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
+import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.search.spans.MultiSpansWrapper;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.search.spans.Spans;
diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java lucene-flexscoring/lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java
--- lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java 1969-12-31 19:00:00.000000000 -0500
+++ lucene-flexscoring/lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java 2011-08-29 19:05:54.131459517 -0400
@@ -0,0 +1,211 @@
+package org.apache.lucene.search.similarities;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Comparator;
+import java.util.Map;
+
+import org.apache.lucene.index.CorruptIndexException;
+import org.apache.lucene.index.Fields;
+import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.StoredFieldVisitor;
+import org.apache.lucene.index.TermFreqVector;
+import org.apache.lucene.index.TermVectorMapper;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.codecs.PerDocValues;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
+
+/**
+ * Index searcher implementation that takes an {@link BasicStats} instance and
+ * returns statistics accordingly. Most of the methods are not implemented, so
+ * it can only be used for Similarity unit testing.
+ */
+public class SpoofIndexSearcher extends IndexSearcher {
+ public SpoofIndexSearcher(BasicStats stats) {
+ super(new SpoofIndexReader(stats));
+ }
+
+ public static class SpoofIndexReader extends IndexReader {
+ /** The stats the reader has to return. */
+ protected BasicStats stats;
+ /** The fields the reader has to return. */
+ protected SpoofFields fields;
+
+ public SpoofIndexReader(BasicStats stats) {
+ this.stats = stats;
+ this.fields = new SpoofFields(stats);
+ }
+
+ @Override
+ public int numDocs() {
+ return stats.getNumberOfDocuments();
+ }
+
+ @Override
+ public int maxDoc() {
+ return stats.getNumberOfDocuments();
+ }
+
+ @Override
+ public Fields fields() throws IOException {
+ return fields;
+ }
+
+ @Override
+ public Collection This class maintains a list of
+ * {@code SimilarityBase} subclasses. Each test case performs its test on all
+ * items in the list. If a test case fails, the name of the Similarity that
+ * caused the failure is returned as part of the assertion error message. Unit testing is performed by constructing statistics manually and calling
+ * the {@link SimilarityBase#score(BasicStats, float, int)} method of the
+ * Similarities. The statistics represent corner cases of corpus distributions.
+ * For the integration tests, a small (8-document) collection is indexed. The
+ * tests verify that for a specific query, all relevant documents are returned
+ * in the correct order. The collection consists of two poems of English poet
+ * William Blake. Note: the list of Similarities is maintained by hand. If a new Similarity
+ * is added to the {@code org.apache.lucene.search.similarities} package, the
+ * list should be updated accordingly.
+ * In the correctness tests, the score is verified against the result of manual
+ * computation. Since it would be impossible to test all Similarities
+ * (e.g. all possible DFR combinations, all parameter values for LM), only
+ * the best performing setups in the original papers are verified.
+ *