log(1 + (numDocs - docFreq + 0.5)/(docFreq + 0.5)). */
- public float idf(int docFreq, int numDocs) {
- return (float) Math.log(1 + ((numDocs - docFreq + 0.5D)/(docFreq + 0.5D)));
- }
-
- /** Implemented as 1 / (distance + 1). */
- public float sloppyFreq(int distance) {
- return 1.0f / (distance + 1);
- }
-
- /** The default implementation returns 1 */
- public float scorePayload(int doc, int start, int end, BytesRef payload) {
- return 1;
- }
-
- /** return avg doc length across the field (or 1 if the codec does not store sumTotalTermFreq) */
- public float avgFieldLength(IndexSearcher searcher, String field) throws IOException {
- long sumTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), field).getSumTotalTermFreq();
- long maxdoc = searcher.getIndexReader().maxDoc();
- return sumTotalTermFreq == -1 ? 1f : (float) (sumTotalTermFreq / (double) maxdoc);
- }
-
- @Override
- public byte computeNorm(FieldInvertState state) {
- final int numTerms = state.getLength() - state.getNumOverlap();
- return encodeNormValue(state.getBoost() / (float) Math.sqrt(numTerms));
- }
-
- public float decodeNormValue(byte b) {
- return NORM_TABLE[b & 0xFF];
- }
-
- public byte encodeNormValue(float f) {
- return SmallFloat.floatToByte315(f);
- }
-
- /** Cache of decoded bytes. */
- private static final float[] NORM_TABLE = new float[256];
-
- static {
- for (int i = 0; i < 256; i++) {
- float f = SmallFloat.byte315ToFloat((byte)i);
- NORM_TABLE[i] = 1.0f / (f*f);
- }
- }
-
- @Override
- public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException {
- float value = 0.0f;
- final int max = searcher.maxDoc();
-
- for (final TermContext stat : termStats ) {
- value += idf(stat.docFreq(), max);
- }
-
- return new BM25Stats(value, queryBoost, avgFieldLength(searcher, fieldName));
- }
-
- @Override
- public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
- return new ExactBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName));
- }
-
- @Override
- public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
- return new SloppyBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName));
- }
-
- private class ExactBM25DocScorer extends ExactDocScorer {
- private final float weightValue;
- private final byte[] norms;
- private final float avgdl;
-
- ExactBM25DocScorer(BM25Stats stats, byte norms[]) {
- this.weightValue = stats.weight;
- this.avgdl = stats.avgdl;
- this.norms = norms;
- }
-
- // todo: optimize
- @Override
- public float score(int doc, int freq) {
- // if there are no norms, we act as if b=0
- float norm = norms == null ? k1 : k1 * ((1 - b) + b * (decodeNormValue(norms[doc])) / (avgdl));
- return weightValue * (freq * (k1 + 1)) / (freq + norm);
- }
- }
-
- private class SloppyBM25DocScorer extends SloppyDocScorer {
- private final float weightValue;
- private final byte[] norms;
- private final float avgdl;
-
- SloppyBM25DocScorer(BM25Stats stats, byte norms[]) {
- this.weightValue = stats.weight;
- this.avgdl = stats.avgdl;
- this.norms = norms;
- }
-
- // todo: optimize
- @Override
- public float score(int doc, float freq) {
- // if there are no norms, we act as if b=0
- float norm = norms == null ? k1 : k1 * ((1 - b) + b * (decodeNormValue(norms[doc])) / (avgdl));
- return weightValue * (freq * (k1 + 1)) / (freq + norm);
- }
-
- @Override
- public float computeSlopFactor(int distance) {
- return sloppyFreq(distance);
- }
-
- @Override
- public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
- return scorePayload(doc, start, end, payload);
- }
- }
-
- /** Collection statistics for the BM25 model. */
- private static class BM25Stats extends Stats {
- /** BM25's idf */
- private final float idf;
- /** The average document length. */
- private final float avgdl;
- /** query's inner boost */
- private final float queryBoost;
- /** weight (idf * boost) */
- private float weight;
-
- BM25Stats(float idf, float queryBoost, float avgdl) {
- this.idf = idf;
- this.queryBoost = queryBoost;
- this.avgdl = avgdl;
- }
-
- @Override
- public float getValueForNormalization() {
- // we return a TF-IDF like normalization to be nice, but we don't actually normalize ourselves.
- final float queryWeight = idf * queryBoost;
- return queryWeight * queryWeight;
- }
-
- @Override
- public void normalize(float queryNorm, float topLevelBoost) {
- // we don't normalize with queryNorm at all, we just capture the top-level boost
- this.weight = idf * queryBoost * topLevelBoost;
- }
- }
-}
diff --git lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java
deleted file mode 100644
index 79a943b..0000000
--- lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java
+++ /dev/null
@@ -1,32 +0,0 @@
-package org.apache.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-public class MockBM25SimilarityProvider extends DefaultSimilarityProvider {
-
- private MockBM25Similarity impl = new MockBM25Similarity();
-
- public MockBM25Similarity get(String field) {
- return impl;
- }
-
- /** Sets the {@code Similarity} returned by {@link #get}. */
- public void set(MockBM25Similarity sim) {
- impl = sim;
- }
-}
diff --git lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java
deleted file mode 100644
index 13010f0..0000000
--- lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java
+++ /dev/null
@@ -1,191 +0,0 @@
-package org.apache.lucene.search;
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import java.io.IOException;
-
-import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.MultiFields;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.TermContext;
-import org.apache.lucene.util.SmallFloat;
-
-/**
- * Dirichlet LM Similarity.
- * - * This uses Terrier's modified formula "Bayesian smoothing with Dirichlet Prior" (which ensures only positive scores) - * from Zhai & Lafferty's A Study of Smoothing Methods for Language Models Applied to Information Retrieval. - * The formula has been modified in several ways: - *
- * log(1 + (tf/(mu * (totalTermFrequency / sumOfTotalTermFrequency)))) + log(mu / (numTerms + mu))
- *
- * - * NOTE: to use this Similarity, use MockLMSimilarityProvider (as this formula already incorporates coord() - * and currently depends upon a disabled queryNorm) - * WARNING: doesn't work with preflex codec - */ -public class MockLMSimilarity extends Similarity { - // TODO: the norm table can probably be per-sim so you can configure this. - // its also pretty nice that we don't bake the parameter into the index... you can tune it at runtime. - private static final float mu = 2000f; - - /** - * Our normalization is log(mu / (doclen + mu)) - * currently we put doclen into the boost byte (divided by boost) for simple quantization - * our decoder precomputes the full formula into the norm table - */ - @Override - public byte computeNorm(FieldInvertState state) { - final int numTerms = state.getLength() - state.getNumOverlap(); - return encodeNormValue(numTerms / state.getBoost()); - } - - /** Cache of decoded bytes. */ - private static final float[] NORM_TABLE = new float[256]; - - static { - for (int i = 0; i < 256; i++) { - float doclen = SmallFloat.byte315ToFloat((byte)i); - NORM_TABLE[i] = (float) Math.log(mu / (doclen + mu)); - } - } - - public float decodeNormValue(byte b) { - return NORM_TABLE[b & 0xFF]; - } - - public byte encodeNormValue(float f) { - return SmallFloat.floatToByte315(f); - } - - // weight for a term as 1 / (mu * (totalTermFrequency / sumOfTotalTermFrequency)) - @Override - public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException { - float value = 0.0f; - final StringBuilder exp = new StringBuilder(); - final long sumOfTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), fieldName).getSumTotalTermFreq(); - - for (final TermContext stat : termStats ) { - final long totalTermFrequency = stat.totalTermFreq(); - value += 1 / (mu * ((totalTermFrequency+1L)/(double)(sumOfTotalTermFreq+1L))); - exp.append(" "); - exp.append(totalTermFrequency); - } - - return new LMStats(value, queryBoost); - } - - @Override - public ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { - return new ExactMockLMDocScorer(((LMStats) stats).getValue(), context.reader.norms(fieldName)); - } - - @Override - public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { - return new SloppyMockLMDocScorer(((LMStats) stats).getValue(), context.reader.norms(fieldName)); - } - - /** - * log(1 + (tf/(mu * (totalTermFrequency / sumOfTotalTermFrequency))) ) + log(mu / (numTerms + mu)) - */ - private class ExactMockLMDocScorer extends ExactDocScorer { - private final float weightValue; - private final byte[] norms; - private static final int SCORE_CACHE_SIZE = 32; - private float[] scoreCache = new float[SCORE_CACHE_SIZE]; - - ExactMockLMDocScorer(float weightValue, byte norms[]) { - this.weightValue = weightValue; - this.norms = norms; - for (int i = 0; i < SCORE_CACHE_SIZE; i++) - scoreCache[i] = (float)Math.log(1 + (i*weightValue)); - } - - @Override - public float score(int doc, int freq) { - final float raw = freq < SCORE_CACHE_SIZE // check cache - ? scoreCache[freq] // cache hit - : (float)Math.log(1 + (freq*weightValue)); // cache miss - - return norms == null ? raw : raw + decodeNormValue(norms[doc]); - } - } - - private class SloppyMockLMDocScorer extends SloppyDocScorer { - private final float weightValue; - private final byte[] norms; - - SloppyMockLMDocScorer(float weightValue, byte norms[]) { - this.weightValue = weightValue; - this.norms = norms; - } - - @Override - public float score(int doc, float freq) { - final float raw = (float)Math.log(1 + (freq*weightValue)); - return norms == null ? raw : raw + decodeNormValue(norms[doc]); - } - - @Override - public float computeSlopFactor(int distance) { - return 1.0f / (distance + 1); - } - - @Override - public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { - return 1; - } - } - - public static class LMStats extends Stats { - // dunno if this idf-like thing has a real name, its part1 of the formula to me. - private final float part1; - private final float queryBoost; - private float value; - - public LMStats(float part1, float queryBoost) { - this.part1 = part1; - this.queryBoost = queryBoost; - } - - @Override - public float getValueForNormalization() { - // we return a TF-IDF like normalization to be nice, but we don't actually normalize ourselves. - final float queryWeight = part1 * queryBoost; - return queryWeight * queryWeight; - } - - @Override - public void normalize(float queryNorm, float topLevelBoost) { - // set our value here - this.value = part1 * queryBoost * topLevelBoost; - } - - public float getValue() { - return value; - } - } -} diff --git lucene/src/test/org/apache/lucene/search/MockLMSimilarityProvider.java lucene/src/test/org/apache/lucene/search/MockLMSimilarityProvider.java deleted file mode 100644 index ce56343..0000000 --- lucene/src/test/org/apache/lucene/search/MockLMSimilarityProvider.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * SimilarityProvider for {@link MockLMSimilarity} - *
log(1 + (numDocs - docFreq + 0.5)/(docFreq + 0.5)). */
+ public float idf(int docFreq, int numDocs) {
+ return (float) Math.log(1 + ((numDocs - docFreq + 0.5D)/(docFreq + 0.5D)));
+ }
+
+ /** Implemented as 1 / (distance + 1). */
+ public float sloppyFreq(int distance) {
+ return 1.0f / (distance + 1);
+ }
+
+ /** The default implementation returns 1 */
+ public float scorePayload(int doc, int start, int end, BytesRef payload) {
+ return 1;
+ }
+
+ /** return avg doc length across the field (or 1 if the codec does not store sumTotalTermFreq) */
+ public float avgFieldLength(IndexSearcher searcher, String field) throws IOException {
+ long sumTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), field).getSumTotalTermFreq();
+ long maxdoc = searcher.getIndexReader().maxDoc();
+ return sumTotalTermFreq == -1 ? 1f : (float) (sumTotalTermFreq / (double) maxdoc);
+ }
+
+ @Override
+ public byte computeNorm(FieldInvertState state) {
+ final int numTerms = state.getLength() - state.getNumOverlap();
+ return encodeNormValue(state.getBoost() / (float) Math.sqrt(numTerms));
+ }
+
+ public float decodeNormValue(byte b) {
+ return NORM_TABLE[b & 0xFF];
+ }
+
+ public byte encodeNormValue(float f) {
+ return SmallFloat.floatToByte315(f);
+ }
+
+ /** Cache of decoded bytes. */
+ private static final float[] NORM_TABLE = new float[256];
+
+ static {
+ for (int i = 0; i < 256; i++) {
+ float f = SmallFloat.byte315ToFloat((byte)i);
+ NORM_TABLE[i] = 1.0f / (f*f);
+ }
+ }
+
+ @Override
+ public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException {
+ float value = 0.0f;
+ final int max = searcher.maxDoc();
+
+ for (final TermContext stat : termStats ) {
+ value += idf(stat.docFreq(), max);
+ }
+
+ return new BM25Stats(value, queryBoost, avgFieldLength(searcher, fieldName));
+ }
+
+ @Override
+ public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new ExactBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName));
+ }
+
+ @Override
+ public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException {
+ return new SloppyBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName));
+ }
+
+ private class ExactBM25DocScorer extends ExactDocScorer {
+ private final float weightValue;
+ private final byte[] norms;
+ private final float avgdl;
+
+ ExactBM25DocScorer(BM25Stats stats, byte norms[]) {
+ this.weightValue = stats.weight;
+ this.avgdl = stats.avgdl;
+ this.norms = norms;
+ }
+
+ // todo: optimize
+ @Override
+ public float score(int doc, int freq) {
+ // if there are no norms, we act as if b=0
+ float norm = norms == null ? k1 : k1 * ((1 - b) + b * (decodeNormValue(norms[doc])) / (avgdl));
+ return weightValue * (freq * (k1 + 1)) / (freq + norm);
+ }
+ }
+
+ private class SloppyBM25DocScorer extends SloppyDocScorer {
+ private final float weightValue;
+ private final byte[] norms;
+ private final float avgdl;
+
+ SloppyBM25DocScorer(BM25Stats stats, byte norms[]) {
+ this.weightValue = stats.weight;
+ this.avgdl = stats.avgdl;
+ this.norms = norms;
+ }
+
+ // todo: optimize
+ @Override
+ public float score(int doc, float freq) {
+ // if there are no norms, we act as if b=0
+ float norm = norms == null ? k1 : k1 * ((1 - b) + b * (decodeNormValue(norms[doc])) / (avgdl));
+ return weightValue * (freq * (k1 + 1)) / (freq + norm);
+ }
+
+ @Override
+ public float computeSlopFactor(int distance) {
+ return sloppyFreq(distance);
+ }
+
+ @Override
+ public float computePayloadFactor(int doc, int start, int end, BytesRef payload) {
+ return scorePayload(doc, start, end, payload);
+ }
+ }
+
+ /** Collection statistics for the BM25 model. */
+ private static class BM25Stats extends Stats {
+ /** BM25's idf */
+ private final float idf;
+ /** The average document length. */
+ private final float avgdl;
+ /** query's inner boost */
+ private final float queryBoost;
+ /** weight (idf * boost) */
+ private float weight;
+
+ BM25Stats(float idf, float queryBoost, float avgdl) {
+ this.idf = idf;
+ this.queryBoost = queryBoost;
+ this.avgdl = avgdl;
+ }
+
+ @Override
+ public float getValueForNormalization() {
+ // we return a TF-IDF like normalization to be nice, but we don't actually normalize ourselves.
+ final float queryWeight = idf * queryBoost;
+ return queryWeight * queryWeight;
+ }
+
+ @Override
+ public void normalize(float queryNorm, float topLevelBoost) {
+ // we don't normalize with queryNorm at all, we just capture the top-level boost
+ this.weight = idf * queryBoost * topLevelBoost;
+ }
+ }
+}
diff --git lucene/src/test/org/apache/lucene/search/similarities/BM25SimilarityProvider.java lucene/src/test/org/apache/lucene/search/similarities/BM25SimilarityProvider.java
new file mode 100644
index 0000000..964b081
--- /dev/null
+++ lucene/src/test/org/apache/lucene/search/similarities/BM25SimilarityProvider.java
@@ -0,0 +1,34 @@
+package org.apache.lucene.search.similarities;
+
+import org.apache.lucene.search.DefaultSimilarityProvider;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+public class BM25SimilarityProvider extends DefaultSimilarityProvider {
+
+ private BM25Similarity impl = new BM25Similarity();
+
+ public BM25Similarity get(String field) {
+ return impl;
+ }
+
+ /** Sets the {@code Similarity} returned by {@link #get}. */
+ public void set(BM25Similarity sim) {
+ impl = sim;
+ }
+}