diff --git lucene/src/java/org/apache/lucene/search/similarities/AfterEffect.java lucene/src/java/org/apache/lucene/search/similarities/AfterEffect.java new file mode 100644 index 0000000..faee912 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/AfterEffect.java @@ -0,0 +1,63 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * This class acts as the base class for the implementations of the first + * normalization of the informative content in the DFR framework. This + * component is also called the after effect and is defined by the + * formula Inf2 = 1 - Prob2, where + * Prob2 measures the information gain. + * + * @see DFRSimilarity + * @lucene.experimental + */ +public abstract class AfterEffect { + /** Returns the aftereffect score. */ + public abstract float score(EasyStats stats, float tfn); + + /** Returns an explanation for the score. */ + public abstract Explanation explain(EasyStats stats, float tfn); + + /** Implementation used when there is no aftereffect. */ + public static final class NoAfterEffect extends AfterEffect { + @Override + public final float score(EasyStats stats, float tfn) { + return 1f; + } + + @Override + public final Explanation explain(EasyStats stats, float tfn) { + return new Explanation(1, "no aftereffect"); + } + + @Override + public String toString() { + return ""; + } + } + + /** + * Subclasses must override this method to return the code of the + * after effect formula. Refer to the original paper for the list. + */ + @Override + public abstract String toString(); +} diff --git lucene/src/java/org/apache/lucene/search/similarities/AfterEffectB.java lucene/src/java/org/apache/lucene/search/similarities/AfterEffectB.java new file mode 100644 index 0000000..90079b2 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/AfterEffectB.java @@ -0,0 +1,49 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * Model of the information gain based on the ration of two Bernoulli processes. + * @lucene.experimental + */ +public class AfterEffectB extends AfterEffect { + @Override + public final float score(EasyStats stats, float tfn) { + long F = stats.getTotalTermFreq(); + int n = stats.getDocFreq(); + return (F + 1) / (n * (tfn + 1)); + } + + @Override + public final Explanation explain(EasyStats stats, float tfn) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(score(stats, tfn)); + result.addDetail(new Explanation(tfn, "tfn")); + result.addDetail(new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); + result.addDetail(new Explanation(stats.getDocFreq(), "docFreq")); + return result; + } + + @Override + public String toString() { + return "B"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/AfterEffectL.java lucene/src/java/org/apache/lucene/search/similarities/AfterEffectL.java new file mode 100644 index 0000000..3a1f223 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/AfterEffectL.java @@ -0,0 +1,45 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * Model of the information gain based on Laplace's law of succession. + * @lucene.experimental + */ +public class AfterEffectL extends AfterEffect { + @Override + public final float score(EasyStats stats, float tfn) { + return 1 / (tfn + 1); + } + + @Override + public final Explanation explain(EasyStats stats, float tfn) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(score(stats, tfn)); + result.addDetail(new Explanation(tfn, "tfn")); + return result; + } + + @Override + public String toString() { + return "L"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java new file mode 100644 index 0000000..3f116ff --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java @@ -0,0 +1,202 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Similarity; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SmallFloat; +import org.apache.lucene.util.TermContext; + +/** + * BM25 Similarity. + */ +public class BM25Similarity extends Similarity { + private final float k1; + private final float b; + + public BM25Similarity(float k1, float b) { + this.k1 = k1; + this.b = b; + } + + /** Sets the default values for BM25: + * + */ + public BM25Similarity() { + this.k1 = 1.2f; + this.b = 0.75f; + } + + /** Implemented as log(1 + (numDocs - docFreq + 0.5)/(docFreq + 0.5)). */ + public float idf(int docFreq, int numDocs) { + return (float) Math.log(1 + ((numDocs - docFreq + 0.5D)/(docFreq + 0.5D))); + } + + /** Implemented as 1 / (distance + 1). */ + public float sloppyFreq(int distance) { + return 1.0f / (distance + 1); + } + + /** The default implementation returns 1 */ + public float scorePayload(int doc, int start, int end, BytesRef payload) { + return 1; + } + + /** return avg doc length across the field (or 1 if the codec does not store sumTotalTermFreq) */ + public float avgFieldLength(IndexSearcher searcher, String field) throws IOException { + long sumTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), field).getSumTotalTermFreq(); + long maxdoc = searcher.getIndexReader().maxDoc(); + return sumTotalTermFreq == -1 ? 1f : (float) (sumTotalTermFreq / (double) maxdoc); + } + + @Override + public byte computeNorm(FieldInvertState state) { + final int numTerms = state.getLength() - state.getNumOverlap(); + return encodeNormValue(state.getBoost() / (float) Math.sqrt(numTerms)); + } + + public float decodeNormValue(byte b) { + return NORM_TABLE[b & 0xFF]; + } + + public byte encodeNormValue(float f) { + return SmallFloat.floatToByte315(f); + } + + /** Cache of decoded bytes. */ + private static final float[] NORM_TABLE = new float[256]; + + static { + for (int i = 0; i < 256; i++) { + float f = SmallFloat.byte315ToFloat((byte)i); + NORM_TABLE[i] = 1.0f / (f*f); + } + } + + @Override + public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException { + float value = 0.0f; + final int max = searcher.maxDoc(); + + for (final TermContext stat : termStats ) { + value += idf(stat.docFreq(), max); + } + + return new BM25Stats(value, queryBoost, avgFieldLength(searcher, fieldName)); + } + + @Override + public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { + return new ExactBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName)); + } + + @Override + public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { + return new SloppyBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName)); + } + + private class ExactBM25DocScorer extends ExactDocScorer { + private final float weightValue; + private final byte[] norms; + private final float avgdl; + + ExactBM25DocScorer(BM25Stats stats, byte norms[]) { + this.weightValue = stats.weight; + this.avgdl = stats.avgdl; + this.norms = norms; + } + + // todo: optimize + @Override + public float score(int doc, int freq) { + // if there are no norms, we act as if b=0 + float norm = norms == null ? k1 : k1 * ((1 - b) + b * (decodeNormValue(norms[doc])) / (avgdl)); + return weightValue * (freq * (k1 + 1)) / (freq + norm); + } + } + + private class SloppyBM25DocScorer extends SloppyDocScorer { + private final float weightValue; + private final byte[] norms; + private final float avgdl; + + SloppyBM25DocScorer(BM25Stats stats, byte norms[]) { + this.weightValue = stats.weight; + this.avgdl = stats.avgdl; + this.norms = norms; + } + + // todo: optimize + @Override + public float score(int doc, float freq) { + // if there are no norms, we act as if b=0 + float norm = norms == null ? k1 : k1 * ((1 - b) + b * (decodeNormValue(norms[doc])) / (avgdl)); + return weightValue * (freq * (k1 + 1)) / (freq + norm); + } + + @Override + public float computeSlopFactor(int distance) { + return sloppyFreq(distance); + } + + @Override + public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { + return scorePayload(doc, start, end, payload); + } + } + + /** Collection statistics for the BM25 model. */ + private static class BM25Stats extends Stats { + /** BM25's idf */ + private final float idf; + /** The average document length. */ + private final float avgdl; + /** query's inner boost */ + private final float queryBoost; + /** weight (idf * boost) */ + private float weight; + + BM25Stats(float idf, float queryBoost, float avgdl) { + this.idf = idf; + this.queryBoost = queryBoost; + this.avgdl = avgdl; + } + + @Override + public float getValueForNormalization() { + // we return a TF-IDF like normalization to be nice, but we don't actually normalize ourselves. + final float queryWeight = idf * queryBoost; + return queryWeight * queryWeight; + } + + @Override + public void normalize(float queryNorm, float topLevelBoost) { + // we don't normalize with queryNorm at all, we just capture the top-level boost + this.weight = idf * queryBoost * topLevelBoost; + } + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/BM25SimilarityProvider.java lucene/src/java/org/apache/lucene/search/similarities/BM25SimilarityProvider.java new file mode 100644 index 0000000..964b081 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/BM25SimilarityProvider.java @@ -0,0 +1,34 @@ +package org.apache.lucene.search.similarities; + +import org.apache.lucene.search.DefaultSimilarityProvider; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +public class BM25SimilarityProvider extends DefaultSimilarityProvider { + + private BM25Similarity impl = new BM25Similarity(); + + public BM25Similarity get(String field) { + return impl; + } + + /** Sets the {@code Similarity} returned by {@link #get}. */ + public void set(BM25Similarity sim) { + impl = sim; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModel.java lucene/src/java/org/apache/lucene/search/similarities/BasicModel.java new file mode 100644 index 0000000..e811869 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModel.java @@ -0,0 +1,60 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * This class acts as the base class for the specific basic model + * implementations in the DFR framework. Basic models compute the + * informative content Inf1 = -log2Prob1 + * . + * + * @see DFRSimilarity + * @lucene.experimental + */ +public abstract class BasicModel { + /** Returns the informative content score. */ + public abstract float score(EasyStats stats, float tfn); + + /** + * Returns an explanation for the score. + *

Most basic models use the number of documents and the total term + * frequency to compute Inf1. This method provides a generic + * explanation for such models. Subclasses that use other statistics must + * override this method.

+ */ + public Explanation explain(EasyStats stats, float tfn) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(score(stats, tfn)); + result.addDetail(new Explanation(tfn, "tfn")); + result.addDetail( + new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); + result.addDetail( + new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); + return result; + } + + /** + * Subclasses must override this method to return the code of the + * basic model formula. Refer to the original paper for the list. + */ + @Override + public abstract String toString(); +} diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelBE.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelBE.java new file mode 100644 index 0000000..46216d6 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelBE.java @@ -0,0 +1,48 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.EasySimilarity.log2; + +/** + * Limiting form of the Bose-Einstein model. The formula used in Lucene differs + * slightly from the one in the original paper: to avoid underflow for small + * values of {@code N} and {@code F}, {@code N} is increased by {@code 1} and + * {@code F} is ensured to be at least {@code tfn + 1}. + * @lucene.experimental + */ +public class BasicModelBE extends BasicModel { + @Override + public final float score(EasyStats stats, float tfn) { + long N = stats.getNumberOfDocuments() + 1; +// long F = stats.getTotalTermFreq() + 1; + long F = Math.max(stats.getTotalTermFreq(), (long)(tfn + 0.5) + 1); + return (float)(-log2((N - 1) * Math.E) + + f(N + F - 1, N + F - tfn - 2) - f(F, F - tfn)); + } + + /** The f helper function defined for BE. */ + private final double f(long n, float m) { + return (m + 0.5) * log2((double)n / m) + (n - m) * log2(n); + } + + @Override + public String toString() { + return "Be"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelD.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelD.java new file mode 100644 index 0000000..7c253d9 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelD.java @@ -0,0 +1,47 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.EasySimilarity.log2; + +/** + * Implements the approximation of the binomial model with the divergence + * for DFR. The formula used in Lucene differs slightly from the one in the + * original paper: to avoid underflow for small values of {@code N} and + * {@code F}, {@code N} is increased by {@code 1} and + * {@code F} is ensured to be at least {@code tfn + 1}. + * @lucene.experimental + */ +public class BasicModelD extends BasicModel { + @Override + public final float score(EasyStats stats, float tfn) { + long F = Math.max(stats.getTotalTermFreq(), (long)(tfn + 0.5) + 1); +// long F = stats.getTotalTermFreq() + 1; + double phi = (double)tfn / F; + double nphi = 1 - phi; + double p = 1.0 / (stats.getNumberOfDocuments() + 1); + double D = phi * log2(phi / p) + nphi * log2(nphi / (1 - p)); + // nocommit return (float)(D * F + 0.5 * log2(2 * Math.PI * tfn * nphi)); + return (float)(D * F + 0.5 * log2(1 + 2 * Math.PI * tfn * nphi)); + } + + @Override + public String toString() { + return "D"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelG.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelG.java new file mode 100644 index 0000000..4a8e75a --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelG.java @@ -0,0 +1,38 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.EasySimilarity.log2; + +/** + * Geometric as limiting form of the Bose-Einstein model. + * @lucene.experimental + */ +public class BasicModelG extends BasicModel { + @Override + public final float score(EasyStats stats, float tfn) { + double lambda = stats.getTotalTermFreq() / (double) stats.getNumberOfDocuments(); + // -log(1 / (lambda + 1)) -> log(lambda + 1) + return (float)(log2(lambda + 1) + tfn * log2((1 + lambda) / lambda)); + } + + @Override + public String toString() { + return "G"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelIF.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelIF.java new file mode 100644 index 0000000..c646bb8 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelIF.java @@ -0,0 +1,38 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.EasySimilarity.log2; + +/** + * An approximation of the I(ne) model. + * @lucene.experimental + */ +public class BasicModelIF extends BasicModel { + @Override + public final float score(EasyStats stats, float tfn) { + int N = stats.getNumberOfDocuments(); + long F = stats.getTotalTermFreq(); + return tfn * (float)(log2(1 + (N + 1) / (F + 0.5))); + } + + @Override + public String toString() { + return "I(F)"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelIn.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelIn.java new file mode 100644 index 0000000..3d7bf96 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelIn.java @@ -0,0 +1,52 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; +import static org.apache.lucene.search.similarities.EasySimilarity.log2; + +/** + * The basic tf-idf model of randomness. + * @lucene.experimental + */ +public class BasicModelIn extends BasicModel { + @Override + public final float score(EasyStats stats, float tfn) { + int N = stats.getNumberOfDocuments(); + int n = stats.getDocFreq(); + return tfn * (float)(log2((N + 1) / (n + 0.5))); + } + + @Override + public final Explanation explain(EasyStats stats, float tfn) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(score(stats, tfn)); + result.addDetail(new Explanation(tfn, "tfn")); + result.addDetail( + new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); + result.addDetail( + new Explanation(stats.getDocFreq(), "docFreq")); + return result; + } + + @Override + public String toString() { + return "I(n)"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelIne.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelIne.java new file mode 100644 index 0000000..e9b51ac --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelIne.java @@ -0,0 +1,40 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.EasySimilarity.log2; + +/** + * Tf-idf model of randomness, based on a mixture of Poisson and inverse + * document frequency. + * @lucene.experimental + */ +public class BasicModelIne extends BasicModel { + @Override + public final float score(EasyStats stats, float tfn) { + int N = stats.getNumberOfDocuments(); + long F = stats.getTotalTermFreq(); + double ne = N * (1 - Math.pow((N - 1) / (double)N, F)); + return tfn * (float)(log2((N + 1) / (ne + 0.5))); + } + + @Override + public String toString() { + return "I(ne)"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/BasicModelP.java lucene/src/java/org/apache/lucene/search/similarities/BasicModelP.java new file mode 100644 index 0000000..94ff145 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/BasicModelP.java @@ -0,0 +1,46 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.EasySimilarity.log2; + +/** + * Implements the Poisson approximation for the binomial model for DFR. + * @lucene.experimental + */ +public class BasicModelP extends BasicModel { + /** {@code log2(Math.E)}, precomputed. */ + protected static double LOG2_E = log2(Math.E); + + @Override + public final float score(EasyStats stats, float tfn) { + float lambda = (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); +// System.out.printf("tfn=%f, lambda=%f, log1=%f, log2=%f%n", tfn, lambda, +// tfn / lambda, 2 * Math.PI * tfn); + // nocommit + float score = (float)(tfn * log2(tfn / lambda) + + (lambda + 1 / (12 * tfn) - tfn) * LOG2_E + + 0.5 * log2(2 * Math.PI * tfn)); + return score > 0.0f ? score : 0.0f; + } + + @Override + public String toString() { + return "P"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java new file mode 100644 index 0000000..69dffc3 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/DFRSimilarity.java @@ -0,0 +1,106 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * Implements the divergence from randomness (DFR) framework + * introduced in Gianni Amati and Cornelis Joost Van Rijsbergen. 2002. + * Probabilistic models of information retrieval based on measuring the + * divergence from randomness. ACM Trans. Inf. Syst. 20, 4 (October 2002), + * 357-389. + *

The DFR scoring formula is composed of three separate components: the + * basic model, the aftereffect and an additional + * normalization component, represented by the classes + * {@code BasicModel}, {@code AfterEffect} and {@code Normalization}, + * respectively. The names of these classes were chosen to match the names of + * their counterparts in the Terrier IR engine.

+ *

Note that qtf, the multiplicity of term-occurrence in the query, + * is not handled by this implementation.

+ * + * @see BasicModel + * @see AfterEffect + * @see Normalization + * @lucene.experimental + */ +public class DFRSimilarity extends EasySimilarity { + /** The basic model for information content. */ + protected final BasicModel basicModel; + /** The first normalization of the information content. */ + protected final AfterEffect afterEffect; + /** The term frequency normalization. */ + protected final Normalization normalization; + + public DFRSimilarity(BasicModel basicModel, + AfterEffect afterEffect, + Normalization normalization) { + if (basicModel == null || afterEffect == null || normalization == null) { + throw new NullPointerException("null parameters not allowed."); + } + this.basicModel = basicModel; + this.afterEffect = afterEffect; + this.normalization = normalization; + } + + /** Creates a DFR model with no normalization. */ + public DFRSimilarity(BasicModel basicModel, + AfterEffect afterEffect) { + this(basicModel, afterEffect, new Normalization.NoNormalization()); + } + + /** Creates a DFR model with no aftereffect. */ + public DFRSimilarity(BasicModel basicModel, + Normalization normalization) { + this(basicModel, new AfterEffect.NoAfterEffect(), normalization); + } + + /** Creates a DFR model with only a basic model. */ + public DFRSimilarity(BasicModel basicModel) { + this(basicModel, + new AfterEffect.NoAfterEffect(), + new Normalization.NoNormalization()); + } + + @Override + protected float score(EasyStats stats, float freq, float docLen) { + float tfn = normalization.tfn(stats, freq, docLen); + return stats.getTotalBoost() * + basicModel.score(stats, tfn) * afterEffect.score(stats, tfn); + } + + @Override + protected void explain(Explanation expl, + EasyStats stats, int doc, float freq, float docLen) { + if (stats.getTotalBoost() != 1.0f) { + expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); + } + + Explanation normExpl = normalization.explain(stats, freq, docLen); + float tfn = normExpl.getValue(); + expl.addDetail(normExpl); + expl.addDetail(basicModel.explain(stats, tfn)); + expl.addDetail(afterEffect.explain(stats, tfn)); + } + + @Override + public String toString() { + return "DFR " + basicModel.toString() + afterEffect.toString() + + normalization.toString(); + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/Distribution.java lucene/src/java/org/apache/lucene/search/similarities/Distribution.java new file mode 100644 index 0000000..aa2702a --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/Distribution.java @@ -0,0 +1,45 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * The probabilistic distribution used to model term occurrence + * in information-based models. + * @see IBSimilarity + * @lucene.experimental + */ +public abstract class Distribution { + /** Computes the score. */ + public abstract float score(EasyStats stats, float tfn, float lambda); + + /** Explains the score. Returns the name of the model only, since + * both {@code tfn} and {@code lambda} are explained elsewhere. */ + public Explanation explain(EasyStats stats, float tfn, float lambda) { + return new Explanation( + score(stats, tfn, lambda), getClass().getSimpleName()); + } + + /** + * Subclasses must override this method to return the name of the + * distribution. + */ + @Override + public abstract String toString(); +} diff --git lucene/src/java/org/apache/lucene/search/similarities/DistributionLL.java lucene/src/java/org/apache/lucene/search/similarities/DistributionLL.java new file mode 100644 index 0000000..f7a4a4a --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/DistributionLL.java @@ -0,0 +1,37 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Log-logistic distribution. + *

Unlike for DFR, the natural logarithm is used, as + * it is faster to compute and the original paper does not express any + * preference to a specific base.

+ * @lucene.experimental + */ +public class DistributionLL extends Distribution { + @Override + public final float score(EasyStats stats, float tfn, float lambda) { + return (float)-Math.log(lambda / (tfn + lambda)); + } + + @Override + public String toString() { + return "LL"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/DistributionSPL.java lucene/src/java/org/apache/lucene/search/similarities/DistributionSPL.java new file mode 100644 index 0000000..1027543 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/DistributionSPL.java @@ -0,0 +1,42 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * The smoothed power-law (SPL) distribution for the information-based framework + * that is described in the original paper. + *

Unlike for DFR, the natural logarithm is used, as + * it is faster to compute and the original paper does not express any + * preference to a specific base.

+ * @lucene.experimental + */ +public class DistributionSPL extends Distribution { + @Override + public final float score(EasyStats stats, float tfn, float lambda) { + if (lambda == 1f) { + lambda = 0.99f; + } + return (float)-Math.log( + (Math.pow(lambda, (tfn / (tfn + 1))) - lambda) / (1 - lambda)); + } + + @Override + public String toString() { + return "SPL"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/EasySimilarity.java lucene/src/java/org/apache/lucene/search/similarities/EasySimilarity.java new file mode 100644 index 0000000..4425ba5 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/EasySimilarity.java @@ -0,0 +1,298 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Similarity; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.SmallFloat; +import org.apache.lucene.util.TermContext; + +/** + * A subclass of {@code Similarity} that provides a simplified API for its + * descendants. + * @lucene.experimental + */ +public abstract class EasySimilarity extends Similarity { + /** For {@link #log2(double)}. Precomputed for efficiency reasons. */ + private static final double LOG_2 = Math.log(2); + + /** @see #setDiscountOverlaps */ + protected boolean discountOverlaps = true; + + /** Determines whether overlap tokens (Tokens with + * 0 position increment) are ignored when computing + * norm. By default this is true, meaning overlap + * tokens do not count when computing norms. + * + * @lucene.experimental + * + * @see #computeNorm + */ + public void setDiscountOverlaps(boolean v) { + discountOverlaps = v; + } + + /** @see #setDiscountOverlaps */ + public boolean getDiscountOverlaps() { + return discountOverlaps; + } + + /** + * Calls {@link #fillEasyStats(EasyStats, IndexSearcher, String, TermContext...)}. + * Subclasses that override this method may invoke {@code fillStats} with any + * subclass of {@code EasyStats}. + */ + @Override + public EasyStats computeStats(IndexSearcher searcher, String fieldName, + float queryBoost, TermContext... termContexts) throws IOException { + EasyStats stats = new EasyStats(queryBoost); + fillEasyStats(stats, searcher, fieldName, termContexts); + return stats; + } + + /** Fills all member fields defined in {@code EasyStats} in {@code stats}. */ + protected final void fillEasyStats(EasyStats stats, IndexSearcher searcher, + String fieldName, TermContext... termContexts) throws IOException { + IndexReader reader = searcher.getIndexReader(); + int numberOfDocuments = reader.maxDoc(); + long numberOfFieldTokens = MultiFields.getTerms(searcher.getIndexReader(), + fieldName).getSumTotalTermFreq(); + float avgFieldLength = (float)numberOfFieldTokens / numberOfDocuments; + + // nocommit Take the minimum of term frequencies for phrases. This is not + // correct though, we'll need something like a scorePhrase(MultiStats ...) + int docFreq = Integer.MAX_VALUE; + long totalTermFreq = Integer.MAX_VALUE; + for (final TermContext context : termContexts) { + docFreq = Math.min(docFreq, context.docFreq()); + totalTermFreq = Math.min(totalTermFreq, context.totalTermFreq()); + } + + // We have to provide something if codec doesnt supply these measures, + // or if someone omitted frequencies for the field... negative values cause + // NaN/Inf for some scorers. + if (numberOfFieldTokens == -1) { + numberOfFieldTokens = docFreq; + avgFieldLength = 1; + } + if (totalTermFreq == -1) { + totalTermFreq = docFreq; + } + + stats.setNumberOfDocuments(numberOfDocuments); + stats.setNumberOfFieldTokens(numberOfFieldTokens); + stats.setAvgFieldLength(avgFieldLength); + stats.setDocFreq(docFreq); + stats.setTotalTermFreq(totalTermFreq); + } + + /** + * Scores the document {@code doc}. + *

Subclasses must apply their scoring formula in this class.

+ * @param stats the corpus level statistics. + * @param freq the term frequency. + * @param docLen the document length. + * @return the score. + */ + protected abstract float score(EasyStats stats, float freq, float docLen); + + /** + * Subclasses should implement this method to explain the score. {@code expl} + * already contains the score, the name of the class and the doc id, as well + * as the term frequency and its explanation; subclasses can add additional + * clauses to explain details of their scoring formulae. + *

The default implementation does nothing.

+ * + * @param expl the explanation to extend with details. + * @param stats the corpus level statistics. + * @param doc the document id. + * @param freq the term frequency. + * @param docLen the document length. + */ + protected void explain( + Explanation expl, EasyStats stats, int doc, float freq, float docLen) {} + + /** + * Explains the score. The implementation here provides a basic explanation + * in the format score(name-of-similarity, doc=doc-id, + * freq=term-frequency), computed from:, and + * attaches the score (computed via the {@link #score(EasyStats, float, byte)} + * method) and the explanation for the term frequency. Subclasses content with + * this format may add additional details in + * {@link #explain(Explanation, EasyStats, int, float, byte)}. + * + * @param stats the corpus level statistics. + * @param doc the document id. + * @param freq the term frequency and its explanation. + * @param docLen the document length. + * @return the explanation. + */ + protected Explanation explain( + EasyStats stats, int doc, Explanation freq, float docLen) { + Explanation result = new Explanation(); + result.setValue(score(stats, freq.getValue(), docLen)); + result.setDescription("score(" + getClass().getSimpleName() + + ", doc=" + doc + ", freq=" + freq.getValue() +"), computed from:"); + result.addDetail(freq); + + explain(result, stats, doc, freq.getValue(), docLen); + + return result; + } + + @Override + public ExactDocScorer exactDocScorer(Stats stats, String fieldName, + AtomicReaderContext context) throws IOException { + return new EasyExactDocScorer((EasyStats) stats, + context.reader.norms(fieldName)); + } + + @Override + public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, + AtomicReaderContext context) throws IOException { + return new EasySloppyDocScorer((EasyStats) stats, + context.reader.norms(fieldName)); + } + + /** + * Subclasses must override this method to return the name of the Similarity + * and preferably the values of parameters (if any) as well. + */ + @Override + public abstract String toString(); // nocommit: to Similarity? + + // ------------------------------ Norm handling ------------------------------ + + /** Norm -> document length map. */ + private static final float[] NORM_TABLE = new float[256]; + + static { + for (int i = 0; i < 256; i++) { + float floatNorm = SmallFloat.byte315ToFloat((byte)i); + NORM_TABLE[i] = 1.0f / (floatNorm * floatNorm); + } + } + + /** Encodes the document length in the same way as {@link TFIDFSimilarity}. */ + @Override + public byte computeNorm(FieldInvertState state) { + final float numTerms; + if (discountOverlaps) + numTerms = state.getLength() - state.getNumOverlap(); + else + numTerms = state.getLength() / state.getBoost(); + return encodeNormValue(numTerms); + } + + /** Decodes a normalization factor (document length) stored in an index. + * @see #encodeNormValue(float) + */ + protected float decodeNormValue(byte norm) { + return NORM_TABLE[norm & 0xFF]; // & 0xFF maps negative bytes to positive above 127 + } + + /** Encodes the length to a byte via SmallFloat. */ + protected byte encodeNormValue(float length) { + return SmallFloat.floatToByte315((float)(1.0 / Math.sqrt(length))); + } + + // ----------------------------- Static methods ------------------------------ + + /** Returns the base two logarithm of {@code x}. */ + public static double log2(double x) { + // Put this to a 'util' class if we need more of these. + return Math.log(x) / LOG_2; + } + + // --------------------------------- Classes --------------------------------- + + /** Delegates the {@link #score(int, int)} and + * {@link #explain(int, Explanation)} methods to + * {@link EasySimilarity#score(EasyStats, float, int)} and + * {@link EasySimilarity#explain(EasyStats, int, Explanation, int)}, + * respectively. + */ + private class EasyExactDocScorer extends ExactDocScorer { + private final EasyStats stats; + private final byte[] norms; + + EasyExactDocScorer(EasyStats stats, byte norms[]) { + this.stats = stats; + this.norms = norms; + } + + @Override + public float score(int doc, int freq) { + // We have to supply something in case norms are omitted + return EasySimilarity.this.score(stats, freq, + norms == null ? freq : decodeNormValue(norms[doc])); + } + + @Override + public Explanation explain(int doc, Explanation freq) { + return EasySimilarity.this.explain(stats, doc, freq, + norms == null ? freq.getValue() : decodeNormValue(norms[doc])); + } + } + + /** Delegates the {@link #score(int, int)} and + * {@link #explain(int, Explanation)} methods to + * {@link EasySimilarity#score(EasyStats, float, int)} and + * {@link EasySimilarity#explain(EasyStats, int, Explanation, int)}, + * respectively. + */ + private class EasySloppyDocScorer extends SloppyDocScorer { + private final EasyStats stats; + private final byte[] norms; + + EasySloppyDocScorer(EasyStats stats, byte norms[]) { + this.stats = stats; + this.norms = norms; + } + + @Override + public float score(int doc, float freq) { + // We have to supply something in case norms are omitted + return EasySimilarity.this.score(stats, freq, + norms == null ? freq : decodeNormValue(norms[doc])); + } + @Override + public Explanation explain(int doc, Explanation freq) { + return EasySimilarity.this.explain(stats, doc, freq, + norms == null ? freq.getValue() : decodeNormValue(norms[doc])); + } + + @Override + public float computeSlopFactor(int distance) { + return 1.0f / (distance + 1); + } + + @Override + public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { + return 1f; + } + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/EasyStats.java lucene/src/java/org/apache/lucene/search/similarities/EasyStats.java new file mode 100644 index 0000000..89b3263 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/EasyStats.java @@ -0,0 +1,146 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.Similarity; + +/** + * Stores all statistics commonly used ranking methods. + * @lucene.experimental + */ +public class EasyStats extends Similarity.Stats { + /** The number of documents. */ + protected int numberOfDocuments; + /** The total number of tokens in the field. */ + protected long numberOfFieldTokens; + /** The average field length. */ + protected float avgFieldLength; + /** The document frequency. */ + protected int docFreq; + /** The total number of occurrences of this term across all documents. */ + protected long totalTermFreq; + + // -------------------------- Boost-related stuff -------------------------- + + /** Query's inner boost. */ + protected final float queryBoost; + /** Any outer query's boost. */ + protected float topLevelBoost; + /** For most Similarities, the immediate and the top level query boosts are + * not handled differently. Hence, this field is just the product of the + * other two. */ + protected float totalBoost; + + /** Constructor. Sets the query boost. */ + public EasyStats(float queryBoost) { + this.queryBoost = queryBoost; + this.totalBoost = queryBoost; + } + + // ------------------------- Getter/setter methods ------------------------- + + /** Returns the number of documents. */ + public int getNumberOfDocuments() { + return numberOfDocuments; + } + + /** Sets the number of documents. */ + public void setNumberOfDocuments(int numberOfDocuments) { + this.numberOfDocuments = numberOfDocuments; + } + + /** + * Returns the total number of tokens in the field. + * @see Terms#getSumTotalTermFreq() + */ + public long getNumberOfFieldTokens() { + return numberOfFieldTokens; + } + + /** + * Sets the total number of tokens in the field. + * @see Terms#getSumTotalTermFreq() + */ + public void setNumberOfFieldTokens(long numberOfFieldTokens) { + this.numberOfFieldTokens = numberOfFieldTokens; + } + + /** Returns the average field length. */ + public float getAvgFieldLength() { + return avgFieldLength; + } + + /** Sets the average field length. */ + public void setAvgFieldLength(float avgFieldLength) { + this.avgFieldLength = avgFieldLength; + } + + /** Returns the document frequency. */ + public int getDocFreq() { + return docFreq; + } + + /** Sets the document frequency. */ + public void setDocFreq(int docFreq) { + this.docFreq = docFreq; + } + + /** Returns the total number of occurrences of this term across all documents. */ + public long getTotalTermFreq() { + return totalTermFreq; + } + + /** Sets the total number of occurrences of this term across all documents. */ + public void setTotalTermFreq(long totalTermFreq) { + this.totalTermFreq = totalTermFreq; + } + + // -------------------------- Boost-related stuff -------------------------- + + /** The square of the raw normalization value. + * @see #rawNormalizationValue() */ + @Override + public float getValueForNormalization() { + float rawValue = rawNormalizationValue(); + return rawValue * rawValue; + } + + /** Computes the raw normalization value. This basic implementation returns + * the query boost. Subclasses may override this method to include other + * factors (such as idf), or to save the value for inclusion in + * {@link #normalize(float, float)}, etc. + */ + protected float rawNormalizationValue() { + return queryBoost; + } + + /** No normalization is done. {@code topLevelBoost} is saved in the object, + * however. */ + @Override + public void normalize(float queryNorm, float topLevelBoost) { + this.topLevelBoost = topLevelBoost; + totalBoost = queryBoost * topLevelBoost; + } + + /** Returns the total boost. */ + public float getTotalBoost() { + return totalBoost; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/IBSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/IBSimilarity.java new file mode 100644 index 0000000..07d40e5 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/IBSimilarity.java @@ -0,0 +1,99 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * Provides a framework for the family of information-based models, as described + * in Stéphane Clinchant and Eric Gaussier. 2010. Information-based + * models for ad hoc IR. In Proceeding of the 33rd international ACM SIGIR + * conference on Research and development in information retrieval (SIGIR '10). + * ACM, New York, NY, USA, 234-241. + *

The retrieval function is of the form RSV(q, d) = ∑ + * -xqw log Prob(Xw ≥ + * tdw | λw), where + *

+ *

+ *

The framework described in the paper has many similarities to the DFR + * framework (see {@link DFRSimilarity}). It is possible that the two + * Similarities will be merged at one point.

+ * @lucene.experimental + */ +public class IBSimilarity extends EasySimilarity { + /** The probabilistic distribution used to model term occurrence. */ + protected final Distribution distribution; + /** The lambda (λw) parameter. */ + protected final Lambda lambda; + /** The term frequency normalization. */ + protected final Normalization normalization; + + public IBSimilarity(Distribution distribution, + Lambda lambda, + Normalization normalization) { + this.distribution = distribution; + this.lambda = lambda; + this.normalization = normalization; + } + + /** Creates an instance with no normalization. */ + public IBSimilarity(Distribution distribution, Lambda lambda) { + this(distribution, lambda, new Normalization.NoNormalization()); + } + + @Override + protected float score(EasyStats stats, float freq, float docLen) { + return stats.getTotalBoost() * + distribution.score( + stats, + normalization.tfn(stats, freq, docLen), + lambda.lambda(stats)); + } + + @Override + protected void explain( + Explanation expl, EasyStats stats, int doc, float freq, float docLen) { + if (stats.getTotalBoost() != 1.0f) { + expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); + } + Explanation normExpl = normalization.explain(stats, freq, docLen); + Explanation lambdaExpl = lambda.explain(stats); + expl.addDetail(normExpl); + expl.addDetail(lambdaExpl); + expl.addDetail(distribution.explain( + stats, normExpl.getValue(), lambdaExpl.getValue())); + } + + /** + * The name of IB methods follow the pattern + * {@code IB }. The name of the + * distribution is the same as in the original paper; for the names of lambda + * parameters, refer to the javadoc of the {@link Lambda} classes. + */ + @Override + public String toString() { + return "IB " + distribution.toString() + "-" + lambda.toString() + + normalization.toString(); + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java new file mode 100644 index 0000000..6c8f5f4 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/LMDirichletSimilarity.java @@ -0,0 +1,97 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * Bayesian smoothing using Dirichlet priors. From Chengxiang Zhai and John + * Lafferty. 2001. A study of smoothing methods for language models applied to + * Ad Hoc information retrieval. In Proceedings of the 24th annual international + * ACM SIGIR conference on Research and development in information retrieval + * (SIGIR '01). ACM, New York, NY, USA, 334-342. + *

+ * The formula as defined the paper assigns a negative score to documents that + * contain the term, but with fewer occurrences than predicted by the collection + * language model. The Lucene implementation returns {@code 0} for such + * documents. + *

+ * + * @lucene.experimental + */ +public class LMDirichletSimilarity extends LMSimilarity { + /** The μ parameter. */ + private final float mu; + + /** @param mu the μ parameter. */ + public LMDirichletSimilarity(CollectionModel collectionModel, float mu) { + super(collectionModel); + this.mu = mu; + } + + /** @param mu the μ parameter. */ + public LMDirichletSimilarity(float mu) { + this.mu = mu; + } + + /** Instantiates the similarity with the default μ value of 2000. */ + public LMDirichletSimilarity(CollectionModel collectionModel) { + this(collectionModel, 2000); + } + + /** Instantiates the similarity with the default μ value of 2000. */ + public LMDirichletSimilarity() { + this(2000); + } + + @Override + protected float score(EasyStats stats, float freq, float docLen) { + float score = stats.getTotalBoost() * (float)(Math.log(1 + freq / + (mu * ((LMStats)stats).getCollectionProbability())) + + Math.log(mu / (docLen + mu))); + return score > 0.0f ? score : 0.0f; + } + + @Override + protected void explain(Explanation expl, EasyStats stats, int doc, + float freq, float docLen) { + if (stats.getTotalBoost() != 1.0f) { + expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); + } + + expl.addDetail(new Explanation(mu, "mu")); + Explanation weightExpl = new Explanation(); + weightExpl.setValue((float)Math.log(1 + freq / + (mu * ((LMStats)stats).getCollectionProbability()))); + weightExpl.setDescription("term weight"); + expl.addDetail(weightExpl); + expl.addDetail(new Explanation( + (float)Math.log(mu / (docLen + mu)), "document norm")); + super.explain(expl, stats, doc, freq, docLen); + } + + /** Returns the μ parameter. */ + public float getMu() { + return mu; + } + + @Override + public String getName() { + return String.format("Dirichlet(%f)", getMu()); + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java new file mode 100644 index 0000000..32ff804 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java @@ -0,0 +1,78 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.similarities.LMSimilarity.LMStats; + +/** + * Language model based on the Jelinek-Mercer smoothing method. From Chengxiang + * Zhai and John Lafferty. 2001. A study of smoothing methods for language + * models applied to Ad Hoc information retrieval. In Proceedings of the 24th + * annual international ACM SIGIR conference on Research and development in + * information retrieval (SIGIR '01). ACM, New York, NY, USA, 334-342. + *

The model has a single parameter, λ. According to said paper, the + * optimal value depends on both the collection and the query. The optimal value + * is around {@code 0.1} for title queries and {@code 0.7} for long queries.

+ * + * @lucene.experimental + */ +public class LMJelinekMercerSimilarity extends LMSimilarity { + /** The λ parameter. */ + private final float lambda; + + /** @param lambda the λ parameter. */ + public LMJelinekMercerSimilarity( + CollectionModel collectionModel, float lambda) { + super(collectionModel); + this.lambda = lambda; + } + + /** @param lambda the λ parameter. */ + public LMJelinekMercerSimilarity(float lambda) { + this.lambda = lambda; + } + + @Override + protected float score(EasyStats stats, float freq, float docLen) { + return stats.getTotalBoost() * + (float)Math.log(1 + + ((1 - lambda) * freq / docLen) / + (lambda * ((LMStats)stats).getCollectionProbability())); + } + + @Override + protected void explain(Explanation expl, EasyStats stats, int doc, + float freq, float docLen) { + if (stats.getTotalBoost() != 1.0f) { + expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); + } + expl.addDetail(new Explanation(lambda, "lambda")); + super.explain(expl, stats, doc, freq, docLen); + } + + /** Returns the λ parameter. */ + public float getLambda() { + return lambda; + } + + @Override + public String getName() { + return String.format("Jelinek-Mercer(%f)", getLambda()); + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java new file mode 100644 index 0000000..2aa2e04 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java @@ -0,0 +1,152 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.search.Explanation; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.util.TermContext; + +/** + * Abstract superclass for language modeling Similarities. The following inner + * types are introduced: + *
    + *
  • {@link LMStats}, which defines a new statistic, the probability that + * the collection language model generates the current term;
    • + *
  • {@link CollectionModel}, which is a strategy interface for object that + * compute the collection language model {@code p(w|C)};
    • + *
  • {@link DefaultCollectionModel}, an implementation of the former, that + * computes the term probability as the number of occurrences of the term in the + * collection, divided by the total number of tokens.
    • + *
+ * + * @lucene.experimental + */ +public abstract class LMSimilarity extends EasySimilarity { + /** The collection model. */ + protected final CollectionModel collectionModel; + + /** Creates a new instance with the specified collection language model. */ + public LMSimilarity(CollectionModel collectionModel) { + this.collectionModel = collectionModel; + } + + /** Creates a new instance with the default collection language model. */ + public LMSimilarity() { + this(new DefaultCollectionModel()); + } + + /** + * Computes the collection probability of the current term in addition to the + * usual statistics. + */ + @Override + public EasyStats computeStats(IndexSearcher searcher, String fieldName, + float queryBoost, TermContext... termContexts) throws IOException { + LMStats stats = new LMStats(queryBoost); + fillEasyStats(stats, searcher, fieldName, termContexts); + stats.setCollectionProbability(collectionModel.computeProbability(stats)); + return stats; + } + + @Override + protected void explain(Explanation expl, EasyStats stats, int doc, + float freq, float docLen) { + expl.addDetail(new Explanation(collectionModel.computeProbability(stats), + "collection probability")); + } + + /** + * Returns the name of the LM method. The values of the parameters should be + * included as well. + *

Used in {@link #toString()}

. + */ + public abstract String getName(); + + /** + * Returns the name of the LM method. If a custom collection model strategy is + * used, its name is included as well. + * @see #getName() + * @see CollectionModel#getName() + * @see DefaultCollectionModel + */ + @Override + public String toString() { + String coll = collectionModel.getName(); + if (coll != null) { + return String.format("LM %s - %s", getName(), coll); + } else { + return String.format("LM %s", getName()); + } + } + + /** Stores the collection distribution of the current term. */ + public static class LMStats extends EasyStats { + /** The probability that the current term is generated by the collection. */ + private float collectionProbability; + + public LMStats(float queryBoost) { + super(queryBoost); + } + + /** + * Returns the probability that the current term is generated by the + * collection. + */ + public final float getCollectionProbability() { + return collectionProbability; + } + + /** + * Sets the probability that the current term is generated by the + * collection. + */ + public final void setCollectionProbability(float collectionProbability) { + this.collectionProbability = collectionProbability; + } + } + + /** A strategy for computing the collection language model. */ + public static interface CollectionModel { + /** + * Computes the probability {@code p(w|C)} according to the language model + * strategy for the current term. + */ + public float computeProbability(EasyStats stats); + + /** The name of the collection model strategy. */ + public String getName(); + } + + /** + * Models {@code p(w|C)} as the number of occurrences of the term in the + * collection, divided by the total number of tokens {@code + 1}. + */ + public static class DefaultCollectionModel implements CollectionModel { + @Override + public float computeProbability(EasyStats stats) { + return (float)stats.getTotalTermFreq() / (stats.getNumberOfFieldTokens() +1); + } + + @Override + public String getName() { + return null; + } + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/Lambda.java lucene/src/java/org/apache/lucene/search/similarities/Lambda.java new file mode 100644 index 0000000..5df1b4a --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/Lambda.java @@ -0,0 +1,42 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * The lambda (λw) parameter in information-based + * models. + * @see IBSimilarity + * @lucene.experimental + */ +public abstract class Lambda { + /** Computes the lambda parameter. */ + public abstract float lambda(EasyStats stats); + /** Explains the lambda parameter. */ + public abstract Explanation explain(EasyStats stats); + + /** + * Subclasses must override this method to return the code of the lambda + * formula. Since the original paper is not very clear on this matter, and + * also uses the DFR naming scheme incorrectly, the codes here were chosen + * arbitrarily. + */ + @Override + public abstract String toString(); +} diff --git lucene/src/java/org/apache/lucene/search/similarities/LambdaDF.java lucene/src/java/org/apache/lucene/search/similarities/LambdaDF.java new file mode 100644 index 0000000..4786ec6 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/LambdaDF.java @@ -0,0 +1,48 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * Computes lambda as {@code totalTermFreq / numberOfDocuments}. + * @lucene.experimental + */ +public class LambdaDF extends Lambda { + @Override + public final float lambda(EasyStats stats) { + return (float)stats.getDocFreq() / stats.getNumberOfDocuments(); + } + + @Override + public final Explanation explain(EasyStats stats) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(lambda(stats)); + result.addDetail( + new Explanation(stats.getDocFreq(), "docFreq")); + result.addDetail( + new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); + return result; + } + + @Override + public String toString() { + return "D"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/LambdaTTF.java lucene/src/java/org/apache/lucene/search/similarities/LambdaTTF.java new file mode 100644 index 0000000..f07f146 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/LambdaTTF.java @@ -0,0 +1,48 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * Computes lambda as {@code docFreq / numberOfDocuments}. + * @lucene.experimental + */ +public class LambdaTTF extends Lambda { + @Override + public final float lambda(EasyStats stats) { + return (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); + } + + @Override + public final Explanation explain(EasyStats stats) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(lambda(stats)); + result.addDetail( + new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); + result.addDetail( + new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); + return result; + } + + @Override + public String toString() { + return "L"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/Normalization.java lucene/src/java/org/apache/lucene/search/similarities/Normalization.java new file mode 100644 index 0000000..6a2afea --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/Normalization.java @@ -0,0 +1,75 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.search.Explanation; + +/** + * This class acts as the base class for the implementations of the term + * frequency normalization methods in the DFR framework. + * + * @see DFRSimilarity + * @lucene.experimental + */ +public abstract class Normalization { + /** Returns the normalized term frequency. + * @param len the field length. */ + public abstract float tfn(EasyStats stats, float tf, float len); + + /** Returns an explanation for the normalized term frequency. + *

The default normalization methods use the field length of the document + * and the average field length to compute the normalized term frequency. + * This method provides a generic explanation for such methods. + * Subclasses that use other statistics must override this method.

+ */ + public Explanation explain(EasyStats stats, float tf, float len) { + Explanation result = new Explanation(); + result.setDescription(getClass().getSimpleName() + ", computed from: "); + result.setValue(tfn(stats, tf, len)); + result.addDetail(new Explanation(tf, "tf")); + result.addDetail( + new Explanation(stats.getAvgFieldLength(), "avgFieldLength")); + result.addDetail(new Explanation(len, "len")); + return result; + } + + /** Implementation used when there is no normalization. */ + public static final class NoNormalization extends Normalization { + @Override + public final float tfn(EasyStats stats, float tf, float len) { + return tf; + } + + @Override + public final Explanation explain(EasyStats stats, float tf, float len) { + return new Explanation(1, "no normalization"); + } + + @Override + public String toString() { + return ""; + } + } + + /** + * Subclasses must override this method to return the code of the + * normalization formula. Refer to the original paper for the list. + */ + @Override + public abstract String toString(); +} diff --git lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java new file mode 100644 index 0000000..d289f61 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/NormalizationH1.java @@ -0,0 +1,33 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Normalization model that assumes a uniform distribution of the term frequency. + */ +public class NormalizationH1 extends Normalization { + @Override + public final float tfn(EasyStats stats, float tf, float len) { + return tf * stats.getAvgFieldLength() / len; + } + + @Override + public String toString() { + return "1"; + } +} diff --git lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java new file mode 100644 index 0000000..c49eaf4 --- /dev/null +++ lucene/src/java/org/apache/lucene/search/similarities/NormalizationH2.java @@ -0,0 +1,36 @@ +package org.apache.lucene.search.similarities; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.search.similarities.EasySimilarity.log2; + +/** + * Normalization model in which the term frequency is inversely related to the + * length. + */ +public class NormalizationH2 extends Normalization { + @Override + public final float tfn(EasyStats stats, float tf, float len) { + return (float)(tf * log2(1 + stats.getAvgFieldLength() / len)); + } + + @Override + public String toString() { + return "2"; + } +} \ No newline at end of file diff --git lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java deleted file mode 100644 index b8d98da..0000000 --- lucene/src/test/org/apache/lucene/search/MockBM25Similarity.java +++ /dev/null @@ -1,200 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.MultiFields; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.TermContext; -import org.apache.lucene.util.SmallFloat; - -/** - * BM25 Similarity. - */ -public class MockBM25Similarity extends Similarity { - private final float k1; - private final float b; - - public MockBM25Similarity(float k1, float b) { - this.k1 = k1; - this.b = b; - } - - /** Sets the default values for BM25: - *
    - *
  • {@code k1 = 1.2}, - *
  • {@code b = 0.75}.
    • - *
- */ - public MockBM25Similarity() { - this.k1 = 1.2f; - this.b = 0.75f; - } - - /** Implemented as log(1 + (numDocs - docFreq + 0.5)/(docFreq + 0.5)). */ - public float idf(int docFreq, int numDocs) { - return (float) Math.log(1 + ((numDocs - docFreq + 0.5D)/(docFreq + 0.5D))); - } - - /** Implemented as 1 / (distance + 1). */ - public float sloppyFreq(int distance) { - return 1.0f / (distance + 1); - } - - /** The default implementation returns 1 */ - public float scorePayload(int doc, int start, int end, BytesRef payload) { - return 1; - } - - /** return avg doc length across the field (or 1 if the codec does not store sumTotalTermFreq) */ - public float avgFieldLength(IndexSearcher searcher, String field) throws IOException { - long sumTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), field).getSumTotalTermFreq(); - long maxdoc = searcher.getIndexReader().maxDoc(); - return sumTotalTermFreq == -1 ? 1f : (float) (sumTotalTermFreq / (double) maxdoc); - } - - @Override - public byte computeNorm(FieldInvertState state) { - final int numTerms = state.getLength() - state.getNumOverlap(); - return encodeNormValue(state.getBoost() / (float) Math.sqrt(numTerms)); - } - - public float decodeNormValue(byte b) { - return NORM_TABLE[b & 0xFF]; - } - - public byte encodeNormValue(float f) { - return SmallFloat.floatToByte315(f); - } - - /** Cache of decoded bytes. */ - private static final float[] NORM_TABLE = new float[256]; - - static { - for (int i = 0; i < 256; i++) { - float f = SmallFloat.byte315ToFloat((byte)i); - NORM_TABLE[i] = 1.0f / (f*f); - } - } - - @Override - public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException { - float value = 0.0f; - final int max = searcher.maxDoc(); - - for (final TermContext stat : termStats ) { - value += idf(stat.docFreq(), max); - } - - return new BM25Stats(value, queryBoost, avgFieldLength(searcher, fieldName)); - } - - @Override - public final ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { - return new ExactBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName)); - } - - @Override - public final SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { - return new SloppyBM25DocScorer((BM25Stats) stats, context.reader.norms(fieldName)); - } - - private class ExactBM25DocScorer extends ExactDocScorer { - private final float weightValue; - private final byte[] norms; - private final float avgdl; - - ExactBM25DocScorer(BM25Stats stats, byte norms[]) { - this.weightValue = stats.weight; - this.avgdl = stats.avgdl; - this.norms = norms; - } - - // todo: optimize - @Override - public float score(int doc, int freq) { - // if there are no norms, we act as if b=0 - float norm = norms == null ? k1 : k1 * ((1 - b) + b * (decodeNormValue(norms[doc])) / (avgdl)); - return weightValue * (freq * (k1 + 1)) / (freq + norm); - } - } - - private class SloppyBM25DocScorer extends SloppyDocScorer { - private final float weightValue; - private final byte[] norms; - private final float avgdl; - - SloppyBM25DocScorer(BM25Stats stats, byte norms[]) { - this.weightValue = stats.weight; - this.avgdl = stats.avgdl; - this.norms = norms; - } - - // todo: optimize - @Override - public float score(int doc, float freq) { - // if there are no norms, we act as if b=0 - float norm = norms == null ? k1 : k1 * ((1 - b) + b * (decodeNormValue(norms[doc])) / (avgdl)); - return weightValue * (freq * (k1 + 1)) / (freq + norm); - } - - @Override - public float computeSlopFactor(int distance) { - return sloppyFreq(distance); - } - - @Override - public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { - return scorePayload(doc, start, end, payload); - } - } - - /** Collection statistics for the BM25 model. */ - private static class BM25Stats extends Stats { - /** BM25's idf */ - private final float idf; - /** The average document length. */ - private final float avgdl; - /** query's inner boost */ - private final float queryBoost; - /** weight (idf * boost) */ - private float weight; - - BM25Stats(float idf, float queryBoost, float avgdl) { - this.idf = idf; - this.queryBoost = queryBoost; - this.avgdl = avgdl; - } - - @Override - public float getValueForNormalization() { - // we return a TF-IDF like normalization to be nice, but we don't actually normalize ourselves. - final float queryWeight = idf * queryBoost; - return queryWeight * queryWeight; - } - - @Override - public void normalize(float queryNorm, float topLevelBoost) { - // we don't normalize with queryNorm at all, we just capture the top-level boost - this.weight = idf * queryBoost * topLevelBoost; - } - } -} diff --git lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java deleted file mode 100644 index 79a943b..0000000 --- lucene/src/test/org/apache/lucene/search/MockBM25SimilarityProvider.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class MockBM25SimilarityProvider extends DefaultSimilarityProvider { - - private MockBM25Similarity impl = new MockBM25Similarity(); - - public MockBM25Similarity get(String field) { - return impl; - } - - /** Sets the {@code Similarity} returned by {@link #get}. */ - public void set(MockBM25Similarity sim) { - impl = sim; - } -} diff --git lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java deleted file mode 100644 index 13010f0..0000000 --- lucene/src/test/org/apache/lucene/search/MockLMSimilarity.java +++ /dev/null @@ -1,191 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.MultiFields; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.TermContext; -import org.apache.lucene.util.SmallFloat; - -/** - * Dirichlet LM Similarity. - *

- * This uses Terrier's modified formula "Bayesian smoothing with Dirichlet Prior" (which ensures only positive scores) - * from Zhai & Lafferty's A Study of Smoothing Methods for Language Models Applied to Information Retrieval. - * The formula has been modified in several ways: - *

    - *
  • Supports doc/field/query boosting - *
  • Uses natural log instead of base2 for simplicity. - *
  • The formula has been re-arranged: as part is computed in the weight, we use totalTermFrequency+1/sumOfTotalTermFrequency+1 - * to prevent any divide by zero, other parts of the formula re-arranged for performance - *
- *

- * - * log(1 + (tf/(mu * (totalTermFrequency / sumOfTotalTermFrequency)))) + log(mu / (numTerms + mu)) - * - *

- * NOTE: to use this Similarity, use MockLMSimilarityProvider (as this formula already incorporates coord() - * and currently depends upon a disabled queryNorm) - * WARNING: doesn't work with preflex codec - */ -public class MockLMSimilarity extends Similarity { - // TODO: the norm table can probably be per-sim so you can configure this. - // its also pretty nice that we don't bake the parameter into the index... you can tune it at runtime. - private static final float mu = 2000f; - - /** - * Our normalization is log(mu / (doclen + mu)) - * currently we put doclen into the boost byte (divided by boost) for simple quantization - * our decoder precomputes the full formula into the norm table - */ - @Override - public byte computeNorm(FieldInvertState state) { - final int numTerms = state.getLength() - state.getNumOverlap(); - return encodeNormValue(numTerms / state.getBoost()); - } - - /** Cache of decoded bytes. */ - private static final float[] NORM_TABLE = new float[256]; - - static { - for (int i = 0; i < 256; i++) { - float doclen = SmallFloat.byte315ToFloat((byte)i); - NORM_TABLE[i] = (float) Math.log(mu / (doclen + mu)); - } - } - - public float decodeNormValue(byte b) { - return NORM_TABLE[b & 0xFF]; - } - - public byte encodeNormValue(float f) { - return SmallFloat.floatToByte315(f); - } - - // weight for a term as 1 / (mu * (totalTermFrequency / sumOfTotalTermFrequency)) - @Override - public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException { - float value = 0.0f; - final StringBuilder exp = new StringBuilder(); - final long sumOfTotalTermFreq = MultiFields.getTerms(searcher.getIndexReader(), fieldName).getSumTotalTermFreq(); - - for (final TermContext stat : termStats ) { - final long totalTermFrequency = stat.totalTermFreq(); - value += 1 / (mu * ((totalTermFrequency+1L)/(double)(sumOfTotalTermFreq+1L))); - exp.append(" "); - exp.append(totalTermFrequency); - } - - return new LMStats(value, queryBoost); - } - - @Override - public ExactDocScorer exactDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { - return new ExactMockLMDocScorer(((LMStats) stats).getValue(), context.reader.norms(fieldName)); - } - - @Override - public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, AtomicReaderContext context) throws IOException { - return new SloppyMockLMDocScorer(((LMStats) stats).getValue(), context.reader.norms(fieldName)); - } - - /** - * log(1 + (tf/(mu * (totalTermFrequency / sumOfTotalTermFrequency))) ) + log(mu / (numTerms + mu)) - */ - private class ExactMockLMDocScorer extends ExactDocScorer { - private final float weightValue; - private final byte[] norms; - private static final int SCORE_CACHE_SIZE = 32; - private float[] scoreCache = new float[SCORE_CACHE_SIZE]; - - ExactMockLMDocScorer(float weightValue, byte norms[]) { - this.weightValue = weightValue; - this.norms = norms; - for (int i = 0; i < SCORE_CACHE_SIZE; i++) - scoreCache[i] = (float)Math.log(1 + (i*weightValue)); - } - - @Override - public float score(int doc, int freq) { - final float raw = freq < SCORE_CACHE_SIZE // check cache - ? scoreCache[freq] // cache hit - : (float)Math.log(1 + (freq*weightValue)); // cache miss - - return norms == null ? raw : raw + decodeNormValue(norms[doc]); - } - } - - private class SloppyMockLMDocScorer extends SloppyDocScorer { - private final float weightValue; - private final byte[] norms; - - SloppyMockLMDocScorer(float weightValue, byte norms[]) { - this.weightValue = weightValue; - this.norms = norms; - } - - @Override - public float score(int doc, float freq) { - final float raw = (float)Math.log(1 + (freq*weightValue)); - return norms == null ? raw : raw + decodeNormValue(norms[doc]); - } - - @Override - public float computeSlopFactor(int distance) { - return 1.0f / (distance + 1); - } - - @Override - public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { - return 1; - } - } - - public static class LMStats extends Stats { - // dunno if this idf-like thing has a real name, its part1 of the formula to me. - private final float part1; - private final float queryBoost; - private float value; - - public LMStats(float part1, float queryBoost) { - this.part1 = part1; - this.queryBoost = queryBoost; - } - - @Override - public float getValueForNormalization() { - // we return a TF-IDF like normalization to be nice, but we don't actually normalize ourselves. - final float queryWeight = part1 * queryBoost; - return queryWeight * queryWeight; - } - - @Override - public void normalize(float queryNorm, float topLevelBoost) { - // set our value here - this.value = part1 * queryBoost * topLevelBoost; - } - - public float getValue() { - return value; - } - } -} diff --git lucene/src/test/org/apache/lucene/search/MockLMSimilarityProvider.java lucene/src/test/org/apache/lucene/search/MockLMSimilarityProvider.java deleted file mode 100644 index ce56343..0000000 --- lucene/src/test/org/apache/lucene/search/MockLMSimilarityProvider.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * SimilarityProvider for {@link MockLMSimilarity} - *

    - *
  • disables coord, because its already factored into the formula - *
  • disables queryNorm, because we (currently) shove part of the formula in there as "idf" - *
- * WARNING: doesn't work with preflex codec - */ -public class MockLMSimilarityProvider implements SimilarityProvider { - private static final Similarity impl = new MockLMSimilarity(); - - public float coord(int overlap, int maxOverlap) { - return 1f; - } - - public float queryNorm(float sumOfSquaredWeights) { - return 1f; - } - - public Similarity get(String field) { - return impl; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java deleted file mode 100644 index faee912..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/AfterEffect.java +++ /dev/null @@ -1,63 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * This class acts as the base class for the implementations of the first - * normalization of the informative content in the DFR framework. This - * component is also called the after effect and is defined by the - * formula Inf2 = 1 - Prob2, where - * Prob2 measures the information gain. - * - * @see DFRSimilarity - * @lucene.experimental - */ -public abstract class AfterEffect { - /** Returns the aftereffect score. */ - public abstract float score(EasyStats stats, float tfn); - - /** Returns an explanation for the score. */ - public abstract Explanation explain(EasyStats stats, float tfn); - - /** Implementation used when there is no aftereffect. */ - public static final class NoAfterEffect extends AfterEffect { - @Override - public final float score(EasyStats stats, float tfn) { - return 1f; - } - - @Override - public final Explanation explain(EasyStats stats, float tfn) { - return new Explanation(1, "no aftereffect"); - } - - @Override - public String toString() { - return ""; - } - } - - /** - * Subclasses must override this method to return the code of the - * after effect formula. Refer to the original paper for the list. - */ - @Override - public abstract String toString(); -} diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java deleted file mode 100644 index 90079b2..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/AfterEffectB.java +++ /dev/null @@ -1,49 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * Model of the information gain based on the ration of two Bernoulli processes. - * @lucene.experimental - */ -public class AfterEffectB extends AfterEffect { - @Override - public final float score(EasyStats stats, float tfn) { - long F = stats.getTotalTermFreq(); - int n = stats.getDocFreq(); - return (F + 1) / (n * (tfn + 1)); - } - - @Override - public final Explanation explain(EasyStats stats, float tfn) { - Explanation result = new Explanation(); - result.setDescription(getClass().getSimpleName() + ", computed from: "); - result.setValue(score(stats, tfn)); - result.addDetail(new Explanation(tfn, "tfn")); - result.addDetail(new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); - result.addDetail(new Explanation(stats.getDocFreq(), "docFreq")); - return result; - } - - @Override - public String toString() { - return "B"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java deleted file mode 100644 index 3a1f223..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/AfterEffectL.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * Model of the information gain based on Laplace's law of succession. - * @lucene.experimental - */ -public class AfterEffectL extends AfterEffect { - @Override - public final float score(EasyStats stats, float tfn) { - return 1 / (tfn + 1); - } - - @Override - public final Explanation explain(EasyStats stats, float tfn) { - Explanation result = new Explanation(); - result.setDescription(getClass().getSimpleName() + ", computed from: "); - result.setValue(score(stats, tfn)); - result.addDetail(new Explanation(tfn, "tfn")); - return result; - } - - @Override - public String toString() { - return "L"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java deleted file mode 100644 index e811869..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModel.java +++ /dev/null @@ -1,60 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * This class acts as the base class for the specific basic model - * implementations in the DFR framework. Basic models compute the - * informative content Inf1 = -log2Prob1 - * . - * - * @see DFRSimilarity - * @lucene.experimental - */ -public abstract class BasicModel { - /** Returns the informative content score. */ - public abstract float score(EasyStats stats, float tfn); - - /** - * Returns an explanation for the score. - *

Most basic models use the number of documents and the total term - * frequency to compute Inf1. This method provides a generic - * explanation for such models. Subclasses that use other statistics must - * override this method.

- */ - public Explanation explain(EasyStats stats, float tfn) { - Explanation result = new Explanation(); - result.setDescription(getClass().getSimpleName() + ", computed from: "); - result.setValue(score(stats, tfn)); - result.addDetail(new Explanation(tfn, "tfn")); - result.addDetail( - new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); - result.addDetail( - new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); - return result; - } - - /** - * Subclasses must override this method to return the code of the - * basic model formula. Refer to the original paper for the list. - */ - @Override - public abstract String toString(); -} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java deleted file mode 100644 index 46216d6..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelBE.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.apache.lucene.search.similarities.EasySimilarity.log2; - -/** - * Limiting form of the Bose-Einstein model. The formula used in Lucene differs - * slightly from the one in the original paper: to avoid underflow for small - * values of {@code N} and {@code F}, {@code N} is increased by {@code 1} and - * {@code F} is ensured to be at least {@code tfn + 1}. - * @lucene.experimental - */ -public class BasicModelBE extends BasicModel { - @Override - public final float score(EasyStats stats, float tfn) { - long N = stats.getNumberOfDocuments() + 1; -// long F = stats.getTotalTermFreq() + 1; - long F = Math.max(stats.getTotalTermFreq(), (long)(tfn + 0.5) + 1); - return (float)(-log2((N - 1) * Math.E) - + f(N + F - 1, N + F - tfn - 2) - f(F, F - tfn)); - } - - /** The f helper function defined for BE. */ - private final double f(long n, float m) { - return (m + 0.5) * log2((double)n / m) + (n - m) * log2(n); - } - - @Override - public String toString() { - return "Be"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java deleted file mode 100644 index 7c253d9..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelD.java +++ /dev/null @@ -1,47 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.apache.lucene.search.similarities.EasySimilarity.log2; - -/** - * Implements the approximation of the binomial model with the divergence - * for DFR. The formula used in Lucene differs slightly from the one in the - * original paper: to avoid underflow for small values of {@code N} and - * {@code F}, {@code N} is increased by {@code 1} and - * {@code F} is ensured to be at least {@code tfn + 1}. - * @lucene.experimental - */ -public class BasicModelD extends BasicModel { - @Override - public final float score(EasyStats stats, float tfn) { - long F = Math.max(stats.getTotalTermFreq(), (long)(tfn + 0.5) + 1); -// long F = stats.getTotalTermFreq() + 1; - double phi = (double)tfn / F; - double nphi = 1 - phi; - double p = 1.0 / (stats.getNumberOfDocuments() + 1); - double D = phi * log2(phi / p) + nphi * log2(nphi / (1 - p)); - // nocommit return (float)(D * F + 0.5 * log2(2 * Math.PI * tfn * nphi)); - return (float)(D * F + 0.5 * log2(1 + 2 * Math.PI * tfn * nphi)); - } - - @Override - public String toString() { - return "D"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java deleted file mode 100644 index 4a8e75a..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelG.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.apache.lucene.search.similarities.EasySimilarity.log2; - -/** - * Geometric as limiting form of the Bose-Einstein model. - * @lucene.experimental - */ -public class BasicModelG extends BasicModel { - @Override - public final float score(EasyStats stats, float tfn) { - double lambda = stats.getTotalTermFreq() / (double) stats.getNumberOfDocuments(); - // -log(1 / (lambda + 1)) -> log(lambda + 1) - return (float)(log2(lambda + 1) + tfn * log2((1 + lambda) / lambda)); - } - - @Override - public String toString() { - return "G"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java deleted file mode 100644 index c646bb8..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelIF.java +++ /dev/null @@ -1,38 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.apache.lucene.search.similarities.EasySimilarity.log2; - -/** - * An approximation of the I(ne) model. - * @lucene.experimental - */ -public class BasicModelIF extends BasicModel { - @Override - public final float score(EasyStats stats, float tfn) { - int N = stats.getNumberOfDocuments(); - long F = stats.getTotalTermFreq(); - return tfn * (float)(log2(1 + (N + 1) / (F + 0.5))); - } - - @Override - public String toString() { - return "I(F)"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java deleted file mode 100644 index 3d7bf96..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelIn.java +++ /dev/null @@ -1,52 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; -import static org.apache.lucene.search.similarities.EasySimilarity.log2; - -/** - * The basic tf-idf model of randomness. - * @lucene.experimental - */ -public class BasicModelIn extends BasicModel { - @Override - public final float score(EasyStats stats, float tfn) { - int N = stats.getNumberOfDocuments(); - int n = stats.getDocFreq(); - return tfn * (float)(log2((N + 1) / (n + 0.5))); - } - - @Override - public final Explanation explain(EasyStats stats, float tfn) { - Explanation result = new Explanation(); - result.setDescription(getClass().getSimpleName() + ", computed from: "); - result.setValue(score(stats, tfn)); - result.addDetail(new Explanation(tfn, "tfn")); - result.addDetail( - new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); - result.addDetail( - new Explanation(stats.getDocFreq(), "docFreq")); - return result; - } - - @Override - public String toString() { - return "I(n)"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java deleted file mode 100644 index e9b51ac..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelIne.java +++ /dev/null @@ -1,40 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.apache.lucene.search.similarities.EasySimilarity.log2; - -/** - * Tf-idf model of randomness, based on a mixture of Poisson and inverse - * document frequency. - * @lucene.experimental - */ -public class BasicModelIne extends BasicModel { - @Override - public final float score(EasyStats stats, float tfn) { - int N = stats.getNumberOfDocuments(); - long F = stats.getTotalTermFreq(); - double ne = N * (1 - Math.pow((N - 1) / (double)N, F)); - return tfn * (float)(log2((N + 1) / (ne + 0.5))); - } - - @Override - public String toString() { - return "I(ne)"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java deleted file mode 100644 index 94ff145..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/BasicModelP.java +++ /dev/null @@ -1,46 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.apache.lucene.search.similarities.EasySimilarity.log2; - -/** - * Implements the Poisson approximation for the binomial model for DFR. - * @lucene.experimental - */ -public class BasicModelP extends BasicModel { - /** {@code log2(Math.E)}, precomputed. */ - protected static double LOG2_E = log2(Math.E); - - @Override - public final float score(EasyStats stats, float tfn) { - float lambda = (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); -// System.out.printf("tfn=%f, lambda=%f, log1=%f, log2=%f%n", tfn, lambda, -// tfn / lambda, 2 * Math.PI * tfn); - // nocommit - float score = (float)(tfn * log2(tfn / lambda) - + (lambda + 1 / (12 * tfn) - tfn) * LOG2_E - + 0.5 * log2(2 * Math.PI * tfn)); - return score > 0.0f ? score : 0.0f; - } - - @Override - public String toString() { - return "P"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java deleted file mode 100644 index 69dffc3..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/DFRSimilarity.java +++ /dev/null @@ -1,106 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * Implements the divergence from randomness (DFR) framework - * introduced in Gianni Amati and Cornelis Joost Van Rijsbergen. 2002. - * Probabilistic models of information retrieval based on measuring the - * divergence from randomness. ACM Trans. Inf. Syst. 20, 4 (October 2002), - * 357-389. - *

The DFR scoring formula is composed of three separate components: the - * basic model, the aftereffect and an additional - * normalization component, represented by the classes - * {@code BasicModel}, {@code AfterEffect} and {@code Normalization}, - * respectively. The names of these classes were chosen to match the names of - * their counterparts in the Terrier IR engine.

- *

Note that qtf, the multiplicity of term-occurrence in the query, - * is not handled by this implementation.

- * - * @see BasicModel - * @see AfterEffect - * @see Normalization - * @lucene.experimental - */ -public class DFRSimilarity extends EasySimilarity { - /** The basic model for information content. */ - protected final BasicModel basicModel; - /** The first normalization of the information content. */ - protected final AfterEffect afterEffect; - /** The term frequency normalization. */ - protected final Normalization normalization; - - public DFRSimilarity(BasicModel basicModel, - AfterEffect afterEffect, - Normalization normalization) { - if (basicModel == null || afterEffect == null || normalization == null) { - throw new NullPointerException("null parameters not allowed."); - } - this.basicModel = basicModel; - this.afterEffect = afterEffect; - this.normalization = normalization; - } - - /** Creates a DFR model with no normalization. */ - public DFRSimilarity(BasicModel basicModel, - AfterEffect afterEffect) { - this(basicModel, afterEffect, new Normalization.NoNormalization()); - } - - /** Creates a DFR model with no aftereffect. */ - public DFRSimilarity(BasicModel basicModel, - Normalization normalization) { - this(basicModel, new AfterEffect.NoAfterEffect(), normalization); - } - - /** Creates a DFR model with only a basic model. */ - public DFRSimilarity(BasicModel basicModel) { - this(basicModel, - new AfterEffect.NoAfterEffect(), - new Normalization.NoNormalization()); - } - - @Override - protected float score(EasyStats stats, float freq, float docLen) { - float tfn = normalization.tfn(stats, freq, docLen); - return stats.getTotalBoost() * - basicModel.score(stats, tfn) * afterEffect.score(stats, tfn); - } - - @Override - protected void explain(Explanation expl, - EasyStats stats, int doc, float freq, float docLen) { - if (stats.getTotalBoost() != 1.0f) { - expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); - } - - Explanation normExpl = normalization.explain(stats, freq, docLen); - float tfn = normExpl.getValue(); - expl.addDetail(normExpl); - expl.addDetail(basicModel.explain(stats, tfn)); - expl.addDetail(afterEffect.explain(stats, tfn)); - } - - @Override - public String toString() { - return "DFR " + basicModel.toString() + afterEffect.toString() - + normalization.toString(); - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/Distribution.java lucene/src/test/org/apache/lucene/search/similarities/Distribution.java deleted file mode 100644 index aa2702a..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/Distribution.java +++ /dev/null @@ -1,45 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * The probabilistic distribution used to model term occurrence - * in information-based models. - * @see IBSimilarity - * @lucene.experimental - */ -public abstract class Distribution { - /** Computes the score. */ - public abstract float score(EasyStats stats, float tfn, float lambda); - - /** Explains the score. Returns the name of the model only, since - * both {@code tfn} and {@code lambda} are explained elsewhere. */ - public Explanation explain(EasyStats stats, float tfn, float lambda) { - return new Explanation( - score(stats, tfn, lambda), getClass().getSimpleName()); - } - - /** - * Subclasses must override this method to return the name of the - * distribution. - */ - @Override - public abstract String toString(); -} diff --git lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java deleted file mode 100644 index f7a4a4a..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/DistributionLL.java +++ /dev/null @@ -1,37 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Log-logistic distribution. - *

Unlike for DFR, the natural logarithm is used, as - * it is faster to compute and the original paper does not express any - * preference to a specific base.

- * @lucene.experimental - */ -public class DistributionLL extends Distribution { - @Override - public final float score(EasyStats stats, float tfn, float lambda) { - return (float)-Math.log(lambda / (tfn + lambda)); - } - - @Override - public String toString() { - return "LL"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java deleted file mode 100644 index 1027543..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/DistributionSPL.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * The smoothed power-law (SPL) distribution for the information-based framework - * that is described in the original paper. - *

Unlike for DFR, the natural logarithm is used, as - * it is faster to compute and the original paper does not express any - * preference to a specific base.

- * @lucene.experimental - */ -public class DistributionSPL extends Distribution { - @Override - public final float score(EasyStats stats, float tfn, float lambda) { - if (lambda == 1f) { - lambda = 0.99f; - } - return (float)-Math.log( - (Math.pow(lambda, (tfn / (tfn + 1))) - lambda) / (1 - lambda)); - } - - @Override - public String toString() { - return "SPL"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java deleted file mode 100644 index 4425ba5..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/EasySimilarity.java +++ /dev/null @@ -1,298 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.MultiFields; -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Similarity; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.SmallFloat; -import org.apache.lucene.util.TermContext; - -/** - * A subclass of {@code Similarity} that provides a simplified API for its - * descendants. - * @lucene.experimental - */ -public abstract class EasySimilarity extends Similarity { - /** For {@link #log2(double)}. Precomputed for efficiency reasons. */ - private static final double LOG_2 = Math.log(2); - - /** @see #setDiscountOverlaps */ - protected boolean discountOverlaps = true; - - /** Determines whether overlap tokens (Tokens with - * 0 position increment) are ignored when computing - * norm. By default this is true, meaning overlap - * tokens do not count when computing norms. - * - * @lucene.experimental - * - * @see #computeNorm - */ - public void setDiscountOverlaps(boolean v) { - discountOverlaps = v; - } - - /** @see #setDiscountOverlaps */ - public boolean getDiscountOverlaps() { - return discountOverlaps; - } - - /** - * Calls {@link #fillEasyStats(EasyStats, IndexSearcher, String, TermContext...)}. - * Subclasses that override this method may invoke {@code fillStats} with any - * subclass of {@code EasyStats}. - */ - @Override - public EasyStats computeStats(IndexSearcher searcher, String fieldName, - float queryBoost, TermContext... termContexts) throws IOException { - EasyStats stats = new EasyStats(queryBoost); - fillEasyStats(stats, searcher, fieldName, termContexts); - return stats; - } - - /** Fills all member fields defined in {@code EasyStats} in {@code stats}. */ - protected final void fillEasyStats(EasyStats stats, IndexSearcher searcher, - String fieldName, TermContext... termContexts) throws IOException { - IndexReader reader = searcher.getIndexReader(); - int numberOfDocuments = reader.maxDoc(); - long numberOfFieldTokens = MultiFields.getTerms(searcher.getIndexReader(), - fieldName).getSumTotalTermFreq(); - float avgFieldLength = (float)numberOfFieldTokens / numberOfDocuments; - - // nocommit Take the minimum of term frequencies for phrases. This is not - // correct though, we'll need something like a scorePhrase(MultiStats ...) - int docFreq = Integer.MAX_VALUE; - long totalTermFreq = Integer.MAX_VALUE; - for (final TermContext context : termContexts) { - docFreq = Math.min(docFreq, context.docFreq()); - totalTermFreq = Math.min(totalTermFreq, context.totalTermFreq()); - } - - // We have to provide something if codec doesnt supply these measures, - // or if someone omitted frequencies for the field... negative values cause - // NaN/Inf for some scorers. - if (numberOfFieldTokens == -1) { - numberOfFieldTokens = docFreq; - avgFieldLength = 1; - } - if (totalTermFreq == -1) { - totalTermFreq = docFreq; - } - - stats.setNumberOfDocuments(numberOfDocuments); - stats.setNumberOfFieldTokens(numberOfFieldTokens); - stats.setAvgFieldLength(avgFieldLength); - stats.setDocFreq(docFreq); - stats.setTotalTermFreq(totalTermFreq); - } - - /** - * Scores the document {@code doc}. - *

Subclasses must apply their scoring formula in this class.

- * @param stats the corpus level statistics. - * @param freq the term frequency. - * @param docLen the document length. - * @return the score. - */ - protected abstract float score(EasyStats stats, float freq, float docLen); - - /** - * Subclasses should implement this method to explain the score. {@code expl} - * already contains the score, the name of the class and the doc id, as well - * as the term frequency and its explanation; subclasses can add additional - * clauses to explain details of their scoring formulae. - *

The default implementation does nothing.

- * - * @param expl the explanation to extend with details. - * @param stats the corpus level statistics. - * @param doc the document id. - * @param freq the term frequency. - * @param docLen the document length. - */ - protected void explain( - Explanation expl, EasyStats stats, int doc, float freq, float docLen) {} - - /** - * Explains the score. The implementation here provides a basic explanation - * in the format score(name-of-similarity, doc=doc-id, - * freq=term-frequency), computed from:, and - * attaches the score (computed via the {@link #score(EasyStats, float, byte)} - * method) and the explanation for the term frequency. Subclasses content with - * this format may add additional details in - * {@link #explain(Explanation, EasyStats, int, float, byte)}. - * - * @param stats the corpus level statistics. - * @param doc the document id. - * @param freq the term frequency and its explanation. - * @param docLen the document length. - * @return the explanation. - */ - protected Explanation explain( - EasyStats stats, int doc, Explanation freq, float docLen) { - Explanation result = new Explanation(); - result.setValue(score(stats, freq.getValue(), docLen)); - result.setDescription("score(" + getClass().getSimpleName() + - ", doc=" + doc + ", freq=" + freq.getValue() +"), computed from:"); - result.addDetail(freq); - - explain(result, stats, doc, freq.getValue(), docLen); - - return result; - } - - @Override - public ExactDocScorer exactDocScorer(Stats stats, String fieldName, - AtomicReaderContext context) throws IOException { - return new EasyExactDocScorer((EasyStats) stats, - context.reader.norms(fieldName)); - } - - @Override - public SloppyDocScorer sloppyDocScorer(Stats stats, String fieldName, - AtomicReaderContext context) throws IOException { - return new EasySloppyDocScorer((EasyStats) stats, - context.reader.norms(fieldName)); - } - - /** - * Subclasses must override this method to return the name of the Similarity - * and preferably the values of parameters (if any) as well. - */ - @Override - public abstract String toString(); // nocommit: to Similarity? - - // ------------------------------ Norm handling ------------------------------ - - /** Norm -> document length map. */ - private static final float[] NORM_TABLE = new float[256]; - - static { - for (int i = 0; i < 256; i++) { - float floatNorm = SmallFloat.byte315ToFloat((byte)i); - NORM_TABLE[i] = 1.0f / (floatNorm * floatNorm); - } - } - - /** Encodes the document length in the same way as {@link TFIDFSimilarity}. */ - @Override - public byte computeNorm(FieldInvertState state) { - final float numTerms; - if (discountOverlaps) - numTerms = state.getLength() - state.getNumOverlap(); - else - numTerms = state.getLength() / state.getBoost(); - return encodeNormValue(numTerms); - } - - /** Decodes a normalization factor (document length) stored in an index. - * @see #encodeNormValue(float) - */ - protected float decodeNormValue(byte norm) { - return NORM_TABLE[norm & 0xFF]; // & 0xFF maps negative bytes to positive above 127 - } - - /** Encodes the length to a byte via SmallFloat. */ - protected byte encodeNormValue(float length) { - return SmallFloat.floatToByte315((float)(1.0 / Math.sqrt(length))); - } - - // ----------------------------- Static methods ------------------------------ - - /** Returns the base two logarithm of {@code x}. */ - public static double log2(double x) { - // Put this to a 'util' class if we need more of these. - return Math.log(x) / LOG_2; - } - - // --------------------------------- Classes --------------------------------- - - /** Delegates the {@link #score(int, int)} and - * {@link #explain(int, Explanation)} methods to - * {@link EasySimilarity#score(EasyStats, float, int)} and - * {@link EasySimilarity#explain(EasyStats, int, Explanation, int)}, - * respectively. - */ - private class EasyExactDocScorer extends ExactDocScorer { - private final EasyStats stats; - private final byte[] norms; - - EasyExactDocScorer(EasyStats stats, byte norms[]) { - this.stats = stats; - this.norms = norms; - } - - @Override - public float score(int doc, int freq) { - // We have to supply something in case norms are omitted - return EasySimilarity.this.score(stats, freq, - norms == null ? freq : decodeNormValue(norms[doc])); - } - - @Override - public Explanation explain(int doc, Explanation freq) { - return EasySimilarity.this.explain(stats, doc, freq, - norms == null ? freq.getValue() : decodeNormValue(norms[doc])); - } - } - - /** Delegates the {@link #score(int, int)} and - * {@link #explain(int, Explanation)} methods to - * {@link EasySimilarity#score(EasyStats, float, int)} and - * {@link EasySimilarity#explain(EasyStats, int, Explanation, int)}, - * respectively. - */ - private class EasySloppyDocScorer extends SloppyDocScorer { - private final EasyStats stats; - private final byte[] norms; - - EasySloppyDocScorer(EasyStats stats, byte norms[]) { - this.stats = stats; - this.norms = norms; - } - - @Override - public float score(int doc, float freq) { - // We have to supply something in case norms are omitted - return EasySimilarity.this.score(stats, freq, - norms == null ? freq : decodeNormValue(norms[doc])); - } - @Override - public Explanation explain(int doc, Explanation freq) { - return EasySimilarity.this.explain(stats, doc, freq, - norms == null ? freq.getValue() : decodeNormValue(norms[doc])); - } - - @Override - public float computeSlopFactor(int distance) { - return 1.0f / (distance + 1); - } - - @Override - public float computePayloadFactor(int doc, int start, int end, BytesRef payload) { - return 1f; - } - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java deleted file mode 100644 index 89b3263..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/EasyStats.java +++ /dev/null @@ -1,146 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.Terms; -import org.apache.lucene.search.Similarity; - -/** - * Stores all statistics commonly used ranking methods. - * @lucene.experimental - */ -public class EasyStats extends Similarity.Stats { - /** The number of documents. */ - protected int numberOfDocuments; - /** The total number of tokens in the field. */ - protected long numberOfFieldTokens; - /** The average field length. */ - protected float avgFieldLength; - /** The document frequency. */ - protected int docFreq; - /** The total number of occurrences of this term across all documents. */ - protected long totalTermFreq; - - // -------------------------- Boost-related stuff -------------------------- - - /** Query's inner boost. */ - protected final float queryBoost; - /** Any outer query's boost. */ - protected float topLevelBoost; - /** For most Similarities, the immediate and the top level query boosts are - * not handled differently. Hence, this field is just the product of the - * other two. */ - protected float totalBoost; - - /** Constructor. Sets the query boost. */ - public EasyStats(float queryBoost) { - this.queryBoost = queryBoost; - this.totalBoost = queryBoost; - } - - // ------------------------- Getter/setter methods ------------------------- - - /** Returns the number of documents. */ - public int getNumberOfDocuments() { - return numberOfDocuments; - } - - /** Sets the number of documents. */ - public void setNumberOfDocuments(int numberOfDocuments) { - this.numberOfDocuments = numberOfDocuments; - } - - /** - * Returns the total number of tokens in the field. - * @see Terms#getSumTotalTermFreq() - */ - public long getNumberOfFieldTokens() { - return numberOfFieldTokens; - } - - /** - * Sets the total number of tokens in the field. - * @see Terms#getSumTotalTermFreq() - */ - public void setNumberOfFieldTokens(long numberOfFieldTokens) { - this.numberOfFieldTokens = numberOfFieldTokens; - } - - /** Returns the average field length. */ - public float getAvgFieldLength() { - return avgFieldLength; - } - - /** Sets the average field length. */ - public void setAvgFieldLength(float avgFieldLength) { - this.avgFieldLength = avgFieldLength; - } - - /** Returns the document frequency. */ - public int getDocFreq() { - return docFreq; - } - - /** Sets the document frequency. */ - public void setDocFreq(int docFreq) { - this.docFreq = docFreq; - } - - /** Returns the total number of occurrences of this term across all documents. */ - public long getTotalTermFreq() { - return totalTermFreq; - } - - /** Sets the total number of occurrences of this term across all documents. */ - public void setTotalTermFreq(long totalTermFreq) { - this.totalTermFreq = totalTermFreq; - } - - // -------------------------- Boost-related stuff -------------------------- - - /** The square of the raw normalization value. - * @see #rawNormalizationValue() */ - @Override - public float getValueForNormalization() { - float rawValue = rawNormalizationValue(); - return rawValue * rawValue; - } - - /** Computes the raw normalization value. This basic implementation returns - * the query boost. Subclasses may override this method to include other - * factors (such as idf), or to save the value for inclusion in - * {@link #normalize(float, float)}, etc. - */ - protected float rawNormalizationValue() { - return queryBoost; - } - - /** No normalization is done. {@code topLevelBoost} is saved in the object, - * however. */ - @Override - public void normalize(float queryNorm, float topLevelBoost) { - this.topLevelBoost = topLevelBoost; - totalBoost = queryBoost * topLevelBoost; - } - - /** Returns the total boost. */ - public float getTotalBoost() { - return totalBoost; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java deleted file mode 100644 index 07d40e5..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/IBSimilarity.java +++ /dev/null @@ -1,99 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * Provides a framework for the family of information-based models, as described - * in Stéphane Clinchant and Eric Gaussier. 2010. Information-based - * models for ad hoc IR. In Proceeding of the 33rd international ACM SIGIR - * conference on Research and development in information retrieval (SIGIR '10). - * ACM, New York, NY, USA, 234-241. - *

The retrieval function is of the form RSV(q, d) = ∑ - * -xqw log Prob(Xw ≥ - * tdw | λw), where - *

    - *
  • xqw is the query boost;
    • - *
  • Xw is a random variable that counts the occurrences - * of word w;
    • - *
  • tdw is the normalized term frequency;
    • - *
  • λw is a parameter.
    • - *
- *

- *

The framework described in the paper has many similarities to the DFR - * framework (see {@link DFRSimilarity}). It is possible that the two - * Similarities will be merged at one point.

- * @lucene.experimental - */ -public class IBSimilarity extends EasySimilarity { - /** The probabilistic distribution used to model term occurrence. */ - protected final Distribution distribution; - /** The lambda (λw) parameter. */ - protected final Lambda lambda; - /** The term frequency normalization. */ - protected final Normalization normalization; - - public IBSimilarity(Distribution distribution, - Lambda lambda, - Normalization normalization) { - this.distribution = distribution; - this.lambda = lambda; - this.normalization = normalization; - } - - /** Creates an instance with no normalization. */ - public IBSimilarity(Distribution distribution, Lambda lambda) { - this(distribution, lambda, new Normalization.NoNormalization()); - } - - @Override - protected float score(EasyStats stats, float freq, float docLen) { - return stats.getTotalBoost() * - distribution.score( - stats, - normalization.tfn(stats, freq, docLen), - lambda.lambda(stats)); - } - - @Override - protected void explain( - Explanation expl, EasyStats stats, int doc, float freq, float docLen) { - if (stats.getTotalBoost() != 1.0f) { - expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); - } - Explanation normExpl = normalization.explain(stats, freq, docLen); - Explanation lambdaExpl = lambda.explain(stats); - expl.addDetail(normExpl); - expl.addDetail(lambdaExpl); - expl.addDetail(distribution.explain( - stats, normExpl.getValue(), lambdaExpl.getValue())); - } - - /** - * The name of IB methods follow the pattern - * {@code IB }. The name of the - * distribution is the same as in the original paper; for the names of lambda - * parameters, refer to the javadoc of the {@link Lambda} classes. - */ - @Override - public String toString() { - return "IB " + distribution.toString() + "-" + lambda.toString() - + normalization.toString(); - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java deleted file mode 100644 index 6c8f5f4..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/LMDirichletSimilarity.java +++ /dev/null @@ -1,97 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * Bayesian smoothing using Dirichlet priors. From Chengxiang Zhai and John - * Lafferty. 2001. A study of smoothing methods for language models applied to - * Ad Hoc information retrieval. In Proceedings of the 24th annual international - * ACM SIGIR conference on Research and development in information retrieval - * (SIGIR '01). ACM, New York, NY, USA, 334-342. - *

- * The formula as defined the paper assigns a negative score to documents that - * contain the term, but with fewer occurrences than predicted by the collection - * language model. The Lucene implementation returns {@code 0} for such - * documents. - *

- * - * @lucene.experimental - */ -public class LMDirichletSimilarity extends LMSimilarity { - /** The μ parameter. */ - private final float mu; - - /** @param mu the μ parameter. */ - public LMDirichletSimilarity(CollectionModel collectionModel, float mu) { - super(collectionModel); - this.mu = mu; - } - - /** @param mu the μ parameter. */ - public LMDirichletSimilarity(float mu) { - this.mu = mu; - } - - /** Instantiates the similarity with the default μ value of 2000. */ - public LMDirichletSimilarity(CollectionModel collectionModel) { - this(collectionModel, 2000); - } - - /** Instantiates the similarity with the default μ value of 2000. */ - public LMDirichletSimilarity() { - this(2000); - } - - @Override - protected float score(EasyStats stats, float freq, float docLen) { - float score = stats.getTotalBoost() * (float)(Math.log(1 + freq / - (mu * ((LMStats)stats).getCollectionProbability())) + - Math.log(mu / (docLen + mu))); - return score > 0.0f ? score : 0.0f; - } - - @Override - protected void explain(Explanation expl, EasyStats stats, int doc, - float freq, float docLen) { - if (stats.getTotalBoost() != 1.0f) { - expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); - } - - expl.addDetail(new Explanation(mu, "mu")); - Explanation weightExpl = new Explanation(); - weightExpl.setValue((float)Math.log(1 + freq / - (mu * ((LMStats)stats).getCollectionProbability()))); - weightExpl.setDescription("term weight"); - expl.addDetail(weightExpl); - expl.addDetail(new Explanation( - (float)Math.log(mu / (docLen + mu)), "document norm")); - super.explain(expl, stats, doc, freq, docLen); - } - - /** Returns the μ parameter. */ - public float getMu() { - return mu; - } - - @Override - public String getName() { - return String.format("Dirichlet(%f)", getMu()); - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java deleted file mode 100644 index 32ff804..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/LMJelinekMercerSimilarity.java +++ /dev/null @@ -1,78 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.similarities.LMSimilarity.LMStats; - -/** - * Language model based on the Jelinek-Mercer smoothing method. From Chengxiang - * Zhai and John Lafferty. 2001. A study of smoothing methods for language - * models applied to Ad Hoc information retrieval. In Proceedings of the 24th - * annual international ACM SIGIR conference on Research and development in - * information retrieval (SIGIR '01). ACM, New York, NY, USA, 334-342. - *

The model has a single parameter, λ. According to said paper, the - * optimal value depends on both the collection and the query. The optimal value - * is around {@code 0.1} for title queries and {@code 0.7} for long queries.

- * - * @lucene.experimental - */ -public class LMJelinekMercerSimilarity extends LMSimilarity { - /** The λ parameter. */ - private final float lambda; - - /** @param lambda the λ parameter. */ - public LMJelinekMercerSimilarity( - CollectionModel collectionModel, float lambda) { - super(collectionModel); - this.lambda = lambda; - } - - /** @param lambda the λ parameter. */ - public LMJelinekMercerSimilarity(float lambda) { - this.lambda = lambda; - } - - @Override - protected float score(EasyStats stats, float freq, float docLen) { - return stats.getTotalBoost() * - (float)Math.log(1 + - ((1 - lambda) * freq / docLen) / - (lambda * ((LMStats)stats).getCollectionProbability())); - } - - @Override - protected void explain(Explanation expl, EasyStats stats, int doc, - float freq, float docLen) { - if (stats.getTotalBoost() != 1.0f) { - expl.addDetail(new Explanation(stats.getTotalBoost(), "boost")); - } - expl.addDetail(new Explanation(lambda, "lambda")); - super.explain(expl, stats, doc, freq, docLen); - } - - /** Returns the λ parameter. */ - public float getLambda() { - return lambda; - } - - @Override - public String getName() { - return String.format("Jelinek-Mercer(%f)", getLambda()); - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java deleted file mode 100644 index 2aa2e04..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/LMSimilarity.java +++ /dev/null @@ -1,152 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.search.Explanation; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.util.TermContext; - -/** - * Abstract superclass for language modeling Similarities. The following inner - * types are introduced: - *
    - *
  • {@link LMStats}, which defines a new statistic, the probability that - * the collection language model generates the current term;
    • - *
  • {@link CollectionModel}, which is a strategy interface for object that - * compute the collection language model {@code p(w|C)};
    • - *
  • {@link DefaultCollectionModel}, an implementation of the former, that - * computes the term probability as the number of occurrences of the term in the - * collection, divided by the total number of tokens.
    • - *
- * - * @lucene.experimental - */ -public abstract class LMSimilarity extends EasySimilarity { - /** The collection model. */ - protected final CollectionModel collectionModel; - - /** Creates a new instance with the specified collection language model. */ - public LMSimilarity(CollectionModel collectionModel) { - this.collectionModel = collectionModel; - } - - /** Creates a new instance with the default collection language model. */ - public LMSimilarity() { - this(new DefaultCollectionModel()); - } - - /** - * Computes the collection probability of the current term in addition to the - * usual statistics. - */ - @Override - public EasyStats computeStats(IndexSearcher searcher, String fieldName, - float queryBoost, TermContext... termContexts) throws IOException { - LMStats stats = new LMStats(queryBoost); - fillEasyStats(stats, searcher, fieldName, termContexts); - stats.setCollectionProbability(collectionModel.computeProbability(stats)); - return stats; - } - - @Override - protected void explain(Explanation expl, EasyStats stats, int doc, - float freq, float docLen) { - expl.addDetail(new Explanation(collectionModel.computeProbability(stats), - "collection probability")); - } - - /** - * Returns the name of the LM method. The values of the parameters should be - * included as well. - *

Used in {@link #toString()}

. - */ - public abstract String getName(); - - /** - * Returns the name of the LM method. If a custom collection model strategy is - * used, its name is included as well. - * @see #getName() - * @see CollectionModel#getName() - * @see DefaultCollectionModel - */ - @Override - public String toString() { - String coll = collectionModel.getName(); - if (coll != null) { - return String.format("LM %s - %s", getName(), coll); - } else { - return String.format("LM %s", getName()); - } - } - - /** Stores the collection distribution of the current term. */ - public static class LMStats extends EasyStats { - /** The probability that the current term is generated by the collection. */ - private float collectionProbability; - - public LMStats(float queryBoost) { - super(queryBoost); - } - - /** - * Returns the probability that the current term is generated by the - * collection. - */ - public final float getCollectionProbability() { - return collectionProbability; - } - - /** - * Sets the probability that the current term is generated by the - * collection. - */ - public final void setCollectionProbability(float collectionProbability) { - this.collectionProbability = collectionProbability; - } - } - - /** A strategy for computing the collection language model. */ - public static interface CollectionModel { - /** - * Computes the probability {@code p(w|C)} according to the language model - * strategy for the current term. - */ - public float computeProbability(EasyStats stats); - - /** The name of the collection model strategy. */ - public String getName(); - } - - /** - * Models {@code p(w|C)} as the number of occurrences of the term in the - * collection, divided by the total number of tokens {@code + 1}. - */ - public static class DefaultCollectionModel implements CollectionModel { - @Override - public float computeProbability(EasyStats stats) { - return (float)stats.getTotalTermFreq() / (stats.getNumberOfFieldTokens() +1); - } - - @Override - public String getName() { - return null; - } - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/Lambda.java lucene/src/test/org/apache/lucene/search/similarities/Lambda.java deleted file mode 100644 index 5df1b4a..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/Lambda.java +++ /dev/null @@ -1,42 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * The lambda (λw) parameter in information-based - * models. - * @see IBSimilarity - * @lucene.experimental - */ -public abstract class Lambda { - /** Computes the lambda parameter. */ - public abstract float lambda(EasyStats stats); - /** Explains the lambda parameter. */ - public abstract Explanation explain(EasyStats stats); - - /** - * Subclasses must override this method to return the code of the lambda - * formula. Since the original paper is not very clear on this matter, and - * also uses the DFR naming scheme incorrectly, the codes here were chosen - * arbitrarily. - */ - @Override - public abstract String toString(); -} diff --git lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java deleted file mode 100644 index 4786ec6..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/LambdaDF.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * Computes lambda as {@code totalTermFreq / numberOfDocuments}. - * @lucene.experimental - */ -public class LambdaDF extends Lambda { - @Override - public final float lambda(EasyStats stats) { - return (float)stats.getDocFreq() / stats.getNumberOfDocuments(); - } - - @Override - public final Explanation explain(EasyStats stats) { - Explanation result = new Explanation(); - result.setDescription(getClass().getSimpleName() + ", computed from: "); - result.setValue(lambda(stats)); - result.addDetail( - new Explanation(stats.getDocFreq(), "docFreq")); - result.addDetail( - new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); - return result; - } - - @Override - public String toString() { - return "D"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java deleted file mode 100644 index f07f146..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/LambdaTTF.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * Computes lambda as {@code docFreq / numberOfDocuments}. - * @lucene.experimental - */ -public class LambdaTTF extends Lambda { - @Override - public final float lambda(EasyStats stats) { - return (float)stats.getTotalTermFreq() / stats.getNumberOfDocuments(); - } - - @Override - public final Explanation explain(EasyStats stats) { - Explanation result = new Explanation(); - result.setDescription(getClass().getSimpleName() + ", computed from: "); - result.setValue(lambda(stats)); - result.addDetail( - new Explanation(stats.getTotalTermFreq(), "totalTermFreq")); - result.addDetail( - new Explanation(stats.getNumberOfDocuments(), "numberOfDocuments")); - return result; - } - - @Override - public String toString() { - return "L"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/Normalization.java lucene/src/test/org/apache/lucene/search/similarities/Normalization.java deleted file mode 100644 index 6a2afea..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/Normalization.java +++ /dev/null @@ -1,75 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.search.Explanation; - -/** - * This class acts as the base class for the implementations of the term - * frequency normalization methods in the DFR framework. - * - * @see DFRSimilarity - * @lucene.experimental - */ -public abstract class Normalization { - /** Returns the normalized term frequency. - * @param len the field length. */ - public abstract float tfn(EasyStats stats, float tf, float len); - - /** Returns an explanation for the normalized term frequency. - *

The default normalization methods use the field length of the document - * and the average field length to compute the normalized term frequency. - * This method provides a generic explanation for such methods. - * Subclasses that use other statistics must override this method.

- */ - public Explanation explain(EasyStats stats, float tf, float len) { - Explanation result = new Explanation(); - result.setDescription(getClass().getSimpleName() + ", computed from: "); - result.setValue(tfn(stats, tf, len)); - result.addDetail(new Explanation(tf, "tf")); - result.addDetail( - new Explanation(stats.getAvgFieldLength(), "avgFieldLength")); - result.addDetail(new Explanation(len, "len")); - return result; - } - - /** Implementation used when there is no normalization. */ - public static final class NoNormalization extends Normalization { - @Override - public final float tfn(EasyStats stats, float tf, float len) { - return tf; - } - - @Override - public final Explanation explain(EasyStats stats, float tf, float len) { - return new Explanation(1, "no normalization"); - } - - @Override - public String toString() { - return ""; - } - } - - /** - * Subclasses must override this method to return the code of the - * normalization formula. Refer to the original paper for the list. - */ - @Override - public abstract String toString(); -} diff --git lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java deleted file mode 100644 index d289f61..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/NormalizationH1.java +++ /dev/null @@ -1,33 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Normalization model that assumes a uniform distribution of the term frequency. - */ -public class NormalizationH1 extends Normalization { - @Override - public final float tfn(EasyStats stats, float tf, float len) { - return tf * stats.getAvgFieldLength() / len; - } - - @Override - public String toString() { - return "1"; - } -} diff --git lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java deleted file mode 100644 index c49eaf4..0000000 --- lucene/src/test/org/apache/lucene/search/similarities/NormalizationH2.java +++ /dev/null @@ -1,36 +0,0 @@ -package org.apache.lucene.search.similarities; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.apache.lucene.search.similarities.EasySimilarity.log2; - -/** - * Normalization model in which the term frequency is inversely related to the - * length. - */ -public class NormalizationH2 extends Normalization { - @Override - public final float tfn(EasyStats stats, float tf, float len) { - return (float)(tf * log2(1 + stats.getAvgFieldLength() / len)); - } - - @Override - public String toString() { - return "2"; - } -} \ No newline at end of file