Matches in longer fields are less precise, so implementations of this
* method usually return smaller values when numTokens is large,
* and larger values when numTokens is small.
+ *
+ *
Additionally, in cases where some tokens appear at the same positions
+ * (i.e. their positionIncrement == 0) it may be useful if
+ * subclasses reduce the total number of tokens by the count of such
+ * overlapping tokens.
+ * This count is provided in numOverlappingTokens argument, and
+ * represents the number of tokens with positionIncrement == 0.
*
- *
That these values are computed under + *
Note that the return values are computed under * {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document)} - * and stored then using + * and then stored using * {@link #encodeNorm(float)}. * Thus they have limited precision, and documents * must be re-indexed if this method is altered. + * + *
The default implementation of this method simply calls
+ * {@link #lengthNorm(String, int)}, i.e. it ignores the last argument.
*
* @param fieldName the name of the field
* @param numTokens the total number of tokens contained in fields named
* fieldName of doc.
+ * @param numOverlapTokens the number of terms with
+ * positionIncrement == 0
* @return a normalization factor for hits on this field of this document
*
* @see org.apache.lucene.document.Field#setBoost(float)
+ * @see org.apache.lucene.analysis.Token#getPositionIncrement()
*/
+ public float lengthNorm(String fieldName, int numTokens,
+ int numOverlapTokens) {
+ return lengthNorm(fieldName, numTokens);
+ }
+
+ /** Computes the normalization value for a field given the total number of
+ * terms contained in a field.
+ * @param fieldName the name of the field
+ * @param numTokens the total number of tokens contained in fields named
+ * fieldName of doc.
+ * @return a normalization factor for hits on this field of this document
+ *
+ * @see org.apache.lucene.document.Field#setBoost(float)
+ * @see #lengthNorm(String, int, int)
+ */
public abstract float lengthNorm(String fieldName, int numTokens);
/** Computes the normalization value for a query given the sum of the squared
Index: src/java/org/apache/lucene/search/SimilarityDelegator.java
===================================================================
--- src/java/org/apache/lucene/search/SimilarityDelegator.java (revision 704002)
+++ src/java/org/apache/lucene/search/SimilarityDelegator.java (working copy)
@@ -32,6 +32,10 @@
this.delegee = delegee;
}
+ public float lengthNorm(String fieldName, int numTerms, int numOverlapTerms) {
+ return delegee.lengthNorm(fieldName, numTerms, numOverlapTerms);
+ }
+
public float lengthNorm(String fieldName, int numTerms) {
return delegee.lengthNorm(fieldName, numTerms);
}
Index: src/java/org/apache/lucene/index/DocInverterPerField.java
===================================================================
--- src/java/org/apache/lucene/index/DocInverterPerField.java (revision 704002)
+++ src/java/org/apache/lucene/index/DocInverterPerField.java (working copy)
@@ -135,6 +135,9 @@
if (token == null) break;
fieldState.position += (token.getPositionIncrement() - 1);
+ if (token.getPositionIncrement() == 0) {
+ fieldState.numOverlap++;
+ }
boolean success = false;
try {
// If we hit an exception in here, we abort
Index: src/java/org/apache/lucene/index/NormsWriterPerField.java
===================================================================
--- src/java/org/apache/lucene/index/NormsWriterPerField.java (revision 704002)
+++ src/java/org/apache/lucene/index/NormsWriterPerField.java (working copy)
@@ -68,7 +68,7 @@
docIDs = ArrayUtil.grow(docIDs, 1+upto);
norms = ArrayUtil.grow(norms, 1+upto);
}
- final float norm = fieldState.boost * docState.similarity.lengthNorm(fieldInfo.name, fieldState.length);
+ final float norm = fieldState.boost * docState.similarity.lengthNorm(fieldInfo.name, fieldState.length, fieldState.numOverlap);
norms[upto] = Similarity.encodeNorm(norm);
docIDs[upto] = docState.docID;
upto++;
Index: src/java/org/apache/lucene/index/IndexReader.java
===================================================================
--- src/java/org/apache/lucene/index/IndexReader.java (revision 704002)
+++ src/java/org/apache/lucene/index/IndexReader.java (working copy)
@@ -694,7 +694,7 @@
/** Expert: Resets the normalization factor for the named field of the named
* document. The norm represents the product of the field's {@link
* org.apache.lucene.document.Fieldable#setBoost(float) boost} and its {@link Similarity#lengthNorm(String,
- * int) length normalization}. Thus, to preserve the length normalization
+ * int, int) length normalization}. Thus, to preserve the length normalization
* values when resetting this, one should base the new value upon the old.
*
* @see #norms(String)
Index: src/java/org/apache/lucene/index/DocInverter.java
===================================================================
--- src/java/org/apache/lucene/index/DocInverter.java (revision 704002)
+++ src/java/org/apache/lucene/index/DocInverter.java (working copy)
@@ -96,12 +96,14 @@
final static class FieldInvertState {
int position;
int length;
+ int numOverlap;
int offset;
float boost;
void reset(float docBoost) {
position = 0;
length = 0;
+ numOverlap = 0;
offset = 0;
boost = docBoost;
}
Index: src/java/org/apache/lucene/document/AbstractField.java
===================================================================
--- src/java/org/apache/lucene/document/AbstractField.java (revision 704002)
+++ src/java/org/apache/lucene/document/AbstractField.java (working copy)
@@ -98,13 +98,13 @@
*
The boost is multiplied by {@link org.apache.lucene.document.Document#getBoost()} of the document * containing this field. If a document has multiple fields with the same * name, all such values are multiplied together. This product is then - * multipled by the value {@link org.apache.lucene.search.Similarity#lengthNorm(String,int)}, and + * multipled by the value {@link org.apache.lucene.search.Similarity#lengthNorm(String,int, int)}, and * rounded by {@link org.apache.lucene.search.Similarity#encodeNorm(float)} before it is stored in the * index. One should attempt to ensure that this product does not overflow * the range of that encoding. * * @see org.apache.lucene.document.Document#setBoost(float) - * @see org.apache.lucene.search.Similarity#lengthNorm(String, int) + * @see org.apache.lucene.search.Similarity#lengthNorm(String, int, int) * @see org.apache.lucene.search.Similarity#encodeNorm(float) */ public void setBoost(float boost) { Index: src/java/org/apache/lucene/document/Fieldable.java =================================================================== --- src/java/org/apache/lucene/document/Fieldable.java (revision 704002) +++ src/java/org/apache/lucene/document/Fieldable.java (working copy) @@ -39,13 +39,13 @@ *
The boost is multiplied by {@link org.apache.lucene.document.Document#getBoost()} of the document
* containing this field. If a document has multiple fields with the same
* name, all such values are multiplied together. This product is then
- * multipled by the value {@link org.apache.lucene.search.Similarity#lengthNorm(String,int)}, and
+ * multipled by the value {@link org.apache.lucene.search.Similarity#lengthNorm(String,int, int)}, and
* rounded by {@link org.apache.lucene.search.Similarity#encodeNorm(float)} before it is stored in the
* index. One should attempt to ensure that this product does not overflow
* the range of that encoding.
*
* @see org.apache.lucene.document.Document#setBoost(float)
- * @see org.apache.lucene.search.Similarity#lengthNorm(String, int)
+ * @see org.apache.lucene.search.Similarity#lengthNorm(String, int, int)
* @see org.apache.lucene.search.Similarity#encodeNorm(float)
*/
void setBoost(float boost);
Index: contrib/miscellaneous/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
===================================================================
--- contrib/miscellaneous/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (revision 704002)
+++ contrib/miscellaneous/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (working copy)
@@ -84,8 +84,8 @@
// seperate sweet spot for certain fields
- ss.setLengthNormFactors("bar",8,13, 0.5f);
- ss.setLengthNormFactors("yak",6,9, 0.5f);
+ ss.setLengthNormFactors("bar",8,13, 0.5f, false);
+ ss.setLengthNormFactors("yak",6,9, 0.5f, false);
for (int i = 3; i <=10; i++) {
@@ -122,8 +122,8 @@
// steepness
- ss.setLengthNormFactors("a",5,8,0.5f);
- ss.setLengthNormFactors("b",5,8,0.1f);
+ ss.setLengthNormFactors("a",5,8,0.5f, false);
+ ss.setLengthNormFactors("b",5,8,0.1f, false);
for (int i = 9; i < 1000; i++) {
assertTrue("s: i="+i+" : a="+ss.lengthNorm("a",i)+
Index: contrib/miscellaneous/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
===================================================================
--- contrib/miscellaneous/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java (revision 704002)
+++ contrib/miscellaneous/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java (working copy)
@@ -49,10 +49,12 @@
private int ln_min = 1;
private int ln_max = 1;
private float ln_steep = 0.5f;
+ private boolean discountOverlaps = false;
private Map ln_mins = new HashMap(7);
private Map ln_maxs = new HashMap(7);
private Map ln_steeps = new HashMap(7);
+ private Map ln_overlaps = new HashMap(7);
private float tf_base = 0.0f;
private float tf_min = 0.0f;
@@ -106,15 +108,25 @@
}
/**
- * Sets the function variables used by lengthNorm for a specific named field
+ * Sets the function variables used by lengthNorm for a specific named field.
+ *
+ * @param field field name
+ * @param min minimum value
+ * @param max maximum value
+ * @param steepness steepness of the curve
+ * @param discountOverlaps if true, numOverlapTokens will be
+ * subtracted from numTokens; if false then
+ * numOverlapTokens will be assumed to be 0 (see
+ * {@link Similarity#lengthNorm(String, int, int)} for details).
*
* @see #lengthNorm
*/
public void setLengthNormFactors(String field, int min, int max,
- float steepness) {
+ float steepness, boolean discountOverlapping) {
ln_mins.put(field, new Integer(min));
ln_maxs.put(field, new Integer(max));
ln_steeps.put(field, new Float(steepness));
+ ln_overlaps.put(field, new Boolean(discountOverlapping));
}
/**
@@ -135,11 +147,12 @@
*
* @see #setLengthNormFactors
*/
- public float lengthNorm(String fieldName, int numTerms) {
+ public float lengthNorm(String fieldName, int numTerms, int numOverlaps) {
int l = ln_min;
int h = ln_max;
float s = ln_steep;
-
+ boolean overlaps = discountOverlaps;
+
if (ln_mins.containsKey(fieldName)) {
l = ((Number)ln_mins.get(fieldName)).intValue();
}
@@ -150,6 +163,14 @@
s = ((Number)ln_steeps.get(fieldName)).floatValue();
}
+ if (ln_overlaps.containsKey(fieldName)) {
+ overlaps = ((Boolean)ln_overlaps.get(fieldName)).booleanValue();
+ }
+
+ if (overlaps) {
+ numTerms -= numOverlaps;
+ }
+
return (float)
(1.0f /
Math.sqrt
@@ -162,6 +183,10 @@
)
);
}
+
+ public float lengthNorm(String fieldName, int numTokens) {
+ return this.lengthNorm(fieldName, numTokens, 0);
+ }
/**
* Delegates to baselineTf
Index: contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
===================================================================
--- contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 704002)
+++ contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy)
@@ -348,6 +348,7 @@
HashMap terms = new HashMap();
int numTokens = 0;
+ int numOverlapTokens = 0;
int pos = -1;
final Token reusableToken = new Token();
for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
@@ -355,6 +356,9 @@
if (term.length() == 0) continue; // nothing to do
// if (DEBUG) System.err.println("token='" + term + "'");
numTokens++;
+ if (nextToken.getPositionIncrement() == 0) {
+ numOverlapTokens++;
+ }
pos += nextToken.getPositionIncrement();
ArrayIntList positions = (ArrayIntList) terms.get(term);
@@ -372,7 +376,7 @@
// ensure infos.numTokens > 0 invariant; needed for correct operation of terms()
if (numTokens > 0) {
boost = boost * docBoost; // see DocumentWriter.addDocument(...)
- fields.put(fieldName, new Info(terms, numTokens, boost));
+ fields.put(fieldName, new Info(terms, numTokens, numOverlapTokens, boost));
sortedFields = null; // invalidate sorted view, if any
}
} catch (IOException e) { // can never happen
@@ -574,6 +578,9 @@
/** Number of added tokens for this field */
private final int numTokens;
+ /** Number of overlapping tokens for this field */
+ private final int numOverlapTokens;
+
/** Boost factor for hits for this field */
private final float boost;
@@ -582,9 +589,10 @@
private static final long serialVersionUID = 2882195016849084649L;
- public Info(HashMap terms, int numTokens, float boost) {
+ public Info(HashMap terms, int numTokens, int numOverlapTokens, float boost) {
this.terms = terms;
this.numTokens = numTokens;
+ this.numOverlapTokens = numOverlapTokens;
this.boost = boost;
}
@@ -1067,7 +1075,8 @@
if (fieldName != cachedFieldName || sim != cachedSimilarity) { // not cached?
Info info = getInfo(fieldName);
int numTokens = info != null ? info.numTokens : 0;
- float n = sim.lengthNorm(fieldName, numTokens);
+ int numOverlapTokens = info != null ? info.numOverlapTokens : 0;
+ float n = sim.lengthNorm(fieldName, numTokens, numOverlapTokens);
float boost = info != null ? info.getBoost() : 1.0f;
n = n * boost; // see DocumentWriter.writeNorms(String segment)
byte norm = Similarity.encodeNorm(n);