Implementations should calculate a float value based on the field
+ * state, and then modify the state input parameter by calling
+ * {@link FieldInvertState#setNorm(float)}.
+ *
+ *
For backward compatibility this method calls + * {@link #lengthNorm(String, int)} passing + * {@link FieldInvertState#getLength()} as the second argument, and + * then multiplies this value by {@link FieldInvertState#getBoost()}.
+ * + *WARNING: This API is new and experimental and may + * suddenly change.
+ * + * @param field field name + * @param state current processing state for this field + */ + public void computeNorm(String field, FieldInvertState state) { + float norm = state.getBoost() * lengthNorm(field, state.getLength()); + state.setNorm(norm); + } + /** Computes the normalization value for a field given the total number of * terms contained in a field. These values, together with field boosts, are * stored in an index and multipled into scores for hits on each field by the @@ -341,14 +366,14 @@ *Matches in longer fields are less precise, so implementations of this
* method usually return smaller values when numTokens is large,
* and larger values when numTokens is small.
- *
- *
That these values are computed under + * + *
Note that the return values are computed under * {@link org.apache.lucene.index.IndexWriter#addDocument(org.apache.lucene.document.Document)} - * and stored then using + * and then stored using * {@link #encodeNorm(float)}. * Thus they have limited precision, and documents * must be re-indexed if this method is altered. - * + * * @param fieldName the name of the field * @param numTokens the total number of tokens contained in fields named * fieldName of doc. Index: src/java/org/apache/lucene/search/SimilarityDelegator.java =================================================================== --- src/java/org/apache/lucene/search/SimilarityDelegator.java (revision 706658) +++ src/java/org/apache/lucene/search/SimilarityDelegator.java (working copy) @@ -1,5 +1,7 @@ package org.apache.lucene.search; +import org.apache.lucene.index.FieldInvertState; + /** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with @@ -32,6 +34,10 @@ this.delegee = delegee; } + public void computeNorm(String fieldName, FieldInvertState state) { + delegee.computeNorm(fieldName, state); + } + public float lengthNorm(String fieldName, int numTerms) { return delegee.lengthNorm(fieldName, numTerms); } Index: src/java/org/apache/lucene/index/DocInverterPerField.java =================================================================== --- src/java/org/apache/lucene/index/DocInverterPerField.java (revision 706658) +++ src/java/org/apache/lucene/index/DocInverterPerField.java (working copy) @@ -39,7 +39,7 @@ final InvertedDocConsumerPerField consumer; final InvertedDocEndConsumerPerField endConsumer; final DocumentsWriter.DocState docState; - final DocInverter.FieldInvertState fieldState; + final FieldInvertState fieldState; public DocInverterPerField(DocInverterPerThread perThread, FieldInfo fieldInfo) { this.perThread = perThread; @@ -135,6 +135,9 @@ if (token == null) break; fieldState.position += (token.getPositionIncrement() - 1); + if (token.getPositionIncrement() == 0) { + fieldState.numOverlap++; + } boolean success = false; try { // If we hit an exception in here, we abort Index: src/java/org/apache/lucene/index/NormsWriterPerField.java =================================================================== --- src/java/org/apache/lucene/index/NormsWriterPerField.java (revision 706658) +++ src/java/org/apache/lucene/index/NormsWriterPerField.java (working copy) @@ -36,7 +36,7 @@ byte[] norms = new byte[1]; int upto; - final DocInverter.FieldInvertState fieldState; + final FieldInvertState fieldState; public void reset() { // Shrink back if we are overallocated now: @@ -68,8 +68,8 @@ docIDs = ArrayUtil.grow(docIDs, 1+upto); norms = ArrayUtil.grow(norms, 1+upto); } - final float norm = fieldState.boost * docState.similarity.lengthNorm(fieldInfo.name, fieldState.length); - norms[upto] = Similarity.encodeNorm(norm); + docState.similarity.computeNorm(fieldInfo.name, fieldState); + norms[upto] = Similarity.encodeNorm(fieldState.norm); docIDs[upto] = docState.docID; upto++; } Index: src/java/org/apache/lucene/index/TermsHashPerField.java =================================================================== --- src/java/org/apache/lucene/index/TermsHashPerField.java (revision 706658) +++ src/java/org/apache/lucene/index/TermsHashPerField.java (working copy) @@ -30,7 +30,7 @@ final TermsHashPerField nextPerField; final TermsHashPerThread perThread; final DocumentsWriter.DocState docState; - final DocInverter.FieldInvertState fieldState; + final FieldInvertState fieldState; // Copied from our perThread final CharBlockPool charPool; Index: src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java =================================================================== --- src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (revision 706658) +++ src/java/org/apache/lucene/index/FreqProxTermsWriterPerField.java (working copy) @@ -30,7 +30,7 @@ final TermsHashPerField termsHashPerField; final FieldInfo fieldInfo; final DocumentsWriter.DocState docState; - final DocInverter.FieldInvertState fieldState; + final FieldInvertState fieldState; boolean omitTf; public FreqProxTermsWriterPerField(TermsHashPerField termsHashPerField, FreqProxTermsWriterPerThread perThread, FieldInfo fieldInfo) { Index: src/java/org/apache/lucene/index/FieldInvertState.java =================================================================== --- src/java/org/apache/lucene/index/FieldInvertState.java (revision 0) +++ src/java/org/apache/lucene/index/FieldInvertState.java (revision 0) @@ -0,0 +1,106 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import org.apache.lucene.search.Similarity; + +/** + * This class tracks the number and position / offset parameters of terms + * being added to the index. The information collected in this class is + * also used to calculate the normalization factor for a field. + * + *
WARNING: This API is new and experimental, and may suddenly + * change.
+ */ +public final class FieldInvertState { + int position; + int length; + int numOverlap; + int offset; + float boost; + float norm; + + /** + * Re-initialize the state, using this boost value. + * @param docBoost boost value to use. + */ + void reset(float docBoost) { + position = 0; + length = 0; + numOverlap = 0; + offset = 0; + boost = docBoost; + } + + /** + * Get the last processed term position. + * @return the position + */ + public int getPosition() { + return position; + } + + /** + * Get total number of terms in this field. + * @return the length + */ + public int getLength() { + return length; + } + + /** + * Get the number of terms withpositionIncrement == 0.
+ * @return the numOverlap
+ */
+ public int getNumOverlap() {
+ return numOverlap;
+ }
+
+ /**
+ * Get end offset of the last processed term.
+ * @return the offset
+ */
+ public int getOffset() {
+ return offset;
+ }
+
+ /**
+ * Get boost value. This is usually a product of document boost and
+ * field boost.
+ * @return the boost
+ */
+ public float getBoost() {
+ return boost;
+ }
+
+ /**
+ * Get norm value calculated by
+ * {@link Similarity#computeNorm(String, FieldInvertState)}.
+ * @return the norm, or 0.0f if none was calculated yet.
+ */
+ public float getNorm() {
+ return norm;
+ }
+
+ /**
+ * Set norm value.
+ * @param norm the norm to set
+ */
+ public void setNorm(float norm) {
+ this.norm = norm;
+ }
+}
\ No newline at end of file
Property changes on: src/java/org/apache/lucene/index/FieldInvertState.java
___________________________________________________________________
Name: svn:eol-style
+ native
Index: src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java
===================================================================
--- src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java (revision 706658)
+++ src/java/org/apache/lucene/index/TermVectorsTermsWriterPerField.java (working copy)
@@ -30,7 +30,7 @@
final TermVectorsTermsWriter termsWriter;
final FieldInfo fieldInfo;
final DocumentsWriter.DocState docState;
- final DocInverter.FieldInvertState fieldState;
+ final FieldInvertState fieldState;
boolean doVectors;
boolean doVectorPositions;
Index: src/java/org/apache/lucene/index/DocInverterPerThread.java
===================================================================
--- src/java/org/apache/lucene/index/DocInverterPerThread.java (revision 706658)
+++ src/java/org/apache/lucene/index/DocInverterPerThread.java (working copy)
@@ -32,7 +32,7 @@
final Token localToken = new Token();
final DocumentsWriter.DocState docState;
- final DocInverter.FieldInvertState fieldState = new DocInverter.FieldInvertState();
+ final FieldInvertState fieldState = new FieldInvertState();
// Used to read a string value for a field
final ReusableStringReader stringReader = new ReusableStringReader();
Index: src/java/org/apache/lucene/index/DocInverter.java
===================================================================
--- src/java/org/apache/lucene/index/DocInverter.java (revision 706658)
+++ src/java/org/apache/lucene/index/DocInverter.java (working copy)
@@ -92,18 +92,4 @@
public DocFieldConsumerPerThread addThread(DocFieldProcessorPerThread docFieldProcessorPerThread) {
return new DocInverterPerThread(docFieldProcessorPerThread, this);
}
-
- final static class FieldInvertState {
- int position;
- int length;
- int offset;
- float boost;
-
- void reset(float docBoost) {
- position = 0;
- length = 0;
- offset = 0;
- boost = docBoost;
- }
- }
}
Index: src/java/org/apache/lucene/document/AbstractField.java
===================================================================
--- src/java/org/apache/lucene/document/AbstractField.java (revision 706658)
+++ src/java/org/apache/lucene/document/AbstractField.java (working copy)
@@ -98,13 +98,14 @@
* The boost is multiplied by {@link org.apache.lucene.document.Document#getBoost()} of the document * containing this field. If a document has multiple fields with the same * name, all such values are multiplied together. This product is then - * multipled by the value {@link org.apache.lucene.search.Similarity#lengthNorm(String,int)}, and + * multipled by the value calculated in + * {@link org.apache.lucene.search.Similarity#computeNorm(String, org.apache.lucene.index.FieldInvertState)}, and * rounded by {@link org.apache.lucene.search.Similarity#encodeNorm(float)} before it is stored in the * index. One should attempt to ensure that this product does not overflow * the range of that encoding. * * @see org.apache.lucene.document.Document#setBoost(float) - * @see org.apache.lucene.search.Similarity#lengthNorm(String, int) + * @see org.apache.lucene.search.Similarity#computeNorm(String, org.apache.lucene.index.FieldInvertState) * @see org.apache.lucene.search.Similarity#encodeNorm(float) */ public void setBoost(float boost) { Index: src/java/org/apache/lucene/document/Fieldable.java =================================================================== --- src/java/org/apache/lucene/document/Fieldable.java (revision 706658) +++ src/java/org/apache/lucene/document/Fieldable.java (working copy) @@ -17,6 +17,7 @@ */ import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.index.FieldInvertState; import java.io.Reader; import java.io.Serializable; @@ -39,13 +40,14 @@ *
The boost is multiplied by {@link org.apache.lucene.document.Document#getBoost()} of the document * containing this field. If a document has multiple fields with the same * name, all such values are multiplied together. This product is then - * multipled by the value {@link org.apache.lucene.search.Similarity#lengthNorm(String,int)}, and + * multipled by the {@link FieldInvertState#getNorm()} value calculated in + * {@link org.apache.lucene.search.Similarity#computeNorm(String, FieldInvertState)}, and * rounded by {@link org.apache.lucene.search.Similarity#encodeNorm(float)} before it is stored in the * index. One should attempt to ensure that this product does not overflow * the range of that encoding. * * @see org.apache.lucene.document.Document#setBoost(float) - * @see org.apache.lucene.search.Similarity#lengthNorm(String, int) + * @see org.apache.lucene.search.Similarity#computeNorm(String, FieldInvertState) * @see org.apache.lucene.search.Similarity#encodeNorm(float) */ void setBoost(float boost);