Index: CHANGES.txt =================================================================== --- CHANGES.txt (revision 755652) +++ CHANGES.txt (working copy) @@ -59,6 +59,9 @@ 9. LUCENE-1186: Add Analyzer.close() to free internal ThreadLocal resources. (Christian Kohlschütter via Mike McCandless) +10. LUCENE-1561: Renamed Field.omitTf to Field.omitTermFreqAndPosition + (Otis Gospodnetic via Mike McCandless) + Bug fixes 1. LUCENE-1415: MultiPhraseQuery has incorrect hashCode() and equals() Index: src/test/org/apache/lucene/index/TestOmitTf.java =================================================================== --- src/test/org/apache/lucene/index/TestOmitTf.java (revision 755652) +++ src/test/org/apache/lucene/index/TestOmitTf.java (working copy) @@ -52,8 +52,8 @@ // Tests whether the DocumentWriter correctly enable the - // omitTf bit in the FieldInfo - public void testOmitTf() throws Exception { + // omitTermFreqAndPositions bit in the FieldInfo + public void testOmitTermFreqAndPositions() throws Exception { Directory ram = new MockRAMDirectory(); Analyzer analyzer = new StandardAnalyzer(); IndexWriter writer = new IndexWriter(ram, analyzer, true, IndexWriter.MaxFieldLength.LIMITED); @@ -65,7 +65,7 @@ // this field will NOT have Tf Field f2 = new Field("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED); - f2.setOmitTf(true); + f2.setOmitTermFreqAndPositions(true); d.add(f2); writer.addDocument(d); @@ -75,10 +75,10 @@ d = new Document(); // Reverese - f1.setOmitTf(true); + f1.setOmitTermFreqAndPositions(true); d.add(f1); - f2.setOmitTf(false); + f2.setOmitTermFreqAndPositions(false); d.add(f2); writer.addDocument(d); @@ -91,15 +91,15 @@ // only one segment in the index, so we can cast to SegmentReader SegmentReader reader = (SegmentReader) IndexReader.open(ram); FieldInfos fi = reader.fieldInfos(); - assertTrue("OmitTf field bit should be set.", fi.fieldInfo("f1").omitTf); - assertTrue("OmitTf field bit should be set.", fi.fieldInfo("f2").omitTf); + assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f1").omitTermFreqAndPositions); + assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f2").omitTermFreqAndPositions); reader.close(); ram.close(); } // Tests whether merging of docs that have different - // omitTf for the same field works + // omitTermFreqAndPositions for the same field works public void testMixedMerge() throws Exception { Directory ram = new MockRAMDirectory(); Analyzer analyzer = new StandardAnalyzer(); @@ -114,7 +114,7 @@ // this field will NOT have Tf Field f2 = new Field("f2", "This field has NO Tf in all docs", Field.Store.NO, Field.Index.ANALYZED); - f2.setOmitTf(true); + f2.setOmitTermFreqAndPositions(true); d.add(f2); for(int i=0;i<30;i++) @@ -125,10 +125,10 @@ d = new Document(); // Reverese - f1.setOmitTf(true); + f1.setOmitTermFreqAndPositions(true); d.add(f1); - f2.setOmitTf(false); + f2.setOmitTermFreqAndPositions(false); d.add(f2); for(int i=0;i<30;i++) @@ -144,15 +144,15 @@ // only one segment in the index, so we can cast to SegmentReader SegmentReader reader = (SegmentReader) IndexReader.open(ram); FieldInfos fi = reader.fieldInfos(); - assertTrue("OmitTf field bit should be set.", fi.fieldInfo("f1").omitTf); - assertTrue("OmitTf field bit should be set.", fi.fieldInfo("f2").omitTf); + assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f1").omitTermFreqAndPositions); + assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f2").omitTermFreqAndPositions); reader.close(); ram.close(); } - // Make sure first adding docs that do not omitTf for - // field X, then adding docs that do omitTf for that same + // Make sure first adding docs that do not omitTermFreqAndPositions for + // field X, then adding docs that do omitTermFreqAndPositions for that same // field, public void testMixedRAM() throws Exception { Directory ram = new MockRAMDirectory(); @@ -173,7 +173,7 @@ for(int i=0;i<5;i++) writer.addDocument(d); - f2.setOmitTf(true); + f2.setOmitTermFreqAndPositions(true); for(int i=0;i<20;i++) writer.addDocument(d); @@ -189,8 +189,8 @@ // only one segment in the index, so we can cast to SegmentReader SegmentReader reader = (SegmentReader) IndexReader.open(ram); FieldInfos fi = reader.fieldInfos(); - assertTrue("OmitTf field bit should not be set.", !fi.fieldInfo("f1").omitTf); - assertTrue("OmitTf field bit should be set.", fi.fieldInfo("f2").omitTf); + assertTrue("OmitTermFreqAndPositions field bit should not be set.", !fi.fieldInfo("f1").omitTermFreqAndPositions); + assertTrue("OmitTermFreqAndPositions field bit should be set.", fi.fieldInfo("f2").omitTermFreqAndPositions); reader.close(); ram.close(); @@ -213,7 +213,7 @@ Document d = new Document(); Field f1 = new Field("f1", "This field has term freqs", Field.Store.NO, Field.Index.ANALYZED); - f1.setOmitTf(true); + f1.setOmitTermFreqAndPositions(true); d.add(f1); for(int i=0;i<30;i++) @@ -250,7 +250,7 @@ sb.append(term).append(" "); String content = sb.toString(); Field noTf = new Field("noTf", content + (i%2==0 ? "" : " notf"), Field.Store.NO, Field.Index.ANALYZED); - noTf.setOmitTf(true); + noTf.setOmitTermFreqAndPositions(true); d.add(noTf); Field tf = new Field("tf", content + (i%2==0 ? " tf" : ""), Field.Store.NO, Field.Index.ANALYZED); Index: src/java/org/apache/lucene/index/FieldInfos.java =================================================================== --- src/java/org/apache/lucene/index/FieldInfos.java (revision 755652) +++ src/java/org/apache/lucene/index/FieldInfos.java (working copy) @@ -40,7 +40,7 @@ static final byte STORE_OFFSET_WITH_TERMVECTOR = 0x8; static final byte OMIT_NORMS = 0x10; static final byte STORE_PAYLOADS = 0x20; - static final byte OMIT_TF = 0x40; + static final byte OMIT_TERM_FREQ_AND_POSITIONS = 0x40; private ArrayList byNumber = new ArrayList(); private HashMap byName = new HashMap(); @@ -88,11 +88,11 @@ } } - /** Returns true if any fields do not omitTf */ + /** Returns true if any fields do not omitTermFreqAndPositions */ boolean hasProx() { final int numFields = byNumber.size(); for(int i=0;i>> 1; Index: src/java/org/apache/lucene/index/SegmentMerger.java =================================================================== --- src/java/org/apache/lucene/index/SegmentMerger.java (revision 755652) +++ src/java/org/apache/lucene/index/SegmentMerger.java (working copy) @@ -208,11 +208,11 @@ } private void addIndexed(IndexReader reader, FieldInfos fieldInfos, Collection names, boolean storeTermVectors, boolean storePositionWithTermVector, - boolean storeOffsetWithTermVector, boolean storePayloads, boolean omitTf) throws IOException { + boolean storeOffsetWithTermVector, boolean storePayloads, boolean omitTermFreqAndPositions) throws IOException { Iterator i = names.iterator(); while (i.hasNext()) { String field = (String)i.next(); - fieldInfos.add(field, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector, !reader.hasNorms(field), storePayloads, omitTf); + fieldInfos.add(field, true, storeTermVectors, storePositionWithTermVector, storeOffsetWithTermVector, !reader.hasNorms(field), storePayloads, omitTermFreqAndPositions); } } @@ -275,7 +275,7 @@ SegmentReader segmentReader = (SegmentReader) reader; for (int j = 0; j < segmentReader.getFieldInfos().size(); j++) { FieldInfo fi = segmentReader.getFieldInfos().fieldInfo(j); - fieldInfos.add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.hasNorms(fi.name), fi.storePayloads, fi.omitTf); + fieldInfos.add(fi.name, fi.isIndexed, fi.storeTermVector, fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, !reader.hasNorms(fi.name), fi.storePayloads, fi.omitTermFreqAndPositions); } } else { addIndexed(reader, fieldInfos, reader.getFieldNames(IndexReader.FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, false); @@ -496,7 +496,7 @@ } } - boolean omitTF; + boolean omitTermFreqAndPositions; private final void mergeTermInfos(final FormatPostingsFieldsConsumer consumer) throws CorruptIndexException, IOException { int base = 0; @@ -544,7 +544,7 @@ termsConsumer.finish(); final FieldInfo fieldInfo = fieldInfos.fieldInfo(currentField); termsConsumer = consumer.addField(fieldInfo); - omitTF = fieldInfo.omitTf; + omitTermFreqAndPositions = fieldInfo.omitTermFreqAndPositions; } int df = appendPostings(termsConsumer, match, matchSize); // add new TermInfo @@ -605,7 +605,7 @@ final int freq = postings.freq(); final FormatPostingsPositionsConsumer posConsumer = docConsumer.addDoc(doc, freq); - if (!omitTF) { + if (!omitTermFreqAndPositions) { for (int j = 0; j < freq; j++) { final int position = postings.nextPosition(); final int payloadLength = postings.getPayloadLength(); Index: src/java/org/apache/lucene/index/FreqProxTermsWriter.java =================================================================== --- src/java/org/apache/lucene/index/FreqProxTermsWriter.java (revision 755652) +++ src/java/org/apache/lucene/index/FreqProxTermsWriter.java (working copy) @@ -176,7 +176,7 @@ FreqProxFieldMergeState[] termStates = new FreqProxFieldMergeState[numFields]; - final boolean currentFieldOmitTf = fields[0].fieldInfo.omitTf; + final boolean currentFieldOmitTermFreqAndPositions = fields[0].fieldInfo.omitTermFreqAndPositions; while(numFields > 0) { @@ -217,8 +217,8 @@ // Carefully copy over the prox + payload info, // changing the format to match Lucene's segment // format. - if (!currentFieldOmitTf) { - // omitTf == false so we do write positions & + if (!currentFieldOmitTermFreqAndPositions) { + // omitTermFreqAndPositions == false so we do write positions & // payload int position = 0; for(int j=0;j 0; + assert omitTermFreqAndPositions || p.docFreq > 0; - if (omitTf) { + if (omitTermFreqAndPositions) { if (docState.docID != p.lastDocID) { assert docState.docID > p.lastDocID; termsHashPerField.writeVInt(0, p.lastDocCode); Index: src/java/org/apache/lucene/index/FieldInfo.java =================================================================== --- src/java/org/apache/lucene/index/FieldInfo.java (revision 755652) +++ src/java/org/apache/lucene/index/FieldInfo.java (working copy) @@ -28,13 +28,13 @@ boolean storePositionWithTermVector; boolean omitNorms; // omit norms associated with indexed fields - boolean omitTf; // omit tf + boolean omitTermFreqAndPositions; boolean storePayloads; // whether this field stores payloads together with term positions FieldInfo(String na, boolean tk, int nu, boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, - boolean omitNorms, boolean storePayloads, boolean omitTf) { + boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) { name = na; isIndexed = tk; number = nu; @@ -43,16 +43,16 @@ this.storePositionWithTermVector = storePositionWithTermVector; this.omitNorms = omitNorms; this.storePayloads = storePayloads; - this.omitTf = omitTf; + this.omitTermFreqAndPositions = omitTermFreqAndPositions; } public Object clone() { return new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, - storeOffsetWithTermVector, omitNorms, storePayloads, omitTf); + storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions); } void update(boolean isIndexed, boolean storeTermVector, boolean storePositionWithTermVector, - boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean omitTf) { + boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) { if (this.isIndexed != isIndexed) { this.isIndexed = true; // once indexed, always index } @@ -68,8 +68,8 @@ if (this.omitNorms != omitNorms) { this.omitNorms = false; // once norms are stored, always store } - if (this.omitTf != omitTf) { - this.omitTf = true; // if one require omitTf at least once, it remains off for life + if (this.omitTermFreqAndPositions != omitTermFreqAndPositions) { + this.omitTermFreqAndPositions = true; // if one require omitTermFreqAndPositions at least once, it remains off for life } if (this.storePayloads != storePayloads) { this.storePayloads = true; Index: src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- src/java/org/apache/lucene/index/IndexReader.java (revision 755652) +++ src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -92,7 +92,9 @@ /** All fields that store payloads */ public static final FieldOption STORES_PAYLOADS = new FieldOption ("STORES_PAYLOADS"); /** All fields that omit tf */ - public static final FieldOption OMIT_TF = new FieldOption ("OMIT_TF"); + public static final FieldOption OMIT_TERM_FREQ_AND_POSITIONS = new FieldOption ("OMIT_TERM_FREQ_AND_POSITIONS"); + /** @deprecated Renamed to {@link #OMIT_TERM_FREQ_AND_POSITIONS} */ + public static final FieldOption OMIT_TF = OMIT_TERM_FREQ_AND_POSITIONS; /** All fields which are not indexed */ public static final FieldOption UNINDEXED = new FieldOption ("UNINDEXED"); /** All fields which are indexed with termvectors enabled */ Index: src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- src/java/org/apache/lucene/index/DocumentsWriter.java (revision 755652) +++ src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -272,7 +272,7 @@ } /** Returns true if any of the fields in the current - * buffered docs have omitTf==false */ + * buffered docs have omitTermFreqAndPositions==false */ boolean hasProx() { return (docFieldProcessor != null) ? docFieldProcessor.fieldInfos.hasProx() : true; Index: src/java/org/apache/lucene/document/AbstractField.java =================================================================== --- src/java/org/apache/lucene/document/AbstractField.java (revision 755652) +++ src/java/org/apache/lucene/document/AbstractField.java (working copy) @@ -33,7 +33,7 @@ protected boolean isBinary = false; protected boolean isCompressed = false; protected boolean lazy = false; - protected boolean omitTf = false; + protected boolean omitTermFreqAndPositions = false; protected float boost = 1.0f; // the one and only data object for all different kind of field values protected Object fieldsData = null; @@ -263,8 +263,9 @@ /** True if norms are omitted for this indexed field */ public boolean getOmitNorms() { return omitNorms; } - /** True if tf is omitted for this indexed field */ - public boolean getOmitTf() { return omitTf; } + public boolean getOmitTf() { return omitTermFreqAndPositions; } + + public boolean getOmitTermFreqAndPositions() { return omitTermFreqAndPositions; } /** Expert: * @@ -273,11 +274,9 @@ */ public void setOmitNorms(boolean omitNorms) { this.omitNorms=omitNorms; } - /** Expert: - * - * If set, omit tf from postings of this indexed field. - */ - public void setOmitTf(boolean omitTf) { this.omitTf=omitTf; } + public void setOmitTf(boolean omitTermFreqAndPositions) { this.omitTermFreqAndPositions=omitTermFreqAndPositions; } + + public void setOmitTermFreqAndPositions(boolean omitTermFreqAndPositions) { this.omitTermFreqAndPositions=omitTermFreqAndPositions; } public boolean isLazy() { return lazy; @@ -326,8 +325,8 @@ if (omitNorms) { result.append(",omitNorms"); } - if (omitTf) { - result.append(",omitTf"); + if (omitTermFreqAndPositions) { + result.append(",omitTermFreqAndPositions"); } if (lazy){ result.append(",lazy"); Index: src/java/org/apache/lucene/document/Fieldable.java =================================================================== --- src/java/org/apache/lucene/document/Fieldable.java (revision 755652) +++ src/java/org/apache/lucene/document/Fieldable.java (working copy) @@ -18,6 +18,8 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.FieldInvertState; +import org.apache.lucene.search.PhraseQuery; // for javadocs +import org.apache.lucene.search.spans.SpanQuery; // for javadocs import java.io.Reader; import java.io.Serializable; @@ -145,14 +147,26 @@ */ void setOmitNorms(boolean omitNorms); + /** @deprecated Renamed to {@link #setOmitTermFreqAndPositions} */ + void setOmitTf(boolean omitTf); + + /** @deprecated Renamed to {@link #getOmitTermFreqAndPositions} */ + boolean getOmitTf(); + /** Expert: * * If set, omit term freq, positions and payloads from postings for this field. + *

NOTE: this is a dangerous option to enable. + * While it reduces storage space required in the index, + * it also means any query requiring positional + * infromation, such as {@link PhraseQuery} or {@link + * SpanQuery} subclasses will silently fail to find + * results. */ - void setOmitTf(boolean omitTf); - + void setOmitTermFreqAndPositions(boolean omitTermFreqAndPositions); + /** True if tf is omitted for this indexed field */ - boolean getOmitTf(); + boolean getOmitTermFreqAndPositions(); /** * Indicates whether a Field is Lazy or not. The semantics of Lazy loading are such that if a Field is lazily loaded, retrieving Index: contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java =================================================================== --- contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java (revision 755652) +++ contrib/queries/src/java/org/apache/lucene/search/trie/TrieUtils.java (working copy) @@ -377,7 +377,7 @@ for (int i=0; i