Index: src/java/org/apache/lucene/document/Field.java =================================================================== --- src/java/org/apache/lucene/document/Field.java (revision 763026) +++ src/java/org/apache/lucene/document/Field.java (working copy) @@ -329,6 +329,8 @@ if (index == Index.NO) { this.isIndexed = false; this.isTokenized = false; + this.omitTermFreqAndPositions = false; + this.omitNorms = true; } else if (index == Index.ANALYZED) { this.isIndexed = true; this.isTokenized = true; @@ -492,6 +494,8 @@ isIndexed = false; isTokenized = false; + omitTermFreqAndPositions = false; + omitNorms = true; isBinary = true; binaryLength = length; Index: src/java/org/apache/lucene/index/FieldInfo.java =================================================================== --- src/java/org/apache/lucene/index/FieldInfo.java (revision 763026) +++ src/java/org/apache/lucene/index/FieldInfo.java (working copy) @@ -38,12 +38,21 @@ name = na; isIndexed = tk; number = nu; - this.storeTermVector = storeTermVector; - this.storeOffsetWithTermVector = storeOffsetWithTermVector; - this.storePositionWithTermVector = storePositionWithTermVector; - this.omitNorms = omitNorms; - this.storePayloads = storePayloads; - this.omitTermFreqAndPositions = omitTermFreqAndPositions; + if (isIndexed) { + this.storeTermVector = storeTermVector; + this.storeOffsetWithTermVector = storeOffsetWithTermVector; + this.storePositionWithTermVector = storePositionWithTermVector; + this.storePayloads = storePayloads; + this.omitNorms = omitNorms; + this.omitTermFreqAndPositions = omitTermFreqAndPositions; + } else { // for non-indexed fields, leave defaults + this.storeTermVector = false; + this.storeOffsetWithTermVector = false; + this.storePositionWithTermVector = false; + this.storePayloads = false; + this.omitNorms = true; + this.omitTermFreqAndPositions = false; + } } public Object clone() { @@ -56,23 +65,25 @@ if (this.isIndexed != isIndexed) { this.isIndexed = true; // once indexed, always index } - if (this.storeTermVector != storeTermVector) { - this.storeTermVector = true; // once vector, always vector + if (isIndexed) { // if updated field data is not for indexing, leave the updates out + if (this.storeTermVector != storeTermVector) { + this.storeTermVector = true; // once vector, always vector + } + if (this.storePositionWithTermVector != storePositionWithTermVector) { + this.storePositionWithTermVector = true; // once vector, always vector + } + if (this.storeOffsetWithTermVector != storeOffsetWithTermVector) { + this.storeOffsetWithTermVector = true; // once vector, always vector + } + if (this.storePayloads != storePayloads) { + this.storePayloads = true; + } + if (this.omitNorms != omitNorms) { + this.omitNorms = false; // once norms are stored, always store + } + if (this.omitTermFreqAndPositions != omitTermFreqAndPositions) { + this.omitTermFreqAndPositions = true; // if one require omitTermFreqAndPositions at least once, it remains off for life + } } - if (this.storePositionWithTermVector != storePositionWithTermVector) { - this.storePositionWithTermVector = true; // once vector, always vector - } - if (this.storeOffsetWithTermVector != storeOffsetWithTermVector) { - this.storeOffsetWithTermVector = true; // once vector, always vector - } - if (this.omitNorms != omitNorms) { - this.omitNorms = false; // once norms are stored, always store - } - if (this.omitTermFreqAndPositions != omitTermFreqAndPositions) { - this.omitTermFreqAndPositions = true; // if one require omitTermFreqAndPositions at least once, it remains off for life - } - if (this.storePayloads != storePayloads) { - this.storePayloads = true; - } } } Index: src/java/org/apache/lucene/index/FieldInfos.java =================================================================== --- src/java/org/apache/lucene/index/FieldInfos.java (revision 763026) +++ src/java/org/apache/lucene/index/FieldInfos.java (working copy) @@ -84,7 +84,7 @@ while (fieldIterator.hasNext()) { Fieldable field = (Fieldable) fieldIterator.next(); add(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(), - field.isStoreOffsetWithTermVector(), field.getOmitNorms()); + field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTf()); } } Index: src/java/org/apache/lucene/index/FieldsReader.java =================================================================== --- src/java/org/apache/lucene/index/FieldsReader.java (revision 763026) +++ src/java/org/apache/lucene/index/FieldsReader.java (working copy) @@ -299,13 +299,17 @@ if (binary) { int toRead = fieldsStream.readVInt(); long pointer = fieldsStream.getFilePointer(); + Fieldable f; if (compressed) { //was: doc.add(new Fieldable(fi.name, uncompress(b), Fieldable.Store.COMPRESS)); - doc.add(new LazyField(fi.name, Field.Store.COMPRESS, toRead, pointer, binary)); + f = new LazyField(fi.name, Field.Store.COMPRESS, toRead, pointer, binary); } else { //was: doc.add(new Fieldable(fi.name, b, Fieldable.Store.YES)); - doc.add(new LazyField(fi.name, Field.Store.YES, toRead, pointer, binary)); + f = new LazyField(fi.name, Field.Store.YES, toRead, pointer, binary); } + f.setOmitNorms(fi.omitNorms); + f.setOmitTf(fi.omitTermFreqAndPositions); + doc.add(f); //Need to move the pointer ahead by toRead positions fieldsStream.seek(pointer + toRead); } else { @@ -322,6 +326,7 @@ //skip over the part that we aren't loading fieldsStream.seek(pointer + toRead); f.setOmitNorms(fi.omitNorms); + f.setOmitTf(fi.omitTermFreqAndPositions); } else { int length = fieldsStream.readVInt(); long pointer = fieldsStream.getFilePointer(); @@ -332,6 +337,7 @@ fieldsStream.skipChars(length); f = new LazyField(fi.name, store, index, termVector, length, pointer, binary); f.setOmitNorms(fi.omitNorms); + f.setOmitTf(fi.omitTermFreqAndPositions); } doc.add(f); } @@ -361,11 +367,14 @@ int toRead = fieldsStream.readVInt(); final byte[] b = new byte[toRead]; fieldsStream.readBytes(b, 0, b.length); + Fieldable f; if (compressed) - doc.add(new Field(fi.name, uncompress(b), Field.Store.COMPRESS)); + f = new Field(fi.name, uncompress(b), Field.Store.COMPRESS); else - doc.add(new Field(fi.name, b, Field.Store.YES)); - + f = new Field(fi.name, b, Field.Store.YES); + f.setOmitTf(fi.omitTermFreqAndPositions); + f.setOmitNorms(fi.omitNorms); + doc.add(f); } else { Field.Store store = Field.Store.YES; Field.Index index = getIndexType(fi, tokenize); @@ -383,6 +392,7 @@ store, index, termVector); + f.setOmitTf(fi.omitTermFreqAndPositions); f.setOmitNorms(fi.omitNorms); } else { f = new Field(fi.name, // name @@ -390,6 +400,7 @@ store, index, termVector); + f.setOmitTf(fi.omitTermFreqAndPositions); f.setOmitNorms(fi.omitNorms); } doc.add(f); @@ -641,6 +652,7 @@ this.name = fi.name.intern(); this.isIndexed = fi.isIndexed; this.omitNorms = fi.omitNorms; + this.omitTermFreqAndPositions = fi.omitTermFreqAndPositions; this.storeOffsetWithTermVector = fi.storeOffsetWithTermVector; this.storePositionWithTermVector = fi.storePositionWithTermVector; this.storeTermVector = fi.storeTermVector; Index: src/test/org/apache/lucene/index/DocHelper.java =================================================================== --- src/test/org/apache/lucene/index/DocHelper.java (revision 763026) +++ src/test/org/apache/lucene/index/DocHelper.java (working copy) @@ -64,6 +64,14 @@ public static Field noNormsField = new Field(NO_NORMS_KEY, NO_NORMS_TEXT, Field.Store.YES, Field.Index.NOT_ANALYZED_NO_NORMS); + public static final String NO_TF_TEXT = "analyzed with no tf and positions"; + public static final String NO_TF_KEY = "omitTermFreqAndPositions"; + public static Field noTFField = new Field(NO_TF_KEY, NO_TF_TEXT, + Field.Store.YES, Field.Index.ANALYZED); + static { + noTFField.setOmitTermFreqAndPositions(true); + } + public static final String UNINDEXED_FIELD_TEXT = "unindexed field text"; public static final String UNINDEXED_FIELD_KEY = "unIndField"; public static Field unIndField = new Field(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT, @@ -119,6 +127,7 @@ compressedTextField2, keyField, noNormsField, + noTFField, unIndField, unStoredField1, unStoredField2, @@ -139,6 +148,7 @@ public static Map notermvector=new HashMap(); public static Map lazy= new HashMap(); public static Map noNorms=new HashMap(); + public static Map noTf=new HashMap(); static { //Initialize the large Lazy Field @@ -167,6 +177,7 @@ if (f.isStored()) add(stored,f); else add(unstored,f); if (f.getOmitNorms()) add(noNorms,f); + if (f.getOmitTf()) add(noTf,f); if (f.isLazy()) add(lazy, f); } } @@ -186,6 +197,7 @@ nameValues.put(TEXT_FIELD_3_KEY, FIELD_3_TEXT); nameValues.put(KEYWORD_FIELD_KEY, KEYWORD_TEXT); nameValues.put(NO_NORMS_KEY, NO_NORMS_TEXT); + nameValues.put(NO_TF_KEY, NO_TF_TEXT); nameValues.put(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT); nameValues.put(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT); nameValues.put(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT); Index: src/test/org/apache/lucene/index/TestDocumentWriter.java =================================================================== --- src/test/org/apache/lucene/index/TestDocumentWriter.java (revision 763026) +++ src/test/org/apache/lucene/index/TestDocumentWriter.java (working copy) @@ -259,14 +259,13 @@ doc.add(new Field("f2", "v1", Store.YES, Index.NOT_ANALYZED, TermVector.WITH_POSITIONS_OFFSETS)); doc.add(new Field("f2", "v2", Store.YES, Index.NOT_ANALYZED, TermVector.NO)); - RAMDirectory ram = new RAMDirectory(); - IndexWriter writer = new IndexWriter(ram, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); + IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); writer.addDocument(doc); writer.close(); - _TestUtil.checkIndex(ram); + _TestUtil.checkIndex(dir); - IndexReader reader = IndexReader.open(ram); + IndexReader reader = IndexReader.open(dir); // f1 TermFreqVector tfv1 = reader.getTermFreqVector(0, "f1"); assertNotNull(tfv1); @@ -276,4 +275,37 @@ assertNotNull(tfv2); assertEquals("the 'with_tv' setting should rule!",2,tfv2.getTerms().length); } + + /** + * Test adding two fields with the same name, one indexed + * the other stored only. The omitNorms and omitTermFreqAndPositions setting + * of the stored field should not affect the indexed one (LUCENE-1590) + */ + public void testLUCENE_1590() throws Exception { + Document doc = new Document(); + // f1 has no norms + doc.add(new Field("f1", "v1", Store.NO, Index.ANALYZED_NO_NORMS)); + doc.add(new Field("f1", "v2", Store.YES, Index.NO)); + // f2 has no TF + Field f = new Field("f2", "v1", Store.NO, Index.ANALYZED); + f.setOmitTermFreqAndPositions(true); + doc.add(f); + doc.add(new Field("f2", "v2", Store.YES, Index.NO)); + + IndexWriter writer = new IndexWriter(dir, new StandardAnalyzer(), true, IndexWriter.MaxFieldLength.LIMITED); + writer.addDocument(doc); + writer.optimize(); // be sure to have a single segment + writer.close(); + + _TestUtil.checkIndex(dir); + + SegmentReader reader = (SegmentReader) IndexReader.open(dir); + FieldInfos fi = reader.fieldInfos(); + // f1 + assertFalse("f1 should have no norms", reader.hasNorms("f1")); + assertFalse("omitTermFreqAndPositions field bit should not be set for f1", fi.fieldInfo("f1").omitTermFreqAndPositions); + // f2 + assertTrue("f2 should have norms", reader.hasNorms("f2")); + assertTrue("omitTermFreqAndPositions field bit should be set for f2", fi.fieldInfo("f2").omitTermFreqAndPositions); + } } Index: src/test/org/apache/lucene/index/TestFieldsReader.java =================================================================== --- src/test/org/apache/lucene/index/TestFieldsReader.java (revision 763026) +++ src/test/org/apache/lucene/index/TestFieldsReader.java (working copy) @@ -72,6 +72,7 @@ assertTrue(field.isStoreOffsetWithTermVector() == true); assertTrue(field.isStorePositionWithTermVector() == true); assertTrue(field.getOmitNorms() == false); + assertTrue(field.getOmitTf() == false); field = doc.getField(DocHelper.TEXT_FIELD_3_KEY); assertTrue(field != null); @@ -79,8 +80,15 @@ assertTrue(field.isStoreOffsetWithTermVector() == false); assertTrue(field.isStorePositionWithTermVector() == false); assertTrue(field.getOmitNorms() == true); + assertTrue(field.getOmitTf() == false); - + field = doc.getField(DocHelper.NO_TF_KEY); + assertTrue(field != null); + assertTrue(field.isTermVectorStored() == false); + assertTrue(field.isStoreOffsetWithTermVector() == false); + assertTrue(field.isStorePositionWithTermVector() == false); + assertTrue(field.getOmitNorms() == false); + assertTrue(field.getOmitTf() == true); reader.close(); }