Index: lucene/src/test/org/apache/lucene/index/TestOmitTf.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestOmitTf.java (revision 1092048) +++ lucene/src/test/org/apache/lucene/index/TestOmitTf.java (working copy) @@ -119,6 +119,7 @@ setMaxBufferedDocs(3). setMergePolicy(newLogMergePolicy(2)) ); + writer.setInfoStream(VERBOSE ? System.out : null); Document d = new Document(); // this field will have Tf Index: lucene/src/java/org/apache/lucene/index/FieldInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInfos.java (revision 1092048) +++ lucene/src/java/org/apache/lucene/index/FieldInfos.java (working copy) @@ -424,8 +424,8 @@ } synchronized private FieldInfo addOrUpdateInternal(String name, int preferredFieldNumber, boolean isIndexed, - boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, - boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) { + boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, + boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) { if (globalFieldNumbers == null) { throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos"); } @@ -567,6 +567,7 @@ output.writeVInt(FORMAT_CURRENT); output.writeVInt(size()); for (FieldInfo fi : this) { + assert !fi.omitTermFreqAndPositions || !fi.storePayloads; byte bits = 0x0; if (fi.isIndexed) bits |= IS_INDEXED; if (fi.storeTermVector) bits |= STORE_TERMVECTOR; @@ -607,6 +608,14 @@ boolean omitNorms = (bits & OMIT_NORMS) != 0; boolean storePayloads = (bits & STORE_PAYLOADS) != 0; boolean omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0; + + // LUCENE-3027: past indices were able to write + // storePayloads=true when omitTFAP is also true, + // which is invalid. We correct that, here: + if (omitTermFreqAndPositions) { + storePayloads = false; + } + final FieldInfo addInternal = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions); addInternal.setCodecId(codecId); } Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (revision 1092048) +++ lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (working copy) @@ -173,9 +173,10 @@ if (totalFieldCount >= fieldHash.length/2) rehash(); } else { - fieldInfos.addOrUpdate(fp.fieldInfo.name, field.isIndexed(), field.isTermVectorStored(), - field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(), - field.getOmitNorms(), false, field.getOmitTermFreqAndPositions()); + FieldInfo fi = fieldInfos.addOrUpdate(fp.fieldInfo.name, field.isIndexed(), field.isTermVectorStored(), + field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(), + field.getOmitNorms(), false, field.getOmitTermFreqAndPositions()); + assert !fi.omitTermFreqAndPositions || !fi.storePayloads; } if (thisFieldGen != fp.lastGen) { Index: lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (revision 1092048) +++ lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (working copy) @@ -104,7 +104,9 @@ // Aggregate the storePayload as seen by the same // field across multiple threads - fieldInfo.storePayloads |= fields[i-start].hasPayloads; + if (!fieldInfo.omitTermFreqAndPositions) { + fieldInfo.storePayloads |= fields[i-start].hasPayloads; + } } // If this field has postings then add them to the Index: lucene/src/java/org/apache/lucene/index/FieldInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInfo.java (revision 1092048) +++ lucene/src/java/org/apache/lucene/index/FieldInfo.java (working copy) @@ -58,6 +58,7 @@ this.omitNorms = false; this.omitTermFreqAndPositions = false; } + assert !omitTermFreqAndPositions || !storePayloads; } void setCodecId(int codecId) { @@ -80,6 +81,7 @@ // should only be called by FieldInfos#addOrUpdate void update(boolean isIndexed, boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, boolean omitNorms, boolean storePayloads, boolean omitTermFreqAndPositions) { + if (this.isIndexed != isIndexed) { this.isIndexed = true; // once indexed, always index } @@ -101,7 +103,9 @@ } if (this.omitTermFreqAndPositions != omitTermFreqAndPositions) { this.omitTermFreqAndPositions = true; // if one require omitTermFreqAndPositions at least once, it remains off for life + this.storePayloads = false; } } + assert !this.omitTermFreqAndPositions || !this.storePayloads; } } Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java (revision 1092048) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepSkipListReader.java (working copy) @@ -177,6 +177,7 @@ @Override protected int readSkipData(int level, IndexInput skipStream) throws IOException { int delta; + assert !omitTF || !currentFieldStoresPayloads; if (currentFieldStoresPayloads) { // the current field stores payloads. // if the doc delta is odd then we have