Index: lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java =================================================================== --- lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 1530990) +++ lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy) @@ -825,6 +825,11 @@ } @Override + public boolean hasFreqs() { + return true; + } + + @Override public boolean hasOffsets() { return storeOffsets; } Index: lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (revision 1530990) +++ lucene/test-framework/src/java/org/apache/lucene/index/BasePostingsFormatTestCase.java (working copy) @@ -527,6 +527,11 @@ } @Override + public boolean hasFreqs() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + } + + @Override public boolean hasOffsets() { return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; } Index: lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (revision 1530990) +++ lucene/test-framework/src/java/org/apache/lucene/codecs/ramonly/RAMOnlyPostingsFormat.java (working copy) @@ -145,6 +145,11 @@ } @Override + public boolean hasFreqs() { + return info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + } + + @Override public boolean hasOffsets() { return info.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; } Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (revision 1530990) +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (working copy) @@ -291,6 +291,11 @@ } @Override + public boolean hasFreqs() { + return true; + } + + @Override public boolean hasOffsets() { return hasOffsets; } Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (revision 1530990) +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (working copy) @@ -604,6 +604,11 @@ } @Override + public boolean hasFreqs() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + } + + @Override public boolean hasOffsets() { return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; } Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java (revision 1530990) +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsWriter.java (working copy) @@ -85,10 +85,7 @@ boolean wroteField = false; boolean hasPositions = terms.hasPositions(); - - // TODO: shouldn't we add hasFreqs to Terms? - // then we don't need FieldInfos here? - boolean hasFreqs = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_ONLY) > 0; + boolean hasFreqs = terms.hasFreqs(); boolean hasPayloads = fieldInfo.hasPayloads(); boolean hasOffsets = terms.hasOffsets(); Index: lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (revision 1530990) +++ lucene/codecs/src/java/org/apache/lucene/codecs/blockterms/BlockTermsReader.java (working copy) @@ -249,6 +249,11 @@ } @Override + public boolean hasFreqs() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + } + + @Override public boolean hasOffsets() { return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; } Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (revision 1530990) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermsReader.java (working copy) @@ -176,6 +176,11 @@ } @Override + public boolean hasFreqs() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + } + + @Override public boolean hasOffsets() { return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; } Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (revision 1530990) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (working copy) @@ -817,6 +817,11 @@ } @Override + public boolean hasFreqs() { + return field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + } + + @Override public boolean hasOffsets() { return field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; } Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (revision 1530990) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (working copy) @@ -660,6 +660,11 @@ } @Override + public boolean hasFreqs() { + return hasFreq; + } + + @Override public boolean hasOffsets() { return hasOffsets; } Index: lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (revision 1530990) +++ lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (working copy) @@ -277,6 +277,11 @@ } @Override + public boolean hasFreqs() { + return delegateTerms.hasFreqs(); + } + + @Override public boolean hasOffsets() { return delegateTerms.hasOffsets(); } Index: lucene/core/src/test/org/apache/lucene/index/TestCodecs.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (revision 1530990) +++ lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (working copy) @@ -691,6 +691,11 @@ } @Override + public boolean hasFreqs() { + return fieldData.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + } + + @Override public boolean hasOffsets() { return fieldData.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; } Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (revision 1530990) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (working copy) @@ -360,6 +360,11 @@ } @Override + public boolean hasFreqs() { + return true; + } + + @Override public boolean hasOffsets() { return storeOffsets; } Index: lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java (revision 1530990) +++ lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java (working copy) @@ -741,6 +741,11 @@ } @Override + public boolean hasFreqs() { + return true; + } + + @Override public boolean hasOffsets() { return (flags & OFFSETS) != 0; } Index: lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (revision 1530990) +++ lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (working copy) @@ -497,6 +497,11 @@ } @Override + public boolean hasFreqs() { + return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + } + + @Override public boolean hasOffsets() { return fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; } Index: lucene/core/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java (revision 1530990) +++ lucene/core/src/java/org/apache/lucene/search/FieldCacheRewriteMethod.java (working copy) @@ -114,6 +114,11 @@ } @Override + public boolean hasFreqs() { + return false; + } + + @Override public boolean hasOffsets() { return false; } Index: lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRewriteMethod.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRewriteMethod.java (revision 1530990) +++ lucene/core/src/java/org/apache/lucene/search/DocTermOrdsRewriteMethod.java (working copy) @@ -114,6 +114,11 @@ } @Override + public boolean hasFreqs() { + return false; + } + + @Override public boolean hasOffsets() { return false; } Index: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (revision 1530990) +++ lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (working copy) @@ -744,11 +744,41 @@ continue; } + final boolean hasFreqs = terms.hasFreqs(); final boolean hasPositions = terms.hasPositions(); + final boolean hasPayloads = terms.hasPayloads(); final boolean hasOffsets = terms.hasOffsets(); - // term vectors cannot omit TF - final boolean hasFreqs = isVectors || fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + // term vectors cannot omit TF: + final boolean expectedHasFreqs = (isVectors || fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0); + + if (hasFreqs != expectedHasFreqs) { + throw new RuntimeException("field \"" + field + "\" should have hasFreqs=" + expectedHasFreqs + " but got " + hasFreqs); + } + + if (hasFreqs == false) { + if (terms.getSumTotalTermFreq() != -1) { + throw new RuntimeException("field \"" + field + "\" hasFreqs is false, but Terms.getSumTotalTermFreq()=" + terms.getSumTotalTermFreq() + " (should be -1)"); + } + } + + if (!isVectors) { + final boolean expectedHasPositions = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; + if (hasPositions != expectedHasPositions) { + throw new RuntimeException("field \"" + field + "\" should have hasPositions=" + expectedHasPositions + " but got " + hasPositions); + } + + final boolean expectedHasPayloads = fieldInfo.hasPayloads(); + if (hasPayloads != expectedHasPayloads) { + throw new RuntimeException("field \"" + field + "\" should have hasPayloads=" + expectedHasPayloads + " but got " + hasPayloads); + } + + final boolean expectedHasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; + if (hasOffsets != expectedHasOffsets) { + throw new RuntimeException("field \"" + field + "\" should have hasOffsets=" + expectedHasOffsets + " but got " + hasOffsets); + } + } + final TermsEnum termsEnum = terms.iterator(null); boolean hasOrd = true; @@ -787,6 +817,12 @@ docs = termsEnum.docs(liveDocs, docs); postings = termsEnum.docsAndPositions(liveDocs, postings); + + if (hasFreqs == false) { + if (termsEnum.totalTermFreq() != -1) { + throw new RuntimeException("field \"" + field + "\" hasFreqs is false, but TermsEnum.totalTermFreq()=" + termsEnum.totalTermFreq() + " (should be -1)"); + } + } if (hasOrd) { long ord = -1; @@ -829,6 +865,13 @@ } status.totPos += freq; totalTermFreq += freq; + } else { + // When a field didn't index freq, it must + // consistently "lie" and pretend that freq was + // 1: + if (docs2.freq() != 1) { + throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " != 1 when Terms.hasFreqs() is false"); + } } docCount++; Index: lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java (revision 1530990) +++ lucene/core/src/java/org/apache/lucene/index/FreqProxFields.java (working copy) @@ -104,6 +104,11 @@ } @Override + public boolean hasFreqs() { + return terms.fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0; + } + + @Override public boolean hasOffsets() { // NOTE: the in-memory buffer may have indexed offsets // because that's what FieldInfo said when we started, Index: lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (revision 1530990) +++ lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (working copy) @@ -119,6 +119,11 @@ } @Override + public boolean hasFreqs() { + return in.hasFreqs(); + } + + @Override public boolean hasOffsets() { return in.hasOffsets(); } Index: lucene/core/src/java/org/apache/lucene/index/Terms.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/Terms.java (revision 1530990) +++ lucene/core/src/java/org/apache/lucene/index/Terms.java (working copy) @@ -102,8 +102,10 @@ * into account. */ public abstract int getDocCount() throws IOException; - // TODO: shouldn't we have hasFreq() as well? - + /** Returns true if documents in this field store + * per-document term frequency ({@link DocsEnum#freq}). */ + public abstract boolean hasFreqs(); + /** Returns true if documents in this field store offsets. */ public abstract boolean hasOffsets(); Index: lucene/core/src/java/org/apache/lucene/index/MultiTerms.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MultiTerms.java (revision 1530990) +++ lucene/core/src/java/org/apache/lucene/index/MultiTerms.java (working copy) @@ -35,6 +35,7 @@ public final class MultiTerms extends Terms { private final Terms[] subs; private final ReaderSlice[] subSlices; + private final boolean hasFreqs; private final boolean hasOffsets; private final boolean hasPositions; private final boolean hasPayloads; @@ -50,15 +51,18 @@ this.subSlices = subSlices; assert subs.length > 0 : "inefficient: don't use MultiTerms over one sub"; + boolean _hasFreqs = true; boolean _hasOffsets = true; boolean _hasPositions = true; boolean _hasPayloads = false; for(int i=0;i