Index: lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/BlockTreeTermsReader.java (working copy) @@ -902,20 +902,14 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { if (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { // Positions were not indexed: return null; } - if (needsOffsets && - fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) < 0) { - // Offsets were not indexed: - return null; - } - currentFrame.decodeMetaData(); - return postingsReader.docsAndPositions(fieldInfo, currentFrame.termState, skipDocs, reuse, needsOffsets); + return postingsReader.docsAndPositions(fieldInfo, currentFrame.termState, skipDocs, reuse, flags); } private int getState() { @@ -2137,21 +2131,15 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits skipDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { if (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { // Positions were not indexed: return null; } - if (needsOffsets && - fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) < 0) { - // Offsets were not indexed: - return null; - } - assert !eof; currentFrame.decodeMetaData(); - return postingsReader.docsAndPositions(fieldInfo, currentFrame.state, skipDocs, reuse, needsOffsets); + return postingsReader.docsAndPositions(fieldInfo, currentFrame.state, skipDocs, reuse, flags); } @Override Index: lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/TermVectorsWriter.java (working copy) @@ -224,16 +224,9 @@ // TODO: we need a "query" API where we can ask (via // flex API) what this term was indexed with... // Both positions & offsets: - docsAndPositionsEnum = termsEnum.docsAndPositions(null, null, true); - final boolean hasOffsets; + docsAndPositionsEnum = termsEnum.docsAndPositions(null, null); + boolean hasOffsets = false; boolean hasPositions = false; - if (docsAndPositionsEnum == null) { - // Fallback: no offsets - docsAndPositionsEnum = termsEnum.docsAndPositions(null, null, false); - hasOffsets = false; - } else { - hasOffsets = true; - } if (docsAndPositionsEnum != null) { final int docID = docsAndPositionsEnum.nextDoc(); @@ -242,23 +235,19 @@ for(int posUpto=0; posUpto 0; hasPositions = pos != -1; + hasOffsets = startOffset != -1; startField(fieldInfo, numTerms, hasPositions, hasOffsets); startTerm(termsEnum.term(), freq); startedField = true; } - final int startOffset; - final int endOffset; if (hasOffsets) { - startOffset = docsAndPositionsEnum.startOffset(); - endOffset = docsAndPositionsEnum.endOffset(); assert startOffset != -1; assert endOffset != -1; - } else { - startOffset = -1; - endOffset = -1; } assert !hasPositions || pos >= 0; addPosition(pos, startOffset, endOffset); Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40PostingsReader.java (working copy) @@ -250,11 +250,14 @@ @Override public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, - DocsAndPositionsEnum reuse, boolean needsOffsets) + DocsAndPositionsEnum reuse, int flags) throws IOException { boolean hasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; + // TODO: can we optimize if FLAG_PAYLOADS / FLAG_OFFSETS + // isn't passed? + // TODO: refactor if (fieldInfo.hasPayloads() || hasOffsets) { SegmentFullPositionsEnum docsEnum; Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java (working copy) @@ -490,10 +490,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { - if (needsOffsets && !storeOffsets) { - return null; - } + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { if (!storePositions && !storeOffsets) { return null; @@ -640,14 +637,20 @@ @Override public int startOffset() { - assert startOffsets != null; - return startOffsets[nextPos-1]; + if (startOffsets == null) { + return -1; + } else { + return startOffsets[nextPos-1]; + } } @Override public int endOffset() { - assert endOffsets != null; - return endOffsets[nextPos-1]; + if (endOffsets == null) { + return -1; + } else { + return endOffsets[nextPos-1]; + } } } Index: lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/PostingsReaderBase.java (working copy) @@ -56,7 +56,7 @@ /** Must fully consume state, since after this call that * TermState may be reused. */ public abstract DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsAndPositionsEnum reuse, - boolean needsOffsets) throws IOException; + int flags) throws IOException; public abstract void close() throws IOException; Index: lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/memory/MemoryPostingsFormat.java (working copy) @@ -714,13 +714,9 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) { boolean hasOffsets = field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; - if (needsOffsets && !hasOffsets) { - return null; // not available - } - if (field.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { return null; } Index: lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (working copy) @@ -314,7 +314,7 @@ termOffsets[count+1] = termOffset; if (hasPos) { - docsAndPositionsEnum = termsEnum.docsAndPositions(null, docsAndPositionsEnum, hasOffsets); + docsAndPositionsEnum = termsEnum.docsAndPositions(null, docsAndPositionsEnum); } else { docsEnum = termsEnum.docs(null, docsEnum, hasFreq); } @@ -858,13 +858,10 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) { if (!hasPos) { return null; } - if (needsOffsets && !hasOffsets) { - return null; - } // TODO: implement reuse, something like Pulsing: // it's hairy! @@ -1420,13 +1417,10 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) { if (!hasPos) { return null; } - if (needsOffsets && !hasOffsets) { - return null; - } // TODO: implement reuse, something like Pulsing: // it's hairy! Index: lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/TermsConsumer.java (working copy) @@ -21,6 +21,7 @@ import java.util.Comparator; import org.apache.lucene.index.FieldInfo; // javadocs +import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.MergeState; import org.apache.lucene.index.TermsEnum; @@ -150,7 +151,7 @@ while((term = termsEnum.next()) != null) { // We can pass null for liveDocs, because the // mapping enum will skip the non-live docs: - postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(null, postingsEnumIn, false); + postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(null, postingsEnumIn, DocsAndPositionsEnum.FLAG_PAYLOADS); assert postingsEnumIn != null; postingsEnum.reset(postingsEnumIn); // set PayloadProcessor @@ -184,7 +185,7 @@ while((term = termsEnum.next()) != null) { // We can pass null for liveDocs, because the // mapping enum will skip the non-live docs: - postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(null, postingsEnumIn, true); + postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(null, postingsEnumIn); assert postingsEnumIn != null; postingsEnum.reset(postingsEnumIn); // set PayloadProcessor Index: lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java (working copy) @@ -278,13 +278,9 @@ @Override public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, - DocsAndPositionsEnum reuse, boolean needsOffsets) + DocsAndPositionsEnum reuse, int flags) throws IOException { - if (needsOffsets) { - return null; - } - assert fieldInfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; final SepTermState termState = (SepTermState) _termState; SepDocsAndPositionsEnum postingsEnum; Index: lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/BlockTermsReader.java (working copy) @@ -694,20 +694,14 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { if (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { // Positions were not indexed: return null; } - if (needsOffsets && - fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) < 0) { - // Offsets were not indexed: - return null; - } - decodeMetaData(); - return postingsReader.docsAndPositions(fieldInfo, state, liveDocs, reuse, needsOffsets); + return postingsReader.docsAndPositions(fieldInfo, state, liveDocs, reuse, flags); } @Override Index: lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java (working copy) @@ -213,7 +213,7 @@ @Override public DocsAndPositionsEnum docsAndPositions(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsAndPositionsEnum reuse, - boolean needsOffsets) throws IOException { + int flags) throws IOException { final PulsingTermState termState = (PulsingTermState) _termState; @@ -240,11 +240,11 @@ } else { if (reuse instanceof PulsingDocsAndPositionsEnum) { DocsAndPositionsEnum wrapped = wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, (DocsAndPositionsEnum) getOther(reuse), - needsOffsets); + flags); setOther(wrapped, reuse); // wrapped.other = reuse return wrapped; } else { - return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, reuse, needsOffsets); + return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, reuse, flags); } } } Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (working copy) @@ -208,19 +208,13 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) { // Positions were not indexed return null; } - if (needsOffsets && - indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) < 0) { - // Offsets were not indexed - return null; - } - SimpleTextDocsAndPositionsEnum docsAndPositionsEnum; if (reuse != null && reuse instanceof SimpleTextDocsAndPositionsEnum && ((SimpleTextDocsAndPositionsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) { docsAndPositionsEnum = (SimpleTextDocsAndPositionsEnum) reuse; Index: lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (working copy) @@ -365,14 +365,11 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { SimpleTVPostings postings = current.getValue(); if (postings.positions == null && postings.startOffsets == null) { return null; } - if (needsOffsets && (postings.startOffsets == null || postings.endOffsets == null)) { - return null; - } // TODO: reuse SimpleTVDocsAndPositionsEnum e = new SimpleTVDocsAndPositionsEnum(); e.reset(liveDocs, postings.positions, postings.startOffsets, postings.endOffsets); @@ -507,12 +504,20 @@ @Override public int startOffset() { - return startOffsets[nextPos-1]; + if (startOffsets == null) { + return -1; + } else { + return startOffsets[nextPos-1]; + } } @Override public int endOffset() { - return endOffsets[nextPos-1]; + if (endOffsets == null) { + return -1; + } else { + return endOffsets[nextPos-1]; + } } } } Index: lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (working copy) @@ -278,8 +278,8 @@ @Override public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, - DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { - return actualEnum.docsAndPositions(liveDocs, reuse, needsOffsets); + DocsAndPositionsEnum reuse, int flags) throws IOException { + return actualEnum.docsAndPositions(liveDocs, reuse, flags); } @Override Index: lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (working copy) @@ -120,7 +120,7 @@ final TermsEnum termsEnum = context.reader().terms(term.field()).iterator(null); termsEnum.seekExact(term.bytes(), state); - final DocsAndPositionsEnum postings = termsEnum.docsAndPositions(acceptDocs, null, false); + final DocsAndPositionsEnum postings = termsEnum.docsAndPositions(acceptDocs, null, DocsAndPositionsEnum.FLAG_PAYLOADS); if (postings != null) { return new TermSpans(postings, term); Index: lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java (working copy) @@ -226,7 +226,7 @@ return null; } termsEnum.seekExact(term.bytes(), termState); - postingsEnum = termsEnum.docsAndPositions(liveDocs, null, false); + postingsEnum = termsEnum.docsAndPositions(liveDocs, null, 0); if (postingsEnum == null) { // term does exist, but has no positions @@ -484,7 +484,7 @@ continue; } termsEnum.seekExact(term.bytes(), termState); - DocsAndPositionsEnum postings = termsEnum.docsAndPositions(liveDocs, null, false); + DocsAndPositionsEnum postings = termsEnum.docsAndPositions(liveDocs, null, 0); if (postings == null) { // term does exist, but has no positions throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")"); Index: lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java (working copy) @@ -264,7 +264,7 @@ return null; } te.seekExact(t.bytes(), state); - DocsAndPositionsEnum postingsEnum = te.docsAndPositions(liveDocs, null, false); + DocsAndPositionsEnum postingsEnum = te.docsAndPositions(liveDocs, null, 0); // PhraseQuery on a field that did not index // positions. Index: lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java =================================================================== --- lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (working copy) @@ -1047,7 +1047,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { throw new UnsupportedOperationException(); } Index: lucene/core/src/java/org/apache/lucene/index/CheckIndex.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (working copy) @@ -34,6 +34,7 @@ import org.apache.lucene.document.FieldType; // for javadocs import org.apache.lucene.index.DocValues.SortedSource; import org.apache.lucene.index.DocValues.Source; +import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; @@ -682,7 +683,6 @@ DocsEnum docs = null; DocsEnum docsAndFreqs = null; DocsAndPositionsEnum postings = null; - DocsAndPositionsEnum offsets = null; String lastField = null; final FieldsEnum fieldsEnum = fields.iterator(); @@ -699,11 +699,11 @@ // check that the field is in fieldinfos, and is indexed. // TODO: add a separate test to check this for different reader impls - FieldInfo fi = fieldInfos.fieldInfo(field); - if (fi == null) { + FieldInfo fieldInfo = fieldInfos.fieldInfo(field); + if (fieldInfo == null) { throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, no fieldInfos for: " + field); } - if (!fi.isIndexed()) { + if (!fieldInfo.isIndexed()) { throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, isIndexed == false for: " + field); } @@ -757,8 +757,7 @@ docs = termsEnum.docs(liveDocs, docs, false); docsAndFreqs = termsEnum.docs(liveDocs, docsAndFreqs, true); - postings = termsEnum.docsAndPositions(liveDocs, postings, false); - offsets = termsEnum.docsAndPositions(liveDocs, offsets, true); + postings = termsEnum.docsAndPositions(liveDocs, postings); if (hasOrd) { long ord = -1; @@ -782,29 +781,20 @@ final DocsEnum docsAndFreqs2; final boolean hasPositions; final boolean hasFreqs; - final boolean hasOffsets; - if (offsets != null) { - docs2 = postings = offsets; - docsAndFreqs2 = postings = offsets; - hasOffsets = true; - hasPositions = true; - hasFreqs = true; - } else if (postings != null) { + final boolean hasOffsets = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; + if (postings != null) { docs2 = postings; docsAndFreqs2 = postings; - hasOffsets = false; hasPositions = true; hasFreqs = true; } else if (docsAndFreqs != null) { docs2 = docsAndFreqs; docsAndFreqs2 = docsAndFreqs; - hasOffsets = false; hasPositions = false; hasFreqs = true; } else { docs2 = docs; docsAndFreqs2 = null; - hasOffsets = false; hasPositions = false; hasFreqs = false; } @@ -927,7 +917,7 @@ if (hasPositions) { for(int idx=0;idx<7;idx++) { final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8); - postings = termsEnum.docsAndPositions(liveDocs, postings, hasOffsets); + postings = termsEnum.docsAndPositions(liveDocs, postings); final int docID = postings.advance(skipDocID); if (docID == DocIdSetIterator.NO_MORE_DOCS) { break; @@ -1460,44 +1450,35 @@ BytesRef term = null; while ((term = termsEnum.next()) != null) { - final boolean hasPositions; - final boolean hasOffsets; + final boolean hasProx; final boolean hasFreqs; // TODO: really we need a reflection/query // API so we can just ask what was indexed // instead of "probing"... + // TODO: cleanup: + // Try offsets: - postings = termsEnum.docsAndPositions(null, postings, true); + postings = termsEnum.docsAndPositions(null, postings); if (postings == null) { - hasOffsets = false; - // Try only positions: - postings = termsEnum.docsAndPositions(null, postings, false); - if (postings == null) { - hasPositions = false; - // Try docIDs & freqs: - docs = termsEnum.docs(null, docs, true); - if (docs == null) { - // OK, only docIDs: - hasFreqs = false; - docs = termsEnum.docs(null, docs, false); - } else { + hasProx = false; + // Try docIDs & freqs: + docs = termsEnum.docs(null, docs, true); + if (docs == null) { + // OK, only docIDs: + hasFreqs = false; + docs = termsEnum.docs(null, docs, false); + } else { hasFreqs = true; - } - } else { - hasPositions = true; - hasFreqs = true; } } else { - hasOffsets = true; - // NOTE: may be a lie... but we accept -1 - hasPositions = true; + hasProx = true; hasFreqs = true; } final DocsEnum docs2; - if (hasPositions || hasOffsets) { + if (hasProx) { assert postings != null; docs2 = postings; } else { @@ -1510,23 +1491,18 @@ if (!postingsTermsEnum.seekExact(term, true)) { throw new RuntimeException("vector term=" + term + " field=" + field + " does not exist in postings; doc=" + j); } - postingsPostings = postingsTermsEnum.docsAndPositions(null, postingsPostings, true); + postingsPostings = postingsTermsEnum.docsAndPositions(null, postingsPostings); if (postingsPostings == null) { - // Term vectors were indexed w/ offsets but postings were not - postingsPostings = postingsTermsEnum.docsAndPositions(null, postingsPostings, false); - if (postingsPostings == null) { - postingsDocs = postingsTermsEnum.docs(null, postingsDocs, true); + // Term vectors were indexed w/ pos but postings were not + postingsDocs = postingsTermsEnum.docs(null, postingsDocs, true); + if (postingsDocs == null) { + postingsHasFreq = false; + postingsDocs = postingsTermsEnum.docs(null, postingsDocs, false); if (postingsDocs == null) { - postingsHasFreq = false; - postingsDocs = postingsTermsEnum.docs(null, postingsDocs, false); - if (postingsDocs == null) { - throw new RuntimeException("vector term=" + term + " field=" + field + " does not exist in postings; doc=" + j); - } - } else { - postingsHasFreq = true; + throw new RuntimeException("vector term=" + term + " field=" + field + " does not exist in postings; doc=" + j); } } else { - postingsHasFreq = true; + postingsHasFreq = true; } } else { postingsHasFreq = true; @@ -1555,7 +1531,7 @@ throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": freq=" + tf + " differs from postings freq=" + postingsDocs2.freq()); } - if (hasPositions || hasOffsets) { + if (hasProx) { for (int i = 0; i < tf; i++) { int pos = postings.nextPosition(); if (postingsPostings != null) { @@ -1565,33 +1541,31 @@ } } - if (hasOffsets) { - // Call the methods to at least make - // sure they don't throw exc: - final int startOffset = postings.startOffset(); - final int endOffset = postings.endOffset(); - // TODO: these are too anal...? - /* - if (endOffset < startOffset) { - throw new RuntimeException("vector startOffset=" + startOffset + " is > endOffset=" + endOffset); - } - if (startOffset < lastStartOffset) { - throw new RuntimeException("vector startOffset=" + startOffset + " is < prior startOffset=" + lastStartOffset); - } - lastStartOffset = startOffset; - */ + // Call the methods to at least make + // sure they don't throw exc: + final int startOffset = postings.startOffset(); + final int endOffset = postings.endOffset(); + // TODO: these are too anal...? + /* + if (endOffset < startOffset) { + throw new RuntimeException("vector startOffset=" + startOffset + " is > endOffset=" + endOffset); + } + if (startOffset < lastStartOffset) { + throw new RuntimeException("vector startOffset=" + startOffset + " is < prior startOffset=" + lastStartOffset); + } + lastStartOffset = startOffset; + */ - if (postingsPostings != null) { - final int postingsStartOffset = postingsPostings.startOffset(); + if (postingsPostings != null) { + final int postingsStartOffset = postingsPostings.startOffset(); - final int postingsEndOffset = postingsPostings.endOffset(); - if (startOffset != -1 && postingsStartOffset != -1 && startOffset != postingsStartOffset) { - throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": startOffset=" + startOffset + " differs from postings startOffset=" + postingsStartOffset); - } - if (endOffset != -1 && postingsEndOffset != -1 && endOffset != postingsEndOffset) { - throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": endOffset=" + endOffset + " differs from postings endOffset=" + postingsEndOffset); - } + final int postingsEndOffset = postingsPostings.endOffset(); + if (startOffset != -1 && postingsStartOffset != -1 && startOffset != postingsStartOffset) { + throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": startOffset=" + startOffset + " differs from postings startOffset=" + postingsStartOffset); } + if (endOffset != -1 && postingsEndOffset != -1 && endOffset != postingsEndOffset) { + throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": endOffset=" + endOffset + " differs from postings endOffset=" + postingsEndOffset); + } } } } Index: lucene/core/src/java/org/apache/lucene/index/MultiFields.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MultiFields.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/index/MultiFields.java (working copy) @@ -139,15 +139,27 @@ /** Returns {@link DocsAndPositionsEnum} for the specified * field & term. This may return null if the term does - * not exist or positions were not indexed. */ - public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, boolean needsOffsets) throws IOException { + * not exist or positions were not indexed. + * @see #getTermPositionsEnum(IndexReader, Bits, String, BytesRef, int) */ + public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException { + return getTermPositionsEnum(r, liveDocs, field, term, DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS); + } + + /** Returns {@link DocsAndPositionsEnum} for the specified + * field & term, with control over whether offsets and payloads are + * required. Some codecs may be able to optimize + * their implementation when offsets and/or payloads are not + * required. This will return null if the term does not + * exist or positions were not indexed. See {@link + * TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}. */ + public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, int flags) throws IOException { assert field != null; assert term != null; final Terms terms = getTerms(r, field); if (terms != null) { final TermsEnum termsEnum = terms.iterator(null); if (termsEnum.seekExact(term, true)) { - return termsEnum.docsAndPositions(liveDocs, null, needsOffsets); + return termsEnum.docsAndPositions(liveDocs, null, flags); } } return null; Index: lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java (working copy) @@ -171,8 +171,8 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits bits, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { - return tenum.docsAndPositions(bits, reuse, needsOffsets); + public DocsAndPositionsEnum docsAndPositions(Bits bits, DocsAndPositionsEnum reuse, int flags) throws IOException { + return tenum.docsAndPositions(bits, reuse, flags); } /** This enum does not support seeking! Index: lucene/core/src/java/org/apache/lucene/index/AtomicReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/AtomicReader.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/index/AtomicReader.java (working copy) @@ -133,12 +133,24 @@ } return null; } + + /** Returns {@link DocsAndPositionsEnum} for the specified + * field & term. This will return null if either the + * field or term does not exist or positions weren't indexed. + * @see #termPositionsEnum(Bits, String, BytesRef, int) */ + public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException { + return termPositionsEnum(liveDocs, field, term, DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS); + } + /** Returns {@link DocsAndPositionsEnum} for the specified - * field & term. This may return null, if either the - * field or term does not exist, or needsOffsets is - * true but offsets were not indexed for this field. */ - public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, boolean needsOffsets) throws IOException { + * field & term, with control over whether offsets and payloads are + * required. Some codecs may be able to optimize their + * implementation when offsets and/or payloads are not required. + * This will return null, if either the field or term + * does not exist or positions weren't indexed. See + * {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}. */ + public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, int flags) throws IOException { assert field != null; assert term != null; final Fields fields = fields(); @@ -147,7 +159,7 @@ if (terms != null) { final TermsEnum termsEnum = terms.iterator(null); if (termsEnum.seekExact(term, true)) { - return termsEnum.docsAndPositions(liveDocs, null, needsOffsets); + return termsEnum.docsAndPositions(liveDocs, null, flags); } } } Index: lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (working copy) @@ -186,8 +186,8 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { - return in.docsAndPositions(liveDocs, reuse, needsOffsets); + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + return in.docsAndPositions(liveDocs, reuse, flags); } @Override Index: lucene/core/src/java/org/apache/lucene/index/TermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/TermsEnum.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/index/TermsEnum.java (working copy) @@ -138,6 +138,8 @@ * deleted documents into account. */ public abstract long totalTermFreq() throws IOException; + // TODO: cutover to flags / make needsFreqs "a hint" / add + // default w/ needsFreqs=true /** Get {@link DocsEnum} for the current term. Do not * call this when the enum is unpositioned. This method * may return null (if needsFreqs is true but freqs were @@ -152,15 +154,33 @@ public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException; /** Get {@link DocsAndPositionsEnum} for the current term. - * Do not call this when the enum is unpositioned. - * This method will only return null if needsOffsets is - * true but offsets were not indexed. + * Do not call this when the enum is unpositioned. This + * method will return null if positions were not + * indexed. + * * @param liveDocs unset bits are documents that should not * be returned * @param reuse pass a prior DocsAndPositionsEnum for possible reuse - * @param needsOffsets true if offsets are required */ - public abstract DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException; + * @see #docsAndPositions(Bits, DocsAndPositionsEnum, int) */ + public final DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { + return docsAndPositions(liveDocs, reuse, DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS); + } + /** Get {@link DocsAndPositionsEnum} for the current term, + * with control over whether offsets and payloads are + * required. Some codecs may be able to optimize their + * implementation when offsets and/or payloads are not required. + * Do not call this when the enum is unpositioned. This + * will return null if positions were not indexed. + + * @param liveDocs unset bits are documents that should not + * be returned + * @param reuse pass a prior DocsAndPositionsEnum for possible reuse + * @param flags specifies which optional per-position values you + * require; see {@link DocsAndPositionsEnum#FLAG_OFFSETS} and + * {@link DocsAndPositionsEnum#FLAG_PAYLOADS}. */ + public abstract DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException; + /** * Expert: Returns the TermsEnums internal state to position the TermsEnum * without re-seeking the term dictionary. @@ -225,7 +245,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) { throw new IllegalStateException("this method should never be called"); } Index: lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java (working copy) @@ -673,8 +673,8 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { - return termsEnum.docsAndPositions(liveDocs, reuse, needsOffsets); + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + return termsEnum.docsAndPositions(liveDocs, reuse, flags); } @Override Index: lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java (working copy) @@ -415,7 +415,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { MultiDocsAndPositionsEnum docsAndPositionsEnum; // Can only reuse if incoming enum is also a MultiDocsAndPositionsEnum if (reuse != null && reuse instanceof MultiDocsAndPositionsEnum) { @@ -466,7 +466,7 @@ } assert entry.index < docsAndPositionsEnum.subDocsAndPositionsEnum.length: entry.index + " vs " + docsAndPositionsEnum.subDocsAndPositionsEnum.length + "; " + subs.length; - final DocsAndPositionsEnum subPostings = entry.terms.docsAndPositions(b, docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index], needsOffsets); + final DocsAndPositionsEnum subPostings = entry.terms.docsAndPositions(b, docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index], flags); if (subPostings != null) { docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index] = subPostings; Index: lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java (revision 1367102) +++ lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java (working copy) @@ -23,7 +23,14 @@ /** Also iterates through positions. */ public abstract class DocsAndPositionsEnum extends DocsEnum { + /** Flag to pass to {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)} + * if you require offsets in the returned enum. */ + public static final int FLAG_OFFSETS = 0x1; + /** Flag to pass to {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)} + * if you require payloads in the returned enum. */ + public static final int FLAG_PAYLOADS = 0x2; + /** Returns the next position. You should only call this * up to {@link DocsEnum#freq()} times else * the behavior is not defined. If positions were not Index: lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/search/TestTermVectors.java (working copy) @@ -134,19 +134,19 @@ assertNotNull(terms); TermsEnum termsEnum = terms.iterator(null); assertEquals("content", termsEnum.next().utf8ToString()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, false); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); assertEquals(expectedPositions[0], dpEnum.nextPosition()); assertEquals("here", termsEnum.next().utf8ToString()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, false); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); assertEquals(expectedPositions[1], dpEnum.nextPosition()); assertEquals("some", termsEnum.next().utf8ToString()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, false); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); assertEquals(expectedPositions[2], dpEnum.nextPosition()); @@ -176,7 +176,7 @@ if (shouldBePosVector || shouldBeOffVector) { while(true) { - dpEnum = termsEnum.docsAndPositions(null, dpEnum, shouldBeOffVector); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertNotNull(dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -426,7 +426,7 @@ assertNotNull(termsEnum.next()); assertEquals("one", termsEnum.term().utf8ToString()); assertEquals(5, termsEnum.totalTermFreq()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertNotNull(dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(5, dpEnum.freq()); @@ -434,7 +434,7 @@ assertEquals(i, dpEnum.nextPosition()); } - dpEnum = termsEnum.docsAndPositions(null, dpEnum, true); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertNotNull(dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(5, dpEnum.freq()); Index: lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (working copy) @@ -103,8 +103,7 @@ DocsAndPositionsEnum pos = MultiFields.getTermPositionsEnum(searcher.getIndexReader(), MultiFields.getLiveDocs(searcher.getIndexReader()), "field", - new BytesRef("1"), - false); + new BytesRef("1")); pos.nextDoc(); // first token should be at position 0 assertEquals(0, pos.nextPosition()); @@ -112,8 +111,7 @@ pos = MultiFields.getTermPositionsEnum(searcher.getIndexReader(), MultiFields.getLiveDocs(searcher.getIndexReader()), "field", - new BytesRef("2"), - false); + new BytesRef("2")); pos.nextDoc(); // second token should be at position 2 assertEquals(2, pos.nextPosition()); @@ -216,8 +214,7 @@ DocsAndPositionsEnum tp = r.termPositionsEnum(r.getLiveDocs(), "content", - new BytesRef("a"), - false); + new BytesRef("a")); int count = 0; assertTrue(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java (working copy) @@ -329,17 +329,17 @@ assertEquals(info, term, rightTermsEnum.next()); assertTermStats(leftTermsEnum, rightTermsEnum); if (deep) { - assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, false), - rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, false)); - assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, false), - rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, false)); + assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions), + rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions)); + assertDocsAndPositionsEnum(leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions), + rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions)); assertPositionsSkipping(leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions, false), - rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions, false)); + leftPositions = leftTermsEnum.docsAndPositions(null, leftPositions), + rightPositions = rightTermsEnum.docsAndPositions(null, rightPositions)); assertPositionsSkipping(leftTermsEnum.docFreq(), - leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions, false), - rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions, false)); + leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions), + rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions)); // with freqs: assertDocsEnum(leftDocs = leftTermsEnum.docs(null, leftDocs, true), Index: lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestPostingsOffsets.java (working copy) @@ -83,7 +83,7 @@ IndexReader r = w.getReader(); w.close(); - DocsAndPositionsEnum dp = MultiFields.getTermPositionsEnum(r, null, "content", new BytesRef("a"), true); + DocsAndPositionsEnum dp = MultiFields.getTermPositionsEnum(r, null, "content", new BytesRef("a")); assertNotNull(dp); assertEquals(0, dp.nextDoc()); assertEquals(2, dp.freq()); @@ -95,7 +95,7 @@ assertEquals(17, dp.endOffset()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, dp.nextDoc()); - dp = MultiFields.getTermPositionsEnum(r, null, "content", new BytesRef("b"), true); + dp = MultiFields.getTermPositionsEnum(r, null, "content", new BytesRef("b")); assertNotNull(dp); assertEquals(0, dp.nextDoc()); assertEquals(1, dp.freq()); @@ -104,7 +104,7 @@ assertEquals(9, dp.endOffset()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, dp.nextDoc()); - dp = MultiFields.getTermPositionsEnum(r, null, "content", new BytesRef("c"), true); + dp = MultiFields.getTermPositionsEnum(r, null, "content", new BytesRef("c")); assertNotNull(dp); assertEquals(0, dp.nextDoc()); assertEquals(1, dp.freq()); @@ -155,7 +155,7 @@ String terms[] = { "one", "two", "three", "four", "five", "six", "seven", "eight", "nine", "ten", "hundred" }; for (String term : terms) { - DocsAndPositionsEnum dp = MultiFields.getTermPositionsEnum(reader, null, "numbers", new BytesRef(term), true); + DocsAndPositionsEnum dp = MultiFields.getTermPositionsEnum(reader, null, "numbers", new BytesRef(term)); int doc; while((doc = dp.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { String storedNumbers = reader.document(doc).get("numbers"); @@ -183,7 +183,7 @@ for (int j = 0; j < numSkippingTests; j++) { int num = _TestUtil.nextInt(random(), 100, Math.min(numDocs-1, 999)); - DocsAndPositionsEnum dp = MultiFields.getTermPositionsEnum(reader, null, "numbers", new BytesRef("hundred"), true); + DocsAndPositionsEnum dp = MultiFields.getTermPositionsEnum(reader, null, "numbers", new BytesRef("hundred")); int doc = dp.advance(num); assertEquals(num, doc); int freq = dp.freq(); @@ -312,7 +312,8 @@ assertEquals(expected.size(), docs.freq()); } - docsAndPositions = termsEnum.docsAndPositions(null, docsAndPositions, false); + // explicitly exclude offsets here + docsAndPositions = termsEnum.docsAndPositions(null, docsAndPositions, DocsAndPositionsEnum.FLAG_PAYLOADS); assertNotNull(docsAndPositions); //System.out.println(" doc/freq/pos"); while((doc = docsAndPositions.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { @@ -327,7 +328,7 @@ } } - docsAndPositionsAndOffsets = termsEnum.docsAndPositions(null, docsAndPositions, true); + docsAndPositionsAndOffsets = termsEnum.docsAndPositions(null, docsAndPositions); assertNotNull(docsAndPositionsAndOffsets); //System.out.println(" doc/freq/pos/offs"); while((doc = docsAndPositionsAndOffsets.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { Index: lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexableField.java (working copy) @@ -264,14 +264,14 @@ TermsEnum termsEnum = tfv.iterator(null); assertEquals(new BytesRef(""+counter), termsEnum.next()); assertEquals(1, termsEnum.totalTermFreq()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); assertEquals(1, dpEnum.nextPosition()); assertEquals(new BytesRef("text"), termsEnum.next()); assertEquals(1, termsEnum.totalTermFreq()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, false); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); assertEquals(0, dpEnum.nextPosition()); Index: lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestFilterAtomicReader.java (working copy) @@ -89,8 +89,8 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException { - return new TestPositions(super.docsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsAndPositionsEnum) reuse).in, needsOffsets)); + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException { + return new TestPositions(super.docsAndPositions(liveDocs, reuse == null ? null : ((FilterDocsAndPositionsEnum) reuse).in, flags)); } } @@ -164,8 +164,7 @@ assertEquals(TermsEnum.SeekStatus.FOUND, terms.seekCeil(new BytesRef("one"))); - DocsAndPositionsEnum positions = terms.docsAndPositions(MultiFields.getLiveDocs(reader), - null, false); + DocsAndPositionsEnum positions = terms.docsAndPositions(MultiFields.getLiveDocs(reader), null); while (positions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { assertTrue((positions.docID() % 2) == 1); } Index: lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (working copy) @@ -86,8 +86,7 @@ counter = 0; DocsAndPositionsEnum tp = reader.termPositionsEnum(reader.getLiveDocs(), term.field(), - new BytesRef(term.text()), - false); + new BytesRef(term.text())); checkSkipTo(tp, 14, 185); // no skips checkSkipTo(tp, 17, 190); // one skip on level 0 Index: lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestDocumentWriter.java (working copy) @@ -129,7 +129,7 @@ SegmentReader reader = new SegmentReader(info, DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random())); DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), - "repeated", new BytesRef("repeated"), false); + "repeated", new BytesRef("repeated")); assertTrue(termPositions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); int freq = termPositions.freq(); assertEquals(2, freq); @@ -200,7 +200,7 @@ writer.close(); SegmentReader reader = new SegmentReader(info, DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random())); - DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, reader.getLiveDocs(), "f1", new BytesRef("a"), false); + DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, reader.getLiveDocs(), "f1", new BytesRef("a")); assertTrue(termPositions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); int freq = termPositions.freq(); assertEquals(3, freq); @@ -243,18 +243,18 @@ writer.close(); SegmentReader reader = new SegmentReader(info, DirectoryReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random())); - DocsAndPositionsEnum termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term1"), false); + DocsAndPositionsEnum termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term1")); assertTrue(termPositions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, termPositions.freq()); assertEquals(0, termPositions.nextPosition()); - termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term2"), false); + termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term2")); assertTrue(termPositions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(2, termPositions.freq()); assertEquals(1, termPositions.nextPosition()); assertEquals(3, termPositions.nextPosition()); - termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term3"), false); + termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term3")); assertTrue(termPositions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, termPositions.freq()); assertEquals(2, termPositions.nextPosition()); Index: lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (working copy) @@ -69,7 +69,7 @@ // Token "" occurred once assertEquals(1, termsEnum.totalTermFreq()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); assertEquals(8, dpEnum.startOffset()); @@ -78,7 +78,7 @@ // Token "abcd" occurred three times assertEquals(new BytesRef("abcd"), termsEnum.next()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, true); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertEquals(3, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -118,7 +118,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -153,7 +153,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -192,7 +192,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -228,7 +228,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(2, termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -265,7 +265,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); @@ -273,14 +273,14 @@ assertEquals(4, dpEnum.endOffset()); assertNotNull(termsEnum.next()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, true); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); assertEquals(11, dpEnum.startOffset()); assertEquals(17, dpEnum.endOffset()); assertNotNull(termsEnum.next()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, true); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); assertEquals(18, dpEnum.startOffset()); @@ -310,7 +310,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(1, (int) termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -319,7 +319,7 @@ assertEquals(7, dpEnum.endOffset()); assertNotNull(termsEnum.next()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, true); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); assertEquals(8, dpEnum.startOffset()); @@ -353,7 +353,7 @@ IndexReader r = DirectoryReader.open(dir); TermsEnum termsEnum = r.getTermVectors(0).terms("field").iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, true); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertEquals(1, (int) termsEnum.totalTermFreq()); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -362,7 +362,7 @@ assertEquals(4, dpEnum.endOffset()); assertNotNull(termsEnum.next()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, true); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); dpEnum.nextPosition(); assertEquals(6, dpEnum.startOffset()); Index: lucene/core/src/test/org/apache/lucene/index/TestDoc.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDoc.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestDoc.java (working copy) @@ -247,7 +247,7 @@ out.print(" term=" + field + ":" + tis.term()); out.println(" DF=" + tis.docFreq()); - DocsAndPositionsEnum positions = tis.docsAndPositions(reader.getLiveDocs(), null, false); + DocsAndPositionsEnum positions = tis.docsAndPositions(reader.getLiveDocs(), null); while (positions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { out.print(" doc=" + positions.docID()); Index: lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestStressIndexing2.java (working copy) @@ -408,7 +408,7 @@ BytesRef term2; while((term2 = termsEnum3.next()) != null) { System.out.println(" " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq()); - dpEnum = termsEnum3.docsAndPositions(null, dpEnum, false); + dpEnum = termsEnum3.docsAndPositions(null, dpEnum); if (dpEnum != null) { assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); final int freq = dpEnum.freq(); @@ -442,7 +442,7 @@ BytesRef term2; while((term2 = termsEnum3.next()) != null) { System.out.println(" " + term2.utf8ToString() + ": freq=" + termsEnum3.totalTermFreq()); - dpEnum = termsEnum3.docsAndPositions(null, dpEnum, false); + dpEnum = termsEnum3.docsAndPositions(null, dpEnum); if (dpEnum != null) { assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); final int freq = dpEnum.freq(); @@ -632,8 +632,8 @@ assertEquals(termsEnum1.totalTermFreq(), termsEnum2.totalTermFreq()); - dpEnum1 = termsEnum1.docsAndPositions(null, dpEnum1, false); - dpEnum2 = termsEnum2.docsAndPositions(null, dpEnum2, false); + dpEnum1 = termsEnum1.docsAndPositions(null, dpEnum1); + dpEnum2 = termsEnum2.docsAndPositions(null, dpEnum2); if (dpEnum1 != null) { assertNotNull(dpEnum2); int docID1 = dpEnum1.nextDoc(); Index: lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestLongPostings.java (working copy) @@ -170,7 +170,7 @@ System.out.println("\nTEST: iter=" + iter + " doS1=" + doS1); } - final DocsAndPositionsEnum postings = MultiFields.getTermPositionsEnum(r, null, "field", new BytesRef(term), false); + final DocsAndPositionsEnum postings = MultiFields.getTermPositionsEnum(r, null, "field", new BytesRef(term)); int docID = -1; while(docID < DocIdSetIterator.NO_MORE_DOCS) { Index: lucene/core/src/test/org/apache/lucene/index/TestCodecs.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestCodecs.java (working copy) @@ -474,7 +474,7 @@ if (field.omitTF) { this.verifyDocs(term.docs, term.positions, _TestUtil.docs(random(), termsEnum, null, null, false), false); } else { - this.verifyDocs(term.docs, term.positions, termsEnum.docsAndPositions(null, null, false), true); + this.verifyDocs(term.docs, term.positions, termsEnum.docsAndPositions(null, null), true); } // Test random seek by ord: @@ -494,7 +494,7 @@ if (field.omitTF) { this.verifyDocs(term.docs, term.positions, _TestUtil.docs(random(), termsEnum, null, null, false), false); } else { - this.verifyDocs(term.docs, term.positions, termsEnum.docsAndPositions(null, null, false), true); + this.verifyDocs(term.docs, term.positions, termsEnum.docsAndPositions(null, null), true); } } @@ -546,7 +546,7 @@ final DocsEnum docsAndFreqs; final DocsAndPositionsEnum postings; if (!field.omitTF) { - postings = termsEnum.docsAndPositions(null, null, false); + postings = termsEnum.docsAndPositions(null, null); if (postings != null) { docs = docsAndFreqs = postings; } else { Index: lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestPayloadProcessorProvider.java (working copy) @@ -160,7 +160,7 @@ IndexReader reader = DirectoryReader.open(dir); try { int numPayloads = 0; - DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text, false); + DocsAndPositionsEnum tpe = MultiFields.getTermPositionsEnum(reader, null, field, text); while (tpe.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { tpe.nextPosition(); if (tpe.hasPayload()) { Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -940,14 +940,14 @@ Terms tpv = r.getTermVectors(0).terms("field"); TermsEnum termsEnum = tpv.iterator(null); assertNotNull(termsEnum.next()); - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null); assertNotNull(dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); assertEquals(100, dpEnum.nextPosition()); assertNotNull(termsEnum.next()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, false); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertNotNull(dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, dpEnum.freq()); @@ -1588,7 +1588,7 @@ // Make sure position is still incremented when // massive term is skipped: - DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader, null, "content", new BytesRef("another"), false); + DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader, null, "content", new BytesRef("another")); assertEquals(0, tps.nextDoc()); assertEquals(1, tps.freq()); assertEquals(3, tps.nextPosition()); Index: lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestSegmentReader.java (working copy) @@ -149,8 +149,7 @@ DocsAndPositionsEnum positions = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), DocHelper.TEXT_FIELD_1_KEY, - new BytesRef("field"), - false); + new BytesRef("field")); // NOTE: prior rev of this test was failing to first // call next here: assertTrue(positions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/core/src/test/org/apache/lucene/index/TestPayloads.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestPayloads.java (working copy) @@ -188,8 +188,7 @@ tps[i] = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), terms[i].field(), - new BytesRef(terms[i].text()), - false); + new BytesRef(terms[i].text())); } while (tps[0].nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { @@ -226,8 +225,7 @@ DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), terms[0].field(), - new BytesRef(terms[0].text()), - false); + new BytesRef(terms[0].text())); tp.nextDoc(); tp.nextPosition(); // NOTE: prior rev of this test was failing to first @@ -255,8 +253,7 @@ tp = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), terms[1].field(), - new BytesRef(terms[1].text()), - false); + new BytesRef(terms[1].text())); tp.nextDoc(); tp.nextPosition(); assertEquals("Wrong payload length.", 1, tp.getPayload().length); @@ -299,8 +296,7 @@ tp = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), fieldName, - new BytesRef(singleTerm), - false); + new BytesRef(singleTerm)); tp.nextDoc(); tp.nextPosition(); @@ -501,7 +497,7 @@ DocsAndPositionsEnum tp = null; while (terms.next() != null) { String termText = terms.term().utf8ToString(); - tp = terms.docsAndPositions(liveDocs, tp, false); + tp = terms.docsAndPositions(liveDocs, tp); while(tp.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { int freq = tp.freq(); for (int i = 0; i < freq; i++) { Index: lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (working copy) @@ -631,8 +631,8 @@ while(enum1.next() != null) { assertEquals("Different terms", enum1.term(), enum2.next()); - DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null, false); - DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null, false); + DocsAndPositionsEnum tp1 = enum1.docsAndPositions(liveDocs, null); + DocsAndPositionsEnum tp2 = enum2.docsAndPositions(liveDocs, null); while(tp1.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { assertTrue(tp2.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (working copy) @@ -157,8 +157,7 @@ DocsAndPositionsEnum tp = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), this.field, - new BytesRef("b"), - false); + new BytesRef("b")); for (int i = 0; i < 10; i++) { tp.nextDoc(); @@ -169,8 +168,7 @@ tp = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), this.field, - new BytesRef("a"), - false); + new BytesRef("a")); for (int i = 0; i < 10; i++) { tp.nextDoc(); Index: lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (working copy) @@ -252,7 +252,7 @@ //System.out.println("Term: " + term); assertEquals(testTerms[i], term); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, false); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertNotNull(dpEnum); int doc = dpEnum.docID(); assertTrue(doc == -1 || doc == DocIdSetIterator.NO_MORE_DOCS); @@ -263,7 +263,7 @@ } assertEquals(DocIdSetIterator.NO_MORE_DOCS, dpEnum.nextDoc()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, true); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); doc = dpEnum.docID(); assertTrue(doc == -1 || doc == DocIdSetIterator.NO_MORE_DOCS); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -306,7 +306,7 @@ String term = text.utf8ToString(); assertEquals(testTerms[i], term); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, false); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertNotNull(dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(dpEnum.freq(), positions[i].length); @@ -315,7 +315,7 @@ } assertEquals(DocIdSetIterator.NO_MORE_DOCS, dpEnum.nextDoc()); - dpEnum = termsEnum.docsAndPositions(null, dpEnum, true); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); assertTrue(dpEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertNotNull(dpEnum); assertEquals(dpEnum.freq(), positions[i].length); Index: lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestDocsAndPositions.java (working copy) @@ -92,7 +92,7 @@ public DocsAndPositionsEnum getDocsAndPositions(AtomicReader reader, BytesRef bytes, Bits liveDocs) throws IOException { - return reader.termPositionsEnum(null, fieldName, bytes, false); + return reader.termPositionsEnum(null, fieldName, bytes); } /** @@ -352,7 +352,7 @@ writer.addDocument(doc); DirectoryReader reader = writer.getReader(); AtomicReader r = getOnlySegmentReader(reader); - DocsAndPositionsEnum disi = r.termPositionsEnum(null, "foo", new BytesRef("bar"), false); + DocsAndPositionsEnum disi = r.termPositionsEnum(null, "foo", new BytesRef("bar")); int docid = disi.docID(); assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -360,7 +360,7 @@ // now reuse and check again TermsEnum te = r.terms("foo").iterator(null); assertTrue(te.seekExact(new BytesRef("bar"), true)); - disi = te.docsAndPositions(null, disi, false); + disi = te.docsAndPositions(null, disi); docid = disi.docID(); assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestOmitPositions.java (working copy) @@ -51,7 +51,7 @@ IndexReader reader = w.getReader(); w.close(); - assertNull(MultiFields.getTermPositionsEnum(reader, null, "foo", new BytesRef("test"), false)); + assertNull(MultiFields.getTermPositionsEnum(reader, null, "foo", new BytesRef("test"))); DocsEnum de = _TestUtil.docs(random(), reader, "foo", new BytesRef("test"), null, null, true); while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { Index: lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/index/TestPostingsFormat.java (working copy) @@ -536,7 +536,7 @@ prevDocsEnum = threadState.reuseDocsAndPositionsEnum; } - threadState.reuseDocsAndPositionsEnum = termsEnum.docsAndPositions(liveDocs, (DocsAndPositionsEnum) prevDocsEnum, false); + threadState.reuseDocsAndPositionsEnum = termsEnum.docsAndPositions(liveDocs, (DocsAndPositionsEnum) prevDocsEnum); docsEnum = threadState.reuseDocsAndPositionsEnum; docsAndPositionsEnum = threadState.reuseDocsAndPositionsEnum; } else { @@ -557,7 +557,7 @@ if (options.contains(Option.REUSE_ENUMS) && random().nextInt(10) < 9) { prevDocsEnum = threadState.reuseDocsAndPositionsEnum; } - threadState.reuseDocsAndPositionsEnum = termsEnum.docsAndPositions(liveDocs, (DocsAndPositionsEnum) prevDocsEnum, doCheckOffsets); + threadState.reuseDocsAndPositionsEnum = termsEnum.docsAndPositions(liveDocs, (DocsAndPositionsEnum) prevDocsEnum); docsEnum = threadState.reuseDocsAndPositionsEnum; docsAndPositionsEnum = threadState.reuseDocsAndPositionsEnum; } Index: lucene/core/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (working copy) @@ -75,8 +75,7 @@ DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), "preanalyzed", - new BytesRef("term1"), - false); + new BytesRef("term1")); assertTrue(termPositions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, termPositions.freq()); assertEquals(0, termPositions.nextPosition()); @@ -84,8 +83,7 @@ termPositions = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), "preanalyzed", - new BytesRef("term2"), - false); + new BytesRef("term2")); assertTrue(termPositions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(2, termPositions.freq()); assertEquals(1, termPositions.nextPosition()); @@ -94,8 +92,7 @@ termPositions = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), "preanalyzed", - new BytesRef("term3"), - false); + new BytesRef("term3")); assertTrue(termPositions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, termPositions.freq()); assertEquals(2, termPositions.nextPosition()); Index: lucene/core/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java =================================================================== --- lucene/core/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java (revision 1367102) +++ lucene/core/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java (working copy) @@ -35,7 +35,6 @@ import org.apache.lucene.index.TermsEnum; import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -71,7 +70,7 @@ DocsAndPositionsEnum posReuse = null; te = segment.terms("foo").iterator(null); while (te.next() != null) { - posReuse = te.docsAndPositions(null, posReuse, false); + posReuse = te.docsAndPositions(null, posReuse); allEnums.put(posReuse, true); } @@ -113,7 +112,7 @@ DocsAndPositionsEnum posReuse = null; te = segment.terms("foo").iterator(null); while (te.next() != null) { - posReuse = te.docsAndPositions(null, posReuse, false); + posReuse = te.docsAndPositions(null, posReuse); allEnums.put(posReuse, true); } Index: lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java =================================================================== --- lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (revision 1367102) +++ lucene/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (working copy) @@ -225,7 +225,7 @@ MemoryIndex memory = new MemoryIndex(true); memory.addField("foo", "bar", analyzer); AtomicReader reader = (AtomicReader) memory.createSearcher().getIndexReader(); - DocsAndPositionsEnum disi = reader.termPositionsEnum(null, "foo", new BytesRef("bar"), false); + DocsAndPositionsEnum disi = reader.termPositionsEnum(null, "foo", new BytesRef("bar")); int docid = disi.docID(); assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -236,7 +236,7 @@ // now reuse and check again TermsEnum te = reader.terms("foo").iterator(null); assertTrue(te.seekExact(new BytesRef("bar"), true)); - disi = te.docsAndPositions(null, disi, false); + disi = te.docsAndPositions(null, disi); docid = disi.docID(); assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java =================================================================== --- lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 1367102) +++ lucene/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy) @@ -880,10 +880,7 @@ } @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) { - if (needsOffsets) { - return null; - } + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) { if (reuse == null || !(reuse instanceof MemoryDocsAndPositionsEnum)) { reuse = new MemoryDocsAndPositionsEnum(); } Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestClassicAnalyzer.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestClassicAnalyzer.java (revision 1367102) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestClassicAnalyzer.java (working copy) @@ -285,8 +285,7 @@ DocsAndPositionsEnum tps = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), "content", - new BytesRef("another"), - false); + new BytesRef("another")); assertTrue(tps.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(1, tps.freq()); assertEquals(3, tps.nextPosition()); Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (revision 1367102) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/sinks/TestTeeSinkTokenFilter.java (working copy) @@ -111,7 +111,7 @@ TermsEnum termsEnum = vector.iterator(null); termsEnum.next(); assertEquals(2, termsEnum.totalTermFreq()); - DocsAndPositionsEnum positions = termsEnum.docsAndPositions(null, null, true); + DocsAndPositionsEnum positions = termsEnum.docsAndPositions(null, null); assertTrue(positions.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); assertEquals(2, positions.freq()); positions.nextPosition(); Index: lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (revision 1367102) +++ lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenStreamFromTermPositionVector.java (working copy) @@ -60,18 +60,14 @@ BytesRef text; DocsAndPositionsEnum dpEnum = null; while((text = termsEnum.next()) != null) { - dpEnum = termsEnum.docsAndPositions(null, dpEnum, true); - final boolean hasOffsets; - if (dpEnum == null) { - hasOffsets = false; - dpEnum = termsEnum.docsAndPositions(null, dpEnum, false); - } else { - hasOffsets = true; - } + dpEnum = termsEnum.docsAndPositions(null, dpEnum); + assert dpEnum != null; // presumably checked by TokenSources.hasPositions earlier + boolean hasOffsets = true; dpEnum.nextDoc(); final int freq = dpEnum.freq(); for (int j = 0; j < freq; j++) { int pos = dpEnum.nextPosition(); + hasOffsets &= dpEnum.startOffset() >= 0; Token token; if (hasOffsets) { token = new Token(text.utf8ToString(), Index: lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java =================================================================== --- lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (revision 1367102) +++ lucene/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (working copy) @@ -127,7 +127,7 @@ private static boolean hasPositions(Terms vector) throws IOException { final TermsEnum termsEnum = vector.iterator(null); if (termsEnum.next() != null) { - DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, false); + DocsAndPositionsEnum dpEnum = termsEnum.docsAndPositions(null, null, DocsAndPositionsEnum.FLAG_PAYLOADS); if (dpEnum != null) { int doc = dpEnum.nextDoc(); assert doc >= 0 && doc != DocIdSetIterator.NO_MORE_DOCS; @@ -222,18 +222,21 @@ DocsAndPositionsEnum dpEnum = null; while ((text = termsEnum.next()) != null) { - dpEnum = termsEnum.docsAndPositions(null, dpEnum, true); + dpEnum = termsEnum.docsAndPositions(null, dpEnum); if (dpEnum == null) { throw new IllegalArgumentException( "Required TermVector Offset information was not found"); } - final String term = text.utf8ToString(); dpEnum.nextDoc(); final int freq = dpEnum.freq(); for(int posUpto=0;posUpto positionsNL = null; NamedList theOffsets = null; - if (useOffsets) { - theOffsets = new NamedList(); - termInfo.add("offsets", theOffsets); - } - NamedList positionsNL = null; - - if (usePositions || theOffsets != null) { + if (usePositions) { for (int i = 0; i < freq; i++) { final int pos = dpEnum.nextPosition(); if (usePositions && pos >= 0) { @@ -361,6 +352,15 @@ positionsNL.add("position", pos); } + if (useOffsets && theOffsets == null) { + if (dpEnum.startOffset() == -1) { + useOffsets = false; + } else { + theOffsets = new NamedList(); + termInfo.add("offsets", theOffsets); + } + } + if (theOffsets != null) { theOffsets.add("start", dpEnum.startOffset()); theOffsets.add("end", dpEnum.endOffset());