Index: modules/facet/src/test/org/apache/lucene/facet/FacetTestBase.java =================================================================== --- modules/facet/src/test/org/apache/lucene/facet/FacetTestBase.java (revision 1209915) +++ modules/facet/src/test/org/apache/lucene/facet/FacetTestBase.java (working copy) @@ -285,7 +285,7 @@ TermsEnum te = terms.iterator(null); DocsEnum de = null; while (te.next() != null) { - de = te.docs(liveDocs, de); + de = te.docs(liveDocs, de, false); int cnt = 0; while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { cnt++; Index: modules/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java =================================================================== --- modules/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java (revision 1209915) +++ modules/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java (working copy) @@ -91,7 +91,7 @@ // Obtain facets results and hand-test them assertCorrectResults(facetsCollector); - DocsEnum td = MultiFields.getTermDocsEnum(ir, MultiFields.getLiveDocs(ir), "$facets", new BytesRef("$fulltree$")); + DocsEnum td = MultiFields.getTermDocsEnum(ir, MultiFields.getLiveDocs(ir), "$facets", new BytesRef("$fulltree$"), false); assertTrue(td.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); tr.close(); @@ -188,7 +188,7 @@ } private void assertPostingListExists(String field, String text, IndexReader ir) throws IOException { - DocsEnum de = MultiFields.getTermDocsEnum(ir, null, field, new BytesRef(text)); + DocsEnum de = MultiFields.getTermDocsEnum(ir, null, field, new BytesRef(text), false); assertTrue(de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); } Index: modules/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java =================================================================== --- modules/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java (revision 1209915) +++ modules/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java (working copy) @@ -106,8 +106,10 @@ Term drillDownTerm = DrillDown.term(searchParams, catPath); // TODO (Facet): avoid Multi*? Bits liveDocs = MultiFields.getLiveDocs(indexReader); - int updatedCount = countIntersection(MultiFields.getTermDocsEnum(indexReader, liveDocs, drillDownTerm.field(), drillDownTerm.bytes()), - docIds.iterator()); + int updatedCount = countIntersection(MultiFields.getTermDocsEnum(indexReader, liveDocs, + drillDownTerm.field(), drillDownTerm.bytes(), + false), + docIds.iterator()); fresNode.setValue(updatedCount); } Index: modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java =================================================================== --- modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (revision 1209915) +++ modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java (working copy) @@ -405,7 +405,8 @@ // TODO (Facet): avoid Multi*? Bits liveDocs = MultiFields.getLiveDocs(reader); DocsEnum docs = MultiFields.getTermDocsEnum(reader, liveDocs, Consts.FULL, - new BytesRef(categoryPath.toString(delimiter))); + new BytesRef(categoryPath.toString(delimiter)), + false); if (docs == null || docs.nextDoc() == DocIdSetIterator.NO_MORE_DOCS) { return -1; // category does not exist in taxonomy } @@ -441,7 +442,8 @@ } Bits liveDocs = MultiFields.getLiveDocs(reader); DocsEnum docs = MultiFields.getTermDocsEnum(reader, liveDocs, Consts.FULL, - new BytesRef(categoryPath.toString(delimiter, prefixLen))); + new BytesRef(categoryPath.toString(delimiter, prefixLen)), + false); if (docs == null || docs.nextDoc() == DocIdSetIterator.NO_MORE_DOCS) { return -1; // category does not exist in taxonomy } @@ -788,7 +790,7 @@ // hence documents), there are no deletions in the index. Therefore, it // is sufficient to call next(), and then doc(), exactly once with no // 'validation' checks. - docsEnum = termsEnum.docs(liveDocs, docsEnum); + docsEnum = termsEnum.docs(liveDocs, docsEnum, false); docsEnum.nextDoc(); cp.clear(); // TODO (Facet): avoid String creation/use bytes? @@ -925,7 +927,7 @@ // like Lucene's merge works, we hope there are few seeks. // TODO (Facet): is there a quicker way? E.g., not specifying the // next term by name every time? - otherdocsEnum[i] = othertes[i].docs(MultiFields.getLiveDocs(otherreaders[i]), otherdocsEnum[i]); + otherdocsEnum[i] = othertes[i].docs(MultiFields.getLiveDocs(otherreaders[i]), otherdocsEnum[i], false); otherdocsEnum[i].nextDoc(); // TODO (Facet): check? int origordinal = otherdocsEnum[i].docID(); ordinalMaps[i].addMapping(origordinal, newordinal); @@ -942,7 +944,7 @@ // to be added because it already existed in the main taxonomy. // TODO (Facet): Again, is there a quicker way? - mainde = mainte.docs(MultiFields.getLiveDocs(mainreader), mainde); + mainde = mainte.docs(MultiFields.getLiveDocs(mainreader), mainde, false); mainde.nextDoc(); // TODO (Facet): check? int newordinal = mainde.docID(); @@ -950,7 +952,7 @@ for (int i=0; i allEnums = new IdentityHashMap(); TermsEnum te = segment.terms("foo").iterator(null); while (te.next() != null) { - reuse = te.docs(null, reuse); + reuse = te.docs(null, reuse, false); allEnums.put(reuse, true); } @@ -101,7 +101,7 @@ Map allEnums = new IdentityHashMap(); TermsEnum te = segment.terms("foo").iterator(null); while (te.next() != null) { - reuse = te.docs(null, reuse); + reuse = te.docs(null, reuse, false); allEnums.put(reuse, true); } Index: lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java (working copy) @@ -17,15 +17,16 @@ * limitations under the License. */ -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.store.Directory; +import java.io.IOException; + import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; -import java.io.IOException; - public class TestSegmentTermDocs extends LuceneTestCase { private Document testDoc = new Document(); private Directory dir; @@ -61,7 +62,7 @@ TermsEnum terms = reader.fields().terms(DocHelper.TEXT_FIELD_2_KEY).iterator(null); terms.seekCeil(new BytesRef("field")); - DocsEnum termDocs = terms.docs(reader.getLiveDocs(), null); + DocsEnum termDocs = _TestUtil.docs(random, terms, reader.getLiveDocs(), null, true); if (termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS) { int docId = termDocs.docID(); assertTrue(docId == 0); @@ -80,9 +81,12 @@ //After adding the document, we should be able to read it back in SegmentReader reader = SegmentReader.get(true, info, indexDivisor, newIOContext(random)); assertTrue(reader != null); - DocsEnum termDocs = reader.termDocsEnum(reader.getLiveDocs(), - "textField2", - new BytesRef("bad")); + DocsEnum termDocs = _TestUtil.docs(random, reader, + "textField2", + new BytesRef("bad"), + reader.getLiveDocs(), + null, + false); assertNull(termDocs); reader.close(); @@ -91,9 +95,12 @@ //After adding the document, we should be able to read it back in SegmentReader reader = SegmentReader.get(true, info, indexDivisor, newIOContext(random)); assertTrue(reader != null); - DocsEnum termDocs = reader.termDocsEnum(reader.getLiveDocs(), - "junk", - new BytesRef("bad")); + DocsEnum termDocs = _TestUtil.docs(random, reader, + "junk", + new BytesRef("bad"), + reader.getLiveDocs(), + null, + false); assertNull(termDocs); reader.close(); } @@ -125,10 +132,12 @@ IndexReader reader = IndexReader.open(dir, null, true, indexDivisor); - DocsEnum tdocs = MultiFields.getTermDocsEnum(reader, - MultiFields.getLiveDocs(reader), - ta.field(), - new BytesRef(ta.text())); + DocsEnum tdocs = _TestUtil.docs(random, reader, + ta.field(), + new BytesRef(ta.text()), + MultiFields.getLiveDocs(reader), + null, + true); // without optimization (assumption skipInterval == 16) @@ -148,10 +157,12 @@ assertFalse(tdocs.advance(10) != DocsEnum.NO_MORE_DOCS); // without next - tdocs = MultiFields.getTermDocsEnum(reader, - MultiFields.getLiveDocs(reader), - ta.field(), - new BytesRef(ta.text())); + tdocs = _TestUtil.docs(random, reader, + ta.field(), + new BytesRef(ta.text()), + MultiFields.getLiveDocs(reader), + null, + false); assertTrue(tdocs.advance(0) != DocsEnum.NO_MORE_DOCS); assertEquals(0, tdocs.docID()); @@ -164,10 +175,12 @@ // exactly skipInterval documents and therefore with optimization // with next - tdocs = MultiFields.getTermDocsEnum(reader, - MultiFields.getLiveDocs(reader), - tb.field(), - new BytesRef(tb.text())); + tdocs = _TestUtil.docs(random, reader, + tb.field(), + new BytesRef(tb.text()), + MultiFields.getLiveDocs(reader), + null, + true); assertTrue(tdocs.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(10, tdocs.docID()); @@ -186,10 +199,12 @@ assertFalse(tdocs.advance(26) != DocsEnum.NO_MORE_DOCS); // without next - tdocs = MultiFields.getTermDocsEnum(reader, - MultiFields.getLiveDocs(reader), - tb.field(), - new BytesRef(tb.text())); + tdocs = _TestUtil.docs(random, reader, + tb.field(), + new BytesRef(tb.text()), + MultiFields.getLiveDocs(reader), + null, + true); assertTrue(tdocs.advance(5) != DocsEnum.NO_MORE_DOCS); assertEquals(10, tdocs.docID()); @@ -204,10 +219,12 @@ // much more than skipInterval documents and therefore with optimization // with next - tdocs = MultiFields.getTermDocsEnum(reader, - MultiFields.getLiveDocs(reader), - tc.field(), - new BytesRef(tc.text())); + tdocs = _TestUtil.docs(random, reader, + tc.field(), + new BytesRef(tc.text()), + MultiFields.getLiveDocs(reader), + null, + true); assertTrue(tdocs.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(26, tdocs.docID()); @@ -228,10 +245,12 @@ assertFalse(tdocs.advance(76) != DocsEnum.NO_MORE_DOCS); //without next - tdocs = MultiFields.getTermDocsEnum(reader, - MultiFields.getLiveDocs(reader), - tc.field(), - new BytesRef(tc.text())); + tdocs = _TestUtil.docs(random, reader, + tc.field(), + new BytesRef(tc.text()), + MultiFields.getLiveDocs(reader), + null, + false); assertTrue(tdocs.advance(5) != DocsEnum.NO_MORE_DOCS); assertEquals(26, tdocs.docID()); assertTrue(tdocs.advance(40) != DocsEnum.NO_MORE_DOCS); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (working copy) @@ -49,9 +49,11 @@ public static int count(Term t, IndexReader r) throws IOException { int count = 0; - DocsEnum td = MultiFields.getTermDocsEnum(r, - MultiFields.getLiveDocs(r), - t.field(), new BytesRef(t.text())); + DocsEnum td = _TestUtil.docs(random, r, + t.field(), new BytesRef(t.text()), + MultiFields.getLiveDocs(r), + null, + false); if (td != null) { while (td.nextDoc() != DocsEnum.NO_MORE_DOCS) { @@ -990,7 +992,7 @@ w.addDocument(doc); IndexReader r = IndexReader.open(w, true).getSequentialSubReaders()[0]; try { - r.termDocsEnum(null, "f", new BytesRef("val")); + _TestUtil.docs(random, r, "f", new BytesRef("val"), null, null, false); fail("should have failed to seek since terms index was not loaded."); } catch (IllegalStateException e) { // expected - we didn't load the term index Index: lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (working copy) @@ -17,19 +17,20 @@ * limitations under the License. */ -import org.apache.lucene.util.InfoStream; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.store.Directory; +import java.io.IOException; +import java.util.Collection; + import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.InfoStream; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; -import java.io.IOException; -import java.util.Collection; - public class TestSegmentMerger extends LuceneTestCase { //The variables for the new merged segment private Directory mergedDir; @@ -98,10 +99,12 @@ assertTrue(newDoc2 != null); assertTrue(DocHelper.numFields(newDoc2) == DocHelper.numFields(doc2) - DocHelper.unstored.size()); - DocsEnum termDocs = MultiFields.getTermDocsEnum(mergedReader, - MultiFields.getLiveDocs(mergedReader), - DocHelper.TEXT_FIELD_2_KEY, - new BytesRef("field")); + DocsEnum termDocs = _TestUtil.docs(random, mergedReader, + DocHelper.TEXT_FIELD_2_KEY, + new BytesRef("field"), + MultiFields.getLiveDocs(mergedReader), + null, + false); assertTrue(termDocs != null); assertTrue(termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS); Index: lucene/src/test/org/apache/lucene/index/TestTermsEnum.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestTermsEnum.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestTermsEnum.java (working copy) @@ -331,7 +331,7 @@ } assertEquals(expected, actual); assertEquals(1, te.docFreq()); - docsEnum = te.docs(null, docsEnum); + docsEnum = _TestUtil.docs(random, te, null, docsEnum, false); final int docID = docsEnum.nextDoc(); assertTrue(docID != DocsEnum.NO_MORE_DOCS); assertEquals(docIDToID[docID], termToID.get(expected).intValue()); Index: lucene/src/test/org/apache/lucene/index/TestStressAdvance.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestStressAdvance.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestStressAdvance.java (working copy) @@ -75,11 +75,11 @@ System.out.println("\nTEST: iter=" + iter + " iter2=" + iter2); } assertEquals(TermsEnum.SeekStatus.FOUND, te.seekCeil(new BytesRef("a"))); - de = te.docs(null, de); + de = _TestUtil.docs(random, te, null, de, false); testOne(de, aDocIDs); assertEquals(TermsEnum.SeekStatus.FOUND, te.seekCeil(new BytesRef("b"))); - de = te.docs(null, de); + de = _TestUtil.docs(random, te, null, de, false); testOne(de, bDocIDs); } Index: lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java (working copy) @@ -19,12 +19,13 @@ import java.io.IOException; -import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; public class TestParallelTermEnum extends LuceneTestCase { private IndexReader ir1; @@ -88,31 +89,31 @@ TermsEnum te = terms.iterator(null); assertEquals("brown", te.next().utf8ToString()); - DocsEnum td = te.docs(liveDocs, null); + DocsEnum td = _TestUtil.docs(random, te, liveDocs, null, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("fox", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("jumps", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("quick", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("the", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); @@ -125,31 +126,31 @@ te = terms.iterator(null); assertEquals("brown", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("fox", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("jumps", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("quick", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("the", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); @@ -162,37 +163,37 @@ te = terms.iterator(null); assertEquals("dog", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("fox", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("jumps", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("lazy", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("over", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); assertEquals("the", te.next().utf8ToString()); - td = te.docs(liveDocs, td); + td = _TestUtil.docs(random, te, liveDocs, td, false); assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); assertEquals(0, td.docID()); assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); Index: lucene/src/test/org/apache/lucene/index/TestDocCount.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDocCount.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestDocCount.java (working copy) @@ -68,11 +68,14 @@ String field; while ((field = e.next()) != null) { Terms terms = fields.terms(field); + if (terms == null) { + continue; + } int docCount = terms.getDocCount(); FixedBitSet visited = new FixedBitSet(ir.maxDoc()); TermsEnum te = terms.iterator(null); while (te.next() != null) { - DocsEnum de = te.docs(null, null); + DocsEnum de = _TestUtil.docs(random, te, null, null, false); while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) { visited.set(de.docID()); } Index: lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java (working copy) @@ -22,7 +22,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; @@ -34,6 +33,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.ReaderUtil; +import org.apache.lucene.util._TestUtil; public class TestDocsAndPositions extends LuceneTestCase { private String fieldName; @@ -99,16 +99,6 @@ return reader.termPositionsEnum(null, fieldName, bytes); } - public DocsEnum getDocsEnum(IndexReader reader, BytesRef bytes, - boolean freqs, Bits liveDocs) throws IOException { - int randInt = random.nextInt(10); - if (randInt == 0) { // once in a while throw in a positions enum - return getDocsAndPositions(reader, bytes, liveDocs); - } else { - return reader.termDocsEnum(liveDocs, fieldName, bytes); - } - } - /** * this test indexes random numbers within a range into a field and checks * their occurrences by searching for a number from that range selected at @@ -232,31 +222,31 @@ AtomicReaderContext[] leaves = ReaderUtil.leaves(topReaderContext); for (AtomicReaderContext context : leaves) { int maxDoc = context.reader.maxDoc(); - DocsEnum docsAndPosEnum = getDocsEnum(context.reader, bytes, true, null); + DocsEnum docsEnum = _TestUtil.docs(random, context.reader, fieldName, bytes, null, null, true); if (findNext(freqInDoc, context.docBase, context.docBase + maxDoc) == Integer.MAX_VALUE) { - assertNull(docsAndPosEnum); + assertNull(docsEnum); continue; } - assertNotNull(docsAndPosEnum); - docsAndPosEnum.nextDoc(); + assertNotNull(docsEnum); + docsEnum.nextDoc(); for (int j = 0; j < maxDoc; j++) { if (freqInDoc[context.docBase + j] != 0) { - assertEquals(j, docsAndPosEnum.docID()); - assertEquals(docsAndPosEnum.freq(), freqInDoc[context.docBase +j]); + assertEquals(j, docsEnum.docID()); + assertEquals(docsEnum.freq(), freqInDoc[context.docBase +j]); if (i % 2 == 0 && random.nextInt(10) == 0) { int next = findNext(freqInDoc, context.docBase+j+1, context.docBase + maxDoc) - context.docBase; - int advancedTo = docsAndPosEnum.advance(next); + int advancedTo = docsEnum.advance(next); if (next >= maxDoc) { assertEquals(DocsEnum.NO_MORE_DOCS, advancedTo); } else { assertTrue("advanced to: " +advancedTo + " but should be <= " + next, next >= advancedTo); } } else { - docsAndPosEnum.nextDoc(); + docsEnum.nextDoc(); } } } - assertEquals("docBase: " + context.docBase + " maxDoc: " + maxDoc + " " + docsAndPosEnum.getClass(), DocsEnum.NO_MORE_DOCS, docsAndPosEnum.docID()); + assertEquals("docBase: " + context.docBase + " maxDoc: " + maxDoc + " " + docsEnum.getClass(), DocsEnum.NO_MORE_DOCS, docsEnum.docID()); } } @@ -343,7 +333,7 @@ writer.addDocument(doc); IndexReader reader = writer.getReader(); IndexReader r = getOnlySegmentReader(reader); - DocsEnum disi = r.termDocsEnum(null, "foo", new BytesRef("bar")); + DocsEnum disi = _TestUtil.docs(random, r, "foo", new BytesRef("bar"), null, null, false); int docid = disi.docID(); assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -351,7 +341,7 @@ // now reuse and check again TermsEnum te = r.terms("foo").iterator(null); assertTrue(te.seekExact(new BytesRef("bar"), true)); - disi = te.docs(null, disi); + disi = _TestUtil.docs(random, te, null, disi, false); docid = disi.docID(); assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java (working copy) @@ -334,7 +334,7 @@ // deleted docs): DocsEnum docs = null; while(termsEnum.next() != null) { - docs = termsEnum.docs(liveDocs1, docs); + docs = _TestUtil.docs(random, termsEnum, null, docs, false); while(docs.nextDoc() != DocsEnum.NO_MORE_DOCS) { fail("r1 is not empty but r2 is"); } @@ -354,9 +354,9 @@ break; } - termDocs1 = termsEnum.docs(liveDocs1, termDocs1); + termDocs1 = _TestUtil.docs(random, termsEnum, liveDocs1, termDocs1, false); if (termsEnum2.seekExact(term, false)) { - termDocs2 = termsEnum2.docs(liveDocs2, termDocs2); + termDocs2 = _TestUtil.docs(random, termsEnum2, liveDocs2, termDocs2, false); } else { termDocs2 = null; } @@ -415,7 +415,7 @@ System.out.println(" pos=" + dpEnum.nextPosition()); } } else { - dEnum = termsEnum3.docs(null, dEnum); + dEnum = _TestUtil.docs(random, termsEnum3, null, dEnum, true); assertNotNull(dEnum); assertTrue(dEnum.nextDoc() != DocsEnum.NO_MORE_DOCS); final int freq = dEnum.freq(); @@ -449,7 +449,7 @@ System.out.println(" pos=" + dpEnum.nextPosition()); } } else { - dEnum = termsEnum3.docs(null, dEnum); + dEnum = _TestUtil.docs(random, termsEnum3, null, dEnum, true); assertNotNull(dEnum); assertTrue(dEnum.nextDoc() != DocsEnum.NO_MORE_DOCS); final int freq = dEnum.freq(); @@ -506,7 +506,7 @@ } //System.out.println("TEST: term1=" + term1); - docs1 = termsEnum1.docs(liveDocs1, docs1); + docs1 = _TestUtil.docs(random, termsEnum1, liveDocs1, docs1, true); while (docs1.nextDoc() != DocsEnum.NO_MORE_DOCS) { int d = docs1.docID(); int f = docs1.freq(); @@ -540,7 +540,7 @@ } //System.out.println("TEST: term1=" + term1); - docs2 = termsEnum2.docs(liveDocs2, docs2); + docs2 = _TestUtil.docs(random, termsEnum2, liveDocs2, docs2, true); while (docs2.nextDoc() != DocsEnum.NO_MORE_DOCS) { int d = r2r1[docs2.docID()]; int f = docs2.freq(); @@ -667,8 +667,8 @@ assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum1.nextDoc()); assertEquals(DocsEnum.NO_MORE_DOCS, dpEnum2.nextDoc()); } else { - dEnum1 = termsEnum1.docs(null, dEnum1); - dEnum2 = termsEnum2.docs(null, dEnum2); + dEnum1 = _TestUtil.docs(random, termsEnum1, null, dEnum1, true); + dEnum2 = _TestUtil.docs(random, termsEnum2, null, dEnum2, true); assertNotNull(dEnum1); assertNotNull(dEnum2); int docID1 = dEnum1.nextDoc(); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -546,10 +546,12 @@ assertEquals(1, reader.numDocs()); Term t = new Term("field", "a"); assertEquals(1, reader.docFreq(t)); - DocsEnum td = MultiFields.getTermDocsEnum(reader, - MultiFields.getLiveDocs(reader), - "field", - new BytesRef("a")); + DocsEnum td = _TestUtil.docs(random, reader, + "field", + new BytesRef("a"), + MultiFields.getLiveDocs(reader), + null, + true); td.nextDoc(); assertEquals(128*1024, td.freq()); reader.close(); @@ -1338,12 +1340,12 @@ // test that the terms were indexed. - assertTrue(MultiFields.getTermDocsEnum(ir, null, "binary", new BytesRef("doc1field1")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); - assertTrue(MultiFields.getTermDocsEnum(ir, null, "binary", new BytesRef("doc2field1")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); - assertTrue(MultiFields.getTermDocsEnum(ir, null, "binary", new BytesRef("doc3field1")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); - assertTrue(MultiFields.getTermDocsEnum(ir, null, "string", new BytesRef("doc1field2")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); - assertTrue(MultiFields.getTermDocsEnum(ir, null, "string", new BytesRef("doc2field2")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); - assertTrue(MultiFields.getTermDocsEnum(ir, null, "string", new BytesRef("doc3field2")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); + assertTrue(_TestUtil.docs(random, ir, "binary", new BytesRef("doc1field1"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); + assertTrue(_TestUtil.docs(random, ir, "binary", new BytesRef("doc2field1"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); + assertTrue(_TestUtil.docs(random, ir, "binary", new BytesRef("doc3field1"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); + assertTrue(_TestUtil.docs(random, ir, "string", new BytesRef("doc1field2"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); + assertTrue(_TestUtil.docs(random, ir, "string", new BytesRef("doc2field2"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); + assertTrue(_TestUtil.docs(random, ir, "string", new BytesRef("doc3field2"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS); ir.close(); dir.close(); @@ -1415,7 +1417,7 @@ TermsEnum t = r.fields().terms("field").iterator(null); int count = 0; while(t.next() != null) { - final DocsEnum docs = t.docs(null, null); + final DocsEnum docs = _TestUtil.docs(random, t, null, null, false); assertEquals(0, docs.nextDoc()); assertEquals(DocIdSetIterator.NO_MORE_DOCS, docs.nextDoc()); count++; Index: lucene/src/test/org/apache/lucene/index/TestMultiFields.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestMultiFields.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestMultiFields.java (working copy) @@ -120,7 +120,7 @@ System.out.println("TEST: seek term="+ UnicodeUtil.toHexString(term.utf8ToString()) + " " + term); } - DocsEnum docsEnum = MultiFields.getTermDocsEnum(reader, liveDocs, "field", term); + DocsEnum docsEnum = _TestUtil.docs(random, reader, "field", term, liveDocs, null, false); assertNotNull(docsEnum); for(int docID : docs.get(term)) { @@ -138,11 +138,12 @@ /* private void verify(IndexReader r, String term, List expected) throws Exception { - DocsEnum docs = MultiFields.getTermDocsEnum(r, - MultiFields.getLiveDocs(r), - "field", - new BytesRef(term)); - + DocsEnum docs = _TestUtil.docs(random, r, + "field", + new BytesRef(term), + MultiFields.getLiveDocs(r), + null, + false); for(int docID : expected) { assertEquals(docID, docs.nextDoc()); } @@ -160,8 +161,8 @@ w.addDocument(d); IndexReader r = w.getReader(); w.close(); - DocsEnum d1 = MultiFields.getTermDocsEnum(r, null, "f", new BytesRef("j")); - DocsEnum d2 = MultiFields.getTermDocsEnum(r, null, "f", new BytesRef("j")); + DocsEnum d1 = _TestUtil.docs(random, r, "f", new BytesRef("j"), null, null, false); + DocsEnum d2 = _TestUtil.docs(random, r, "f", new BytesRef("j"), null, null, false); assertEquals(0, d1.nextDoc()); assertEquals(0, d2.nextDoc()); r.close(); Index: lucene/src/test/org/apache/lucene/index/TestCodecs.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestCodecs.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestCodecs.java (working copy) @@ -279,7 +279,7 @@ // make sure it properly fully resets (rewinds) its // internal state: for(int iter=0;iter<2;iter++) { - docsEnum = termsEnum.docs(null, docsEnum); + docsEnum = _TestUtil.docs(random, termsEnum, null, docsEnum, false); assertEquals(terms[i].docs[0], docsEnum.nextDoc()); assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.nextDoc()); } @@ -479,7 +479,7 @@ assertEquals(status, TermsEnum.SeekStatus.FOUND); assertEquals(term.docs.length, termsEnum.docFreq()); if (field.omitTF) { - this.verifyDocs(term.docs, term.positions, termsEnum.docs(null, null), false); + this.verifyDocs(term.docs, term.positions, _TestUtil.docs(random, termsEnum, null, null, false), false); } else { this.verifyDocs(term.docs, term.positions, termsEnum.docsAndPositions(null, null), true); } @@ -499,7 +499,7 @@ assertTrue(termsEnum.term().bytesEquals(new BytesRef(term.text2))); assertEquals(term.docs.length, termsEnum.docFreq()); if (field.omitTF) { - this.verifyDocs(term.docs, term.positions, termsEnum.docs(null, null), false); + this.verifyDocs(term.docs, term.positions, _TestUtil.docs(random, termsEnum, null, null, false), false); } else { this.verifyDocs(term.docs, term.positions, termsEnum.docsAndPositions(null, null), true); } @@ -549,15 +549,22 @@ do { term = field.terms[upto]; if (TestCodecs.random.nextInt(3) == 1) { - final DocsEnum docs = termsEnum.docs(null, null); - final DocsAndPositionsEnum postings = termsEnum.docsAndPositions(null, null); - - final DocsEnum docsEnum; - if (postings != null) { - docsEnum = postings; + final DocsEnum docs; + final DocsEnum docsAndFreqs; + final DocsAndPositionsEnum postings; + if (!field.omitTF) { + postings = termsEnum.docsAndPositions(null, null); + if (postings != null) { + docs = docsAndFreqs = postings; + } else { + docs = docsAndFreqs = _TestUtil.docs(random, termsEnum, null, null, true); + } } else { - docsEnum = docs; + postings = null; + docsAndFreqs = null; + docs = _TestUtil.docs(random, termsEnum, null, null, false); } + assertNotNull(docs); int upto2 = -1; while(upto2 < term.docs.length-1) { // Maybe skip: @@ -567,10 +574,10 @@ final int inc = 1+TestCodecs.random.nextInt(left-1); upto2 += inc; if (TestCodecs.random.nextInt(2) == 1) { - doc = docsEnum.advance(term.docs[upto2]); + doc = docs.advance(term.docs[upto2]); assertEquals(term.docs[upto2], doc); } else { - doc = docsEnum.advance(1+term.docs[upto2]); + doc = docs.advance(1+term.docs[upto2]); if (doc == DocIdSetIterator.NO_MORE_DOCS) { // skipped past last doc assert upto2 == term.docs.length-1; @@ -584,20 +591,20 @@ } } } else { - doc = docsEnum.nextDoc(); + doc = docs.nextDoc(); assertTrue(doc != -1); upto2++; } assertEquals(term.docs[upto2], doc); if (!field.omitTF) { - assertEquals(term.positions[upto2].length, docsEnum.freq()); + assertEquals(term.positions[upto2].length, postings.freq()); if (TestCodecs.random.nextInt(2) == 1) { this.verifyPositions(term.positions[upto2], postings); } } } - assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsEnum.nextDoc()); + assertEquals(DocIdSetIterator.NO_MORE_DOCS, docs.nextDoc()); } upto++; Index: lucene/src/test/org/apache/lucene/index/TestSegmentReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (revision 1209915) +++ lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (working copy) @@ -22,12 +22,12 @@ import java.util.Iterator; import java.util.List; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.BytesRef; - import org.apache.lucene.document.Document; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; public class TestSegmentReader extends LuceneTestCase { private Directory dir; @@ -132,16 +132,20 @@ } } - DocsEnum termDocs = MultiFields.getTermDocsEnum(reader, - MultiFields.getLiveDocs(reader), - DocHelper.TEXT_FIELD_1_KEY, - new BytesRef("field")); + DocsEnum termDocs = _TestUtil.docs(random, reader, + DocHelper.TEXT_FIELD_1_KEY, + new BytesRef("field"), + MultiFields.getLiveDocs(reader), + null, + false); assertTrue(termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS); - termDocs = MultiFields.getTermDocsEnum(reader, - MultiFields.getLiveDocs(reader), - DocHelper.NO_NORMS_KEY, - new BytesRef(DocHelper.NO_NORMS_TEXT)); + termDocs = _TestUtil.docs(random, reader, + DocHelper.NO_NORMS_KEY, + new BytesRef(DocHelper.NO_NORMS_TEXT), + MultiFields.getLiveDocs(reader), + null, + false); assertTrue(termDocs.nextDoc() != DocsEnum.NO_MORE_DOCS); Index: lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (working copy) @@ -105,7 +105,7 @@ do { // System.out.println(" iter termCount=" + termCount + " term=" + // enumerator.term().toBytesString()); - docsEnum = termsEnum.docs(acceptDocs, docsEnum); + docsEnum = termsEnum.docs(acceptDocs, docsEnum, false); int docid; while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) { bitSet.set(docid); Index: lucene/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java (revision 0) +++ lucene/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java (working copy) @@ -0,0 +1,37 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +/** Scorer for conjunctions, sets of terms, all of which are required. */ +final class MatchOnlyConjunctionTermScorer extends ConjunctionTermScorer { + MatchOnlyConjunctionTermScorer(Weight weight, float coord, + DocsAndFreqs[] docsAndFreqs) throws IOException { + super(weight, coord, docsAndFreqs); + } + + @Override + public float score() throws IOException { + float sum = 0.0f; + for (DocsAndFreqs docs : docsAndFreqs) { + sum += docs.docScorer.score(lastDoc, 1); + } + return sum * coord; + } +} Property changes on: lucene/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/src/java/org/apache/lucene/search/BooleanQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/BooleanQuery.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/search/BooleanQuery.java (working copy) @@ -17,21 +17,22 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.*; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.Term; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.ToStringUtils; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.ConjunctionTermScorer.DocsAndFreqs; +import org.apache.lucene.search.TermQuery.TermWeight; +import org.apache.lucene.search.similarities.Similarity.ExactDocScorer; import org.apache.lucene.search.similarities.SimilarityProvider; -import org.apache.lucene.search.similarities.Similarity.ExactDocScorer; -import org.apache.lucene.search.TermQuery.TermWeight; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.ToStringUtils; -import java.io.IOException; -import java.util.*; - /** A Query that matches documents matching boolean combinations of other * queries, e.g. {@link TermQuery}s, {@link PhraseQuery}s or other * BooleanQuerys. @@ -349,6 +350,11 @@ private Scorer createConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs) throws IOException { + + // TODO: fix scorer API to specify "needsScores" up + // front, so we can do match-only if caller doesn't + // needs scores + final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()]; for (int i = 0; i < docsAndFreqs.length; i++) { final TermWeight weight = (TermWeight) weights.get(i); @@ -357,12 +363,40 @@ return null; } final ExactDocScorer docScorer = weight.createDocScorer(context); - docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docs( - acceptDocs, null), termsEnum.docFreq(), docScorer); + final DocsEnum docsAndFreqsEnum = termsEnum.docs(acceptDocs, null, true); + if (docsAndFreqsEnum == null) { + // At least one sub cannot provide freqs; abort + // and fallback to full match-only scorer: + return createMatchOnlyConjunctionTermScorer(context, acceptDocs); + } + + docsAndFreqs[i] = new DocsAndFreqs(docsAndFreqsEnum, + docsAndFreqsEnum, + termsEnum.docFreq(), docScorer); } return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord( docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs); } + + private Scorer createMatchOnlyConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs) + throws IOException { + + final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()]; + for (int i = 0; i < docsAndFreqs.length; i++) { + final TermWeight weight = (TermWeight) weights.get(i); + final TermsEnum termsEnum = weight.getTermsEnum(context); + if (termsEnum == null) { + return null; + } + final ExactDocScorer docScorer = weight.createDocScorer(context); + docsAndFreqs[i] = new DocsAndFreqs(null, + termsEnum.docs(acceptDocs, null, false), + termsEnum.docFreq(), docScorer); + } + + return new MatchOnlyConjunctionTermScorer(this, disableCoord ? 1.0f : coord( + docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs); + } @Override public boolean scoresDocsOutOfOrder() { Index: lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (working copy) @@ -259,8 +259,8 @@ } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { - return actualEnum.docs(liveDocs, reuse); + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { + return actualEnum.docs(liveDocs, reuse, needsFreqs); } @Override Index: lucene/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java (revision 0) +++ lucene/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java (working copy) @@ -0,0 +1,94 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.search.similarities.Similarity; + +/** Expert: A Scorer for documents matching a + * Term. It treats all documents as having + * one occurrenc (tf=1) for the term. + */ + +final class MatchOnlyTermScorer extends Scorer { + private final DocsEnum docsEnum; + private final Similarity.ExactDocScorer docScorer; + + /** + * Construct a TermScorer. + * + * @param weight + * The weight of the Term in the query. + * @param td + * An iterator over the documents matching the Term. + * @param docScorer + * The Similarity.ExactDocScorer implementation + * to be used for score computations. + */ + MatchOnlyTermScorer(Weight weight, DocsEnum td, Similarity.ExactDocScorer docScorer) throws IOException { + super(weight); + this.docScorer = docScorer; + this.docsEnum = td; + } + + @Override + public int docID() { + return docsEnum.docID(); + } + + @Override + public float freq() { + return 1.0f; + } + + /** + * Advances to the next document matching the query.
+ * + * @return the document matching the query or NO_MORE_DOCS if there are no more documents. + */ + @Override + public int nextDoc() throws IOException { + return docsEnum.nextDoc(); + } + + @Override + public float score() { + assert docID() != NO_MORE_DOCS; + return docScorer.score(docsEnum.docID(), 1); + } + + /** + * Advances to the first match beyond the current whose document number is + * greater than or equal to a given target.
+ * The implementation uses {@link DocsEnum#advance(int)}. + * + * @param target + * The target document number. + * @return the matching document or NO_MORE_DOCS if none exist. + */ + @Override + public int advance(int target) throws IOException { + return docsEnum.advance(target); + } + + /** Returns a string representation of this TermScorer. */ + @Override + public String toString() { return "scorer(" + weight + ")"; } +} Property changes on: lucene/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native Index: lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (working copy) @@ -229,7 +229,7 @@ if (postingsEnum == null) { // term does exist, but has no positions - assert termsEnum.docs(liveDocs, null) != null: "termstate found but no term exists in reader"; + assert termsEnum.docs(liveDocs, null, false) != null: "termstate found but no term exists in reader"; throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")"); } Index: lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java =================================================================== --- lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (working copy) @@ -17,17 +17,18 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.Comparator; + import org.apache.lucene.index.DocsEnum; import org.apache.lucene.search.similarities.Similarity.ExactDocScorer; import org.apache.lucene.util.ArrayUtil; -import java.io.IOException; -import java.util.Comparator; /** Scorer for conjunctions, sets of terms, all of which are required. */ -final class ConjunctionTermScorer extends Scorer { - private final float coord; - private int lastDoc = -1; - private final DocsAndFreqs[] docsAndFreqs; +class ConjunctionTermScorer extends Scorer { + protected final float coord; + protected int lastDoc = -1; + protected final DocsAndFreqs[] docsAndFreqs; private final DocsAndFreqs lead; ConjunctionTermScorer(Weight weight, float coord, @@ -39,7 +40,7 @@ // lead the matching. ArrayUtil.mergeSort(docsAndFreqs, new Comparator() { public int compare(DocsAndFreqs o1, DocsAndFreqs o2) { - return o1.freq - o2.freq; + return o1.docFreq - o2.docFreq; } }); @@ -96,14 +97,16 @@ } static final class DocsAndFreqs { + final DocsEnum docsAndFreqs; final DocsEnum docs; - final int freq; + final int docFreq; final ExactDocScorer docScorer; int doc = -1; - DocsAndFreqs(DocsEnum docs, int freq, ExactDocScorer docScorer) { + DocsAndFreqs(DocsEnum docsAndFreqs, DocsEnum docs, int docFreq, ExactDocScorer docScorer) { + this.docsAndFreqs = docsAndFreqs; this.docs = docs; - this.freq = freq; + this.docFreq = docFreq; this.docScorer = docScorer; } } Index: lucene/src/java/org/apache/lucene/search/PhraseQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/PhraseQuery.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/search/PhraseQuery.java (working copy) @@ -244,7 +244,7 @@ // PhraseQuery on a field that did not index // positions. if (postingsEnum == null) { - assert reader.termDocsEnum(liveDocs, t.field(), t.bytes(), state) != null: "termstate found but no term exists in reader"; + assert reader.termDocsEnum(liveDocs, t.field(), t.bytes(), state, false) != null: "termstate found but no term exists in reader"; // term does exist, but has no positions throw new IllegalStateException("field \"" + t.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + t.text() + ")"); } Index: lucene/src/java/org/apache/lucene/search/TermQuery.java =================================================================== --- lucene/src/java/org/apache/lucene/search/TermQuery.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/search/TermQuery.java (working copy) @@ -21,18 +21,18 @@ import java.util.Set; import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.index.IndexReader.ReaderContext; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.Term; import org.apache.lucene.index.TermState; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.IndexReader.ReaderContext; -import org.apache.lucene.index.Term; +import org.apache.lucene.search.similarities.Similarity.ExactDocScorer; import org.apache.lucene.search.similarities.Similarity; -import org.apache.lucene.search.similarities.Similarity.ExactDocScorer; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.TermContext; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.ToStringUtils; /** A Query that matches documents containing a term. @@ -83,10 +83,15 @@ if (termsEnum == null) { return null; } - // TODO should we reuse the DocsEnum here? - final DocsEnum docs = termsEnum.docs(acceptDocs, null); - assert docs != null; - return new TermScorer(this, docs, createDocScorer(context)); + DocsEnum docs = termsEnum.docs(acceptDocs, null, true); + if (docs != null) { + return new TermScorer(this, docs, createDocScorer(context)); + } else { + // Index does not store freq info + docs = termsEnum.docs(acceptDocs, null, false); + assert docs != null; + return new MatchOnlyTermScorer(this, docs, createDocScorer(context)); + } } /** @@ -120,12 +125,11 @@ @Override public Explanation explain(AtomicReaderContext context, int doc) throws IOException { - IndexReader reader = context.reader; - DocsEnum docs = reader.termDocsEnum(context.reader.getLiveDocs(), term.field(), term.bytes()); - if (docs != null) { - int newDoc = docs.advance(doc); + Scorer scorer = scorer(context, true, false, context.reader.getLiveDocs()); + if (scorer != null) { + int newDoc = scorer.advance(doc); if (newDoc == doc) { - int freq = docs.freq(); + float freq = scorer.freq(); ExactDocScorer docScorer = similarity.exactDocScorer(stats, term.field(), context); ComplexExplanation result = new ComplexExplanation(); result.setDescription("weight("+getQuery()+" in "+doc+") [" + similarity.getClass().getSimpleName() + "], result of:"); @@ -136,8 +140,7 @@ return result; } } - - return new ComplexExplanation(false, 0.0f, "no matching term"); + return new ComplexExplanation(false, 0.0f, "no matching term"); } } Index: lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (working copy) @@ -342,7 +342,7 @@ break; } final byte termval = parser.parseByte(term); - docs = termsEnum.docs(null, docs); + docs = termsEnum.docs(null, docs, false); while (true) { final int docID = docs.nextDoc(); if (docID == DocsEnum.NO_MORE_DOCS) { @@ -415,7 +415,7 @@ break; } final short termval = parser.parseShort(term); - docs = termsEnum.docs(null, docs); + docs = termsEnum.docs(null, docs, false); while (true) { final int docID = docs.nextDoc(); if (docID == DocsEnum.NO_MORE_DOCS) { @@ -519,7 +519,7 @@ retArray = new int[maxDoc]; } - docs = termsEnum.docs(null, docs); + docs = termsEnum.docs(null, docs, false); while (true) { final int docID = docs.nextDoc(); if (docID == DocsEnum.NO_MORE_DOCS) { @@ -586,7 +586,7 @@ res = new FixedBitSet(maxDoc); } - docs = termsEnum.docs(null, docs); + docs = termsEnum.docs(null, docs, false); // TODO: use bulk API while (true) { final int docID = docs.nextDoc(); @@ -669,7 +669,7 @@ retArray = new float[maxDoc]; } - docs = termsEnum.docs(null, docs); + docs = termsEnum.docs(null, docs, false); while (true) { final int docID = docs.nextDoc(); if (docID == DocsEnum.NO_MORE_DOCS) { @@ -757,7 +757,7 @@ retArray = new long[maxDoc]; } - docs = termsEnum.docs(null, docs); + docs = termsEnum.docs(null, docs, false); while (true) { final int docID = docs.nextDoc(); if (docID == DocsEnum.NO_MORE_DOCS) { @@ -846,7 +846,7 @@ retArray = new double[maxDoc]; } - docs = termsEnum.docs(null, docs); + docs = termsEnum.docs(null, docs, false); while (true) { final int docID = docs.nextDoc(); if (docID == DocsEnum.NO_MORE_DOCS) { @@ -1020,7 +1020,7 @@ } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { throw new UnsupportedOperationException(); } @@ -1147,7 +1147,7 @@ termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1)); } termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term)); - docs = termsEnum.docs(null, docs); + docs = termsEnum.docs(null, docs, false); while (true) { final int docID = docs.nextDoc(); if (docID == DocsEnum.NO_MORE_DOCS) { @@ -1268,7 +1268,7 @@ break; } final long pointer = bytes.copyUsingLengthPrefix(term); - docs = termsEnum.docs(null, docs); + docs = termsEnum.docs(null, docs, false); while (true) { final int docID = docs.nextDoc(); if (docID == DocsEnum.NO_MORE_DOCS) { Index: lucene/src/java/org/apache/lucene/index/DocTermOrds.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocTermOrds.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/DocTermOrds.java (working copy) @@ -314,7 +314,7 @@ final int df = te.docFreq(); if (df <= maxTermDocFreq) { - docsEnum = te.docs(liveDocs, docsEnum); + docsEnum = te.docs(liveDocs, docsEnum, false); // dF, but takes deletions into account int actualDF = 0; @@ -650,8 +650,8 @@ } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { - return termsEnum.docs(liveDocs, reuse); + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { + return termsEnum.docs(liveDocs, reuse, needsFreqs); } @Override Index: lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java =================================================================== --- lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (working copy) @@ -387,7 +387,7 @@ // System.out.println(" term=" + term); if (termsEnum.seekExact(term.bytes(), false)) { - DocsEnum docsEnum = termsEnum.docs(reader.getLiveDocs(), docs); + DocsEnum docsEnum = termsEnum.docs(reader.getLiveDocs(), docs, false); //System.out.println("BDS: got docsEnum=" + docsEnum); if (docsEnum != null) { Index: lucene/src/java/org/apache/lucene/index/CheckIndex.java =================================================================== --- lucene/src/java/org/apache/lucene/index/CheckIndex.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/CheckIndex.java (working copy) @@ -683,6 +683,7 @@ } DocsEnum docs = null; + DocsEnum docsAndFreqs = null; DocsAndPositionsEnum postings = null; final FieldsEnum fieldsEnum = fields.iterator(); @@ -740,7 +741,8 @@ status.totFreq += docFreq; sumDocFreq += docFreq; - docs = termsEnum.docs(liveDocs, docs); + docs = termsEnum.docs(liveDocs, docs, false); + docsAndFreqs = termsEnum.docs(liveDocs, docsAndFreqs, true); postings = termsEnum.docsAndPositions(liveDocs, postings); if (hasOrd) { @@ -762,13 +764,24 @@ status.termCount++; final DocsEnum docs2; + final DocsEnum docsAndFreqs2; final boolean hasPositions; + final boolean hasFreqs; if (postings != null) { docs2 = postings; + docsAndFreqs2 = postings; hasPositions = true; + hasFreqs = true; + } else if (docsAndFreqs != null) { + docs2 = docsAndFreqs; + docsAndFreqs2 = docsAndFreqs; + hasPositions = false; + hasFreqs = true; } else { docs2 = docs; + docsAndFreqs2 = null; hasPositions = false; + hasFreqs = false; } int lastDoc = -1; @@ -780,9 +793,15 @@ break; } visitedDocs.set(doc); - final int freq = docs2.freq(); - status.totPos += freq; - totalTermFreq += freq; + int freq = -1; + if (hasFreqs) { + freq = docsAndFreqs2.freq(); + if (freq <= 0) { + throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds"); + } + status.totPos += freq; + totalTermFreq += freq; + } docCount++; if (doc <= lastDoc) { @@ -793,12 +812,9 @@ } lastDoc = doc; - if (freq <= 0) { - throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds"); - } int lastPos = -1; - if (postings != null) { + if (hasPositions) { for(int j=0;j>> 1; // shift off low bit Index: lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (working copy) @@ -51,7 +51,7 @@ /** Must fully consume state, since after this call that * TermState may be reused. */ - public abstract DocsEnum docs(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsEnum reuse) throws IOException; + public abstract DocsEnum docs(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsEnum reuse, boolean needsFreqs) throws IOException; /** Must fully consume state, since after this call that * TermState may be reused. */ Index: lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java (working copy) @@ -26,8 +26,8 @@ import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexFileNames; @@ -36,10 +36,10 @@ import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.PostingsConsumer; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.TermStats; import org.apache.lucene.index.codecs.TermsConsumer; import org.apache.lucene.store.ByteArrayDataInput; @@ -317,7 +317,6 @@ docUpto++; if (indexOptions == IndexOptions.DOCS_ONLY) { accum += in.readVInt(); - freq = 1; } else { final int code = in.readVInt(); accum += code >>> 1; @@ -371,6 +370,7 @@ @Override public int freq() { + assert indexOptions != IndexOptions.DOCS_ONLY; return freq; } } @@ -600,10 +600,13 @@ } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { decodeMetaData(); FSTDocsEnum docsEnum; - if (reuse == null || !(reuse instanceof FSTDocsEnum)) { + + if (needsFreqs && field.indexOptions == IndexOptions.DOCS_ONLY) { + return null; + } else if (reuse == null || !(reuse instanceof FSTDocsEnum)) { docsEnum = new FSTDocsEnum(field.indexOptions, field.storePayloads); } else { docsEnum = (FSTDocsEnum) reuse; Index: lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java (working copy) @@ -51,8 +51,9 @@ public abstract Comparator getComparator() throws IOException; /** Default merge impl */ - private MappingMultiDocsEnum docsEnum = null; - private MappingMultiDocsAndPositionsEnum postingsEnum = null; + private MappingMultiDocsEnum docsEnum; + private MappingMultiDocsEnum docsAndFreqsEnum; + private MappingMultiDocsAndPositionsEnum postingsEnum; public void merge(MergeState mergeState, TermsEnum termsEnum) throws IOException { @@ -63,7 +64,7 @@ long sumDFsinceLastAbortCheck = 0; FixedBitSet visitedDocs = new FixedBitSet(mergeState.mergedDocCount); - if (mergeState.fieldInfo.indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { + if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) { if (docsEnum == null) { docsEnum = new MappingMultiDocsEnum(); } @@ -74,14 +75,14 @@ while((term = termsEnum.next()) != null) { // We can pass null for liveDocs, because the // mapping enum will skip the non-live docs: - docsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsEnumIn); + docsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsEnumIn, false); if (docsEnumIn != null) { docsEnum.reset(docsEnumIn); final PostingsConsumer postingsConsumer = startTerm(term); final TermStats stats = postingsConsumer.merge(mergeState, docsEnum, visitedDocs); if (stats.docFreq > 0) { finishTerm(term, stats); - sumTotalTermFreq += stats.totalTermFreq; + sumTotalTermFreq += stats.docFreq; sumDFsinceLastAbortCheck += stats.docFreq; sumDocFreq += stats.docFreq; if (sumDFsinceLastAbortCheck > 60000) { @@ -91,7 +92,35 @@ } } } + } else if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS) { + if (docsAndFreqsEnum == null) { + docsAndFreqsEnum = new MappingMultiDocsEnum(); + } + docsAndFreqsEnum.setMergeState(mergeState); + + MultiDocsEnum docsAndFreqsEnumIn = null; + + while((term = termsEnum.next()) != null) { + // We can pass null for liveDocs, because the + // mapping enum will skip the non-live docs: + docsAndFreqsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsAndFreqsEnumIn, true); + assert docsAndFreqsEnumIn != null; + docsAndFreqsEnum.reset(docsAndFreqsEnumIn); + final PostingsConsumer postingsConsumer = startTerm(term); + final TermStats stats = postingsConsumer.merge(mergeState, docsAndFreqsEnum, visitedDocs); + if (stats.docFreq > 0) { + finishTerm(term, stats); + sumTotalTermFreq += stats.totalTermFreq; + sumDFsinceLastAbortCheck += stats.docFreq; + sumDocFreq += stats.docFreq; + if (sumDFsinceLastAbortCheck > 60000) { + mergeState.checkAbort.work(sumDFsinceLastAbortCheck/5.0); + sumDFsinceLastAbortCheck = 0; + } + } + } } else { + assert mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; if (postingsEnum == null) { postingsEnum = new MappingMultiDocsAndPositionsEnum(); } @@ -101,27 +130,26 @@ // We can pass null for liveDocs, because the // mapping enum will skip the non-live docs: postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(null, postingsEnumIn); - if (postingsEnumIn != null) { - postingsEnum.reset(postingsEnumIn); - // set PayloadProcessor - if (mergeState.payloadProcessorProvider != null) { - for (int i = 0; i < mergeState.readers.size(); i++) { - if (mergeState.dirPayloadProcessor[i] != null) { - mergeState.currentPayloadProcessor[i] = mergeState.dirPayloadProcessor[i].getProcessor(mergeState.fieldInfo.name, term); - } + assert postingsEnumIn != null; + postingsEnum.reset(postingsEnumIn); + // set PayloadProcessor + if (mergeState.payloadProcessorProvider != null) { + for (int i = 0; i < mergeState.readers.size(); i++) { + if (mergeState.dirPayloadProcessor[i] != null) { + mergeState.currentPayloadProcessor[i] = mergeState.dirPayloadProcessor[i].getProcessor(mergeState.fieldInfo.name, term); } } - final PostingsConsumer postingsConsumer = startTerm(term); - final TermStats stats = postingsConsumer.merge(mergeState, postingsEnum, visitedDocs); - if (stats.docFreq > 0) { - finishTerm(term, stats); - sumTotalTermFreq += stats.totalTermFreq; - sumDFsinceLastAbortCheck += stats.docFreq; - sumDocFreq += stats.docFreq; - if (sumDFsinceLastAbortCheck > 60000) { - mergeState.checkAbort.work(sumDFsinceLastAbortCheck/5.0); - sumDFsinceLastAbortCheck = 0; - } + } + final PostingsConsumer postingsConsumer = startTerm(term); + final TermStats stats = postingsConsumer.merge(mergeState, postingsEnum, visitedDocs); + if (stats.docFreq > 0) { + finishTerm(term, stats); + sumTotalTermFreq += stats.totalTermFreq; + sumDFsinceLastAbortCheck += stats.docFreq; + sumDocFreq += stats.docFreq; + if (sumDFsinceLastAbortCheck > 60000) { + mergeState.checkAbort.work(sumDFsinceLastAbortCheck/5.0); + sumDFsinceLastAbortCheck = 0; } } } Index: lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java (working copy) @@ -112,7 +112,10 @@ } public final int doc() { return doc; } - public final int freq() { return freq; } + public final int freq() { + assert indexOptions != IndexOptions.DOCS_ONLY; + return freq; + } protected void skippingDoc() throws IOException { } @@ -125,7 +128,6 @@ if (indexOptions == IndexOptions.DOCS_ONLY) { doc += docCode; - freq = 1; } else { doc += docCode >>> 1; // shift off low bit if ((docCode & 1) != 0) // if low bit is set Index: lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java (working copy) @@ -27,8 +27,8 @@ import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexFileNames; @@ -950,9 +950,11 @@ } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { PreDocsEnum docsEnum; - if (reuse == null || !(reuse instanceof PreDocsEnum)) { + if (needsFreqs && fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) { + return null; + } else if (reuse == null || !(reuse instanceof PreDocsEnum)) { docsEnum = new PreDocsEnum(); } else { docsEnum = (PreDocsEnum) reuse; Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java (working copy) @@ -272,7 +272,10 @@ } @Override - public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { + if (needsFreqs && fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) { + return null; + } final SepTermState termState = (SepTermState) _termState; SepDocsEnum docsEnum; if (reuse == null || !(reuse instanceof SepDocsEnum)) { @@ -369,8 +372,6 @@ if (!omitTF) { freqIndex.set(termState.freqIndex); freqIndex.seek(freqReader); - } else { - freq = 1; } docFreq = termState.docFreq; @@ -412,6 +413,7 @@ @Override public int freq() { + assert !omitTF; return freq; } Index: lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java (working copy) @@ -25,8 +25,8 @@ import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexFileNames; @@ -689,14 +689,11 @@ } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { //System.out.println("BTR.docs this=" + this); decodeMetaData(); //System.out.println("BTR.docs: state.docFreq=" + state.docFreq); - final DocsEnum docsEnum = postingsReader.docs(fieldInfo, state, liveDocs, reuse); - assert docsEnum != null; - //System.out.println("BTR.docs: return docsEnum=" + docsEnum); - return docsEnum; + return postingsReader.docs(fieldInfo, state, liveDocs, reuse, needsFreqs); } @Override Index: lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java (working copy) @@ -178,7 +178,10 @@ } @Override - public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { + if (needsFreqs && field.indexOptions == IndexOptions.DOCS_ONLY) { + return null; + } PulsingTermState termState = (PulsingTermState) _termState; if (termState.postingsSize != -1) { PulsingDocsEnum postings; @@ -202,11 +205,11 @@ return postings.reset(liveDocs, termState); } else { if (reuse instanceof PulsingDocsEnum) { - DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse)); + DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse), needsFreqs); setOther(wrapped, reuse); // wrapped.other = reuse return wrapped; } else { - return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse); + return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse, needsFreqs); } } } @@ -283,7 +286,6 @@ docID = -1; accum = 0; payloadLength = 0; - freq = 1; this.liveDocs = liveDocs; return this; } @@ -342,6 +344,7 @@ @Override public int freq() { + assert indexOptions != IndexOptions.DOCS_ONLY; return freq; } Index: lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (working copy) @@ -17,19 +17,24 @@ * limitations under the License. */ -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.index.codecs.FieldsProducer; +import java.io.IOException; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; + +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.Terms; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRef; import org.apache.lucene.util.OpenBitSet; import org.apache.lucene.util.StringHelper; @@ -37,14 +42,9 @@ import org.apache.lucene.util.fst.Builder; import org.apache.lucene.util.fst.BytesRefFSTEnum; import org.apache.lucene.util.fst.FST; +import org.apache.lucene.util.fst.PairOutputs; import org.apache.lucene.util.fst.PositiveIntOutputs; -import org.apache.lucene.util.fst.PairOutputs; -import java.io.IOException; -import java.util.Comparator; -import java.util.Map; -import java.util.HashMap; - class SimpleTextFieldsReader extends FieldsProducer { private final IndexInput in; @@ -190,14 +190,17 @@ } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { + if (needsFreqs && indexOptions == IndexOptions.DOCS_ONLY) { + return null; + } SimpleTextDocsEnum docsEnum; if (reuse != null && reuse instanceof SimpleTextDocsEnum && ((SimpleTextDocsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) { docsEnum = (SimpleTextDocsEnum) reuse; } else { docsEnum = new SimpleTextDocsEnum(); } - return docsEnum.reset(docsStart, liveDocs, indexOptions == IndexOptions.DOCS_ONLY); + return docsEnum.reset(docsStart, liveDocs, !needsFreqs); } @Override @@ -245,9 +248,6 @@ in.seek(fp); this.omitTF = omitTF; docID = -1; - if (omitTF) { - tf = 1; - } return this; } @@ -258,6 +258,7 @@ @Override public int freq() { + assert !omitTF; return tf; } Index: lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java (working copy) @@ -365,10 +365,10 @@ } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { // TODO: reuse SimpleTVDocsEnum e = new SimpleTVDocsEnum(); - e.reset(liveDocs, current.getValue().freq); + e.reset(liveDocs, needsFreqs ? current.getValue().freq : -1); return e; } @@ -399,6 +399,7 @@ @Override public int freq() { + assert freq != -1; return freq; } Index: lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java (working copy) @@ -63,13 +63,25 @@ int df = 0; long totTF = 0; - if (mergeState.fieldInfo.indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { + if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) { while(true) { final int doc = postings.nextDoc(); if (doc == DocIdSetIterator.NO_MORE_DOCS) { break; } visitedDocs.set(doc); + this.startDoc(doc, 0); + this.finishDoc(); + df++; + } + totTF = -1; + } else if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS) { + while(true) { + final int doc = postings.nextDoc(); + if (doc == DocIdSetIterator.NO_MORE_DOCS) { + break; + } + visitedDocs.set(doc); final int freq = postings.freq(); this.startDoc(doc, freq); this.finishDoc(); Index: lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java (working copy) @@ -875,9 +875,9 @@ } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { currentFrame.decodeMetaData(); - return postingsReader.docs(fieldInfo, currentFrame.termState, skipDocs, reuse); + return postingsReader.docs(fieldInfo, currentFrame.termState, skipDocs, reuse, needsFreqs); } @Override @@ -2082,7 +2082,7 @@ } @Override - public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException { + public DocsEnum docs(Bits skipDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { assert !eof; //if (DEBUG) { //System.out.println("BTTR.docs seg=" + segment); @@ -2091,10 +2091,7 @@ //if (DEBUG) { //System.out.println(" state=" + currentFrame.state); //} - final DocsEnum docsEnum = postingsReader.docs(fieldInfo, currentFrame.state, skipDocs, reuse); - - assert docsEnum != null; - return docsEnum; + return postingsReader.docs(fieldInfo, currentFrame.state, skipDocs, reuse, needsFreqs); } @Override Index: lucene/src/java/org/apache/lucene/index/MultiFields.java =================================================================== --- lucene/src/java/org/apache/lucene/index/MultiFields.java (revision 1209915) +++ lucene/src/java/org/apache/lucene/index/MultiFields.java (working copy) @@ -151,14 +151,14 @@ /** Returns {@link DocsEnum} for the specified field & * term. This may return null if the term does not * exist. */ - public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException { + public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException { assert field != null; assert term != null; final Terms terms = getTerms(r, field); if (terms != null) { final TermsEnum termsEnum = terms.iterator(null); if (termsEnum.seekExact(term, true)) { - return termsEnum.docs(liveDocs, null); + return termsEnum.docs(liveDocs, null, needsFreqs); } } return null; Index: lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java =================================================================== --- lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java (revision 1209915) +++ lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java (working copy) @@ -383,7 +383,7 @@ } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) { return new RAMDocsEnum(ramField.termToDocs.get(current), liveDocs); } Index: lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java =================================================================== --- lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java (revision 1209915) +++ lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java (working copy) @@ -37,12 +37,18 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.ConcurrentMergeScheduler; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.LogMergePolicy; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.MergeScheduler; +import org.apache.lucene.index.MultiFields; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.PostingsFormat; @@ -520,4 +526,51 @@ return doc2; } + + // Returns a DocsEnum, but randomly sometimes uses a + // DocsAndFreqsEnum, DocsAndPositionsEnum. Returns null + // if field/term doesn't exist: + public static DocsEnum docs(Random random, IndexReader r, String field, BytesRef term, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { + final Terms terms = MultiFields.getTerms(r, field); + if (terms == null) { + return null; + } + final TermsEnum termsEnum = terms.iterator(null); + if (!termsEnum.seekExact(term, random.nextBoolean())) { + return null; + } + if (random.nextBoolean()) { + if (random.nextBoolean()) { + // TODO: cast re-use to D&PE if we can...? + final DocsAndPositionsEnum docsAndPositions = termsEnum.docsAndPositions(liveDocs, null); + if (docsAndPositions != null) { + return docsAndPositions; + } + } + final DocsEnum docsAndFreqs = termsEnum.docs(liveDocs, reuse, true); + if (docsAndFreqs != null) { + return docsAndFreqs; + } + } + return termsEnum.docs(liveDocs, reuse, needsFreqs); + } + + // Returns a DocsEnum from a positioned TermsEnum, but + // randomly sometimes uses a DocsAndFreqsEnum, DocsAndPositionsEnum. + public static DocsEnum docs(Random random, TermsEnum termsEnum, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException { + if (random.nextBoolean()) { + if (random.nextBoolean()) { + // TODO: cast re-use to D&PE if we can...? + final DocsAndPositionsEnum docsAndPositions = termsEnum.docsAndPositions(liveDocs, null); + if (docsAndPositions != null) { + return docsAndPositions; + } + } + final DocsEnum docsAndFreqs = termsEnum.docs(liveDocs, null, true); + if (docsAndFreqs != null) { + return docsAndFreqs; + } + } + return termsEnum.docs(liveDocs, null, needsFreqs); + } } Index: lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (revision 1209915) +++ lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (working copy) @@ -29,11 +29,11 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum.SeekStatus; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -139,7 +139,7 @@ assertEquals(SeekStatus.FOUND, te.seekCeil(new BytesRef("lazy"))); assertEquals(SeekStatus.FOUND, te.seekCeil(new BytesRef("dog"))); assertEquals(SeekStatus.FOUND, te.seekCeil(new BytesRef("the"))); - DocsEnum de = te.docs(null, null); + DocsEnum de = te.docs(null, null, true); assertTrue(de.advance(0) != DocsEnum.NO_MORE_DOCS); assertEquals(2, de.freq()); assertTrue(de.advance(1) != DocsEnum.NO_MORE_DOCS); Index: lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java (revision 1209915) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/FieldNormModifier.java (working copy) @@ -129,12 +129,19 @@ if (terms != null) { TermsEnum termsEnum = terms.iterator(null); DocsEnum docs = null; + DocsEnum docsAndFreqs = null; while(termsEnum.next() != null) { - docs = termsEnum.docs(liveDocs, docs); + docsAndFreqs = termsEnum.docs(liveDocs, docsAndFreqs, true); + final DocsEnum docs2; + if (docsAndFreqs != null) { + docs2 = docsAndFreqs; + } else { + docs2 = docs = termsEnum.docs(liveDocs, docs, false); + } while(true) { - int docID = docs.nextDoc(); + int docID = docs2.nextDoc(); if (docID != docs.NO_MORE_DOCS) { - termCounts[docID] += docs.freq(); + termCounts[docID] += docsAndFreqs == null ? 1 : docsAndFreqs.freq(); } else { break; } Index: lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java (revision 1209915) +++ lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java (working copy) @@ -201,7 +201,7 @@ return; } } - DocsEnum de = r.termDocsEnum(liveDocs, field, termText); + DocsEnum de = r.termDocsEnum(liveDocs, field, termText, true); if (de != null) { while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) totalTF[0] += de.freq(); Index: lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java =================================================================== --- lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java (revision 1209915) +++ lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java (working copy) @@ -57,7 +57,7 @@ return new DocIdSet() { @Override public DocIdSetIterator iterator() throws IOException { - return context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef); + return context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef, false); } @Override @@ -70,7 +70,7 @@ for (int i =0; i< sz; i++) { double boxId = area.get(i).doubleValue(); NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(boxId), 0, bytesRef); - final DocsEnum docsEnum = context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef); + final DocsEnum docsEnum = context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef, false); if (docsEnum == null) continue; // iterate through all documents // which have this boxId Index: lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java =================================================================== --- lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java (revision 1209915) +++ lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java (working copy) @@ -17,6 +17,9 @@ * limitations under the License. */ +import java.io.IOException; +import java.util.HashSet; + import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.StringField; @@ -28,10 +31,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; -import java.io.IOException; -import java.util.HashSet; - public class DuplicateFilterTest extends LuceneTestCase { private static final String KEY_FIELD = "url"; private Directory directory; @@ -134,10 +135,13 @@ for (ScoreDoc hit : hits) { Document d = searcher.doc(hit.doc); String url = d.get(KEY_FIELD); - DocsEnum td = MultiFields.getTermDocsEnum(reader, - MultiFields.getLiveDocs(reader), - KEY_FIELD, - new BytesRef(url)); + DocsEnum td = _TestUtil.docs(random, reader, + KEY_FIELD, + new BytesRef(url), + MultiFields.getLiveDocs(reader), + null, + false); + int lastDoc = 0; while (td.nextDoc() != DocsEnum.NO_MORE_DOCS) { lastDoc = td.docID(); @@ -155,10 +159,13 @@ for (ScoreDoc hit : hits) { Document d = searcher.doc(hit.doc); String url = d.get(KEY_FIELD); - DocsEnum td = MultiFields.getTermDocsEnum(reader, - MultiFields.getLiveDocs(reader), - KEY_FIELD, - new BytesRef(url)); + DocsEnum td = _TestUtil.docs(random, reader, + KEY_FIELD, + new BytesRef(url), + MultiFields.getLiveDocs(reader), + null, + false); + int lastDoc = 0; td.nextDoc(); lastDoc = td.docID(); Index: lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java =================================================================== --- lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java (revision 1209915) +++ lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java (working copy) @@ -93,7 +93,7 @@ if (currTerm == null) { break; } else { - docs = termsEnum.docs(acceptDocs, docs); + docs = termsEnum.docs(acceptDocs, docs, false); int doc = docs.nextDoc(); if (doc != DocsEnum.NO_MORE_DOCS) { if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) { @@ -133,7 +133,7 @@ } else { if (termsEnum.docFreq() > 1) { // unset potential duplicates - docs = termsEnum.docs(acceptDocs, docs); + docs = termsEnum.docs(acceptDocs, docs, false); int doc = docs.nextDoc(); if (doc != DocsEnum.NO_MORE_DOCS) { if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) { Index: lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java =================================================================== --- lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (revision 1209915) +++ lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (working copy) @@ -188,7 +188,7 @@ MemoryIndex memory = new MemoryIndex(); memory.addField("foo", "bar", analyzer); IndexReader reader = memory.createSearcher().getIndexReader(); - DocsEnum disi = reader.termDocsEnum(null, "foo", new BytesRef("bar")); + DocsEnum disi = _TestUtil.docs(random, reader, "foo", new BytesRef("bar"), null, null, false); int docid = disi.docID(); assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); @@ -196,7 +196,7 @@ // now reuse and check again TermsEnum te = reader.terms("foo").iterator(null); assertTrue(te.seekExact(new BytesRef("bar"), true)); - disi = te.docs(null, disi); + disi = te.docs(null, disi, false); docid = disi.docID(); assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS); assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS); Index: lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java =================================================================== --- lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 1209915) +++ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy) @@ -936,7 +936,7 @@ } @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) { + public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) { if (reuse == null || !(reuse instanceof MemoryDocsEnum)) { reuse = new MemoryDocsEnum(); } Index: solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java =================================================================== --- solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java (revision 1209915) +++ solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java (working copy) @@ -17,6 +17,14 @@ package org.apache.solr.search; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -25,6 +33,7 @@ import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util._TestUtil; import org.apache.noggit.ObjectBuilder; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.common.SolrException; @@ -32,14 +41,6 @@ import org.junit.BeforeClass; import org.junit.Test; -import java.io.IOException; -import java.util.*; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - import static org.apache.solr.core.SolrCore.verbose; public class TestRealTimeGet extends SolrTestCaseJ4 { @@ -727,7 +728,7 @@ if (!termsEnum.seekExact(termBytes, false)) { return -1; } - DocsEnum docs = termsEnum.docs(MultiFields.getLiveDocs(r), null); + DocsEnum docs = _TestUtil.docs(random, termsEnum, MultiFields.getLiveDocs(r), null, false); int id = docs.nextDoc(); if (id != DocIdSetIterator.NO_MORE_DOCS) { int next = docs.nextDoc(); Index: solr/core/src/java/org/apache/solr/request/SimpleFacets.java =================================================================== --- solr/core/src/java/org/apache/solr/request/SimpleFacets.java (revision 1209915) +++ solr/core/src/java/org/apache/solr/request/SimpleFacets.java (working copy) @@ -687,7 +687,7 @@ // TODO: specialize when base docset is a bitset or hash set (skipDocs)? or does it matter for this? // TODO: do this per-segment for better efficiency (MultiDocsEnum just uses base class impl) // TODO: would passing deleted docs lead to better efficiency over checking the fastForRandomSet? - docsEnum = termsEnum.docs(null, docsEnum); + docsEnum = termsEnum.docs(null, docsEnum, false); c=0; if (docsEnum instanceof MultiDocsEnum) { Index: solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java =================================================================== --- solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (revision 1209915) +++ solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java (working copy) @@ -314,7 +314,7 @@ if (freq < minDocFreqFrom) { fromTermDirectCount++; // OK to skip liveDocs, since we check for intersection with docs matching query - fromDeState.docsEnum = fromDeState.termsEnum.docs(null, fromDeState.docsEnum); + fromDeState.docsEnum = fromDeState.termsEnum.docs(null, fromDeState.docsEnum, false); DocsEnum docsEnum = fromDeState.docsEnum; if (docsEnum instanceof MultiDocsEnum) { @@ -379,7 +379,7 @@ toTermDirectCount++; // need to use liveDocs here so we don't map to any deleted ones - toDeState.docsEnum = toDeState.termsEnum.docs(toDeState.liveDocs, toDeState.docsEnum); + toDeState.docsEnum = toDeState.termsEnum.docs(toDeState.liveDocs, toDeState.docsEnum, false); DocsEnum docsEnum = toDeState.docsEnum; if (docsEnum instanceof MultiDocsEnum) { Index: solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java =================================================================== --- solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java (revision 1209915) +++ solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java (working copy) @@ -277,7 +277,7 @@ continue; } - docsEnum = termsEnum.docs(null, docsEnum); + docsEnum = termsEnum.docs(null, docsEnum, false); int doc; while ((doc = docsEnum.nextDoc()) != DocsEnum.NO_MORE_DOCS) { vals[doc] = fval; Index: solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java =================================================================== --- solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (revision 1209915) +++ solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (working copy) @@ -559,7 +559,7 @@ if (!termsEnum.seekExact(termBytes, false)) { return -1; } - DocsEnum docs = termsEnum.docs(MultiFields.getLiveDocs(reader), null); + DocsEnum docs = termsEnum.docs(MultiFields.getLiveDocs(reader), null, false); if (docs == null) return -1; int id = docs.nextDoc(); return id == DocIdSetIterator.NO_MORE_DOCS ? -1 : id; @@ -861,7 +861,7 @@ int bitsSet = 0; OpenBitSet obs = null; - DocsEnum docsEnum = deState.termsEnum.docs(deState.liveDocs, deState.docsEnum); + DocsEnum docsEnum = deState.termsEnum.docs(deState.liveDocs, deState.docsEnum, false); if (deState.docsEnum == null) { deState.docsEnum = docsEnum; } @@ -942,7 +942,7 @@ if (terms != null) { final TermsEnum termsEnum = terms.iterator(null); if (termsEnum.seekExact(termBytes, false)) { - docsEnum = termsEnum.docs(MultiFields.getLiveDocs(reader), null); + docsEnum = termsEnum.docs(MultiFields.getLiveDocs(reader), null, false); } }