Index: lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java (revision 1291880) +++ lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java (working copy) @@ -58,6 +58,7 @@ private final int maxDoc, numDocs; private final boolean hasDeletions; private final SortedMap fieldToReader = new TreeMap(); + private final SortedMap tvFieldToReader = new TreeMap(); /** Create a ParallelAtomicReader based on the provided * readers; auto-closes the given readers on {@link #close()}. */ @@ -98,20 +99,36 @@ throw new IllegalArgumentException("All readers must have same maxDoc: "+maxDoc+"!="+reader.maxDoc()); } } - + + // build FieldInfos and fieldToReader map: for (final AtomicReader reader : this.parallelReaders) { final FieldInfos readerFieldInfos = reader.getFieldInfos(); - for(FieldInfo fieldInfo : readerFieldInfos) { // update fieldToReader map + for (FieldInfo fieldInfo : readerFieldInfos) { // NOTE: first reader having a given field "wins": if (!fieldToReader.containsKey(fieldInfo.name)) { fieldInfos.add(fieldInfo); fieldToReader.put(fieldInfo.name, reader); - if (fieldInfo.isIndexed) { - this.fields.addField(fieldInfo.name, reader.terms(fieldInfo.name)); + if (fieldInfo.storeTermVector) { + tvFieldToReader.put(fieldInfo.name, reader); } } } - } + } + + // build Fields instance + for (final AtomicReader reader : this.parallelReaders) { + final Fields readerFields = reader.fields(); + if (readerFields != null) { + final FieldsEnum it = readerFields.iterator(); + String name; + while ((name = it.next()) != null) { + // only add if the reader responsible for that field name is the current: + if (fieldToReader.get(name) == reader) { + this.fields.addField(name, it.terms()); + } + } + } + } // do this finally so any Exceptions occurred before don't affect refcounts: if (!closeSubReaders) { @@ -238,7 +255,7 @@ public Fields getTermVectors(int docID) throws IOException { ensureOpen(); ParallelFields fields = null; - for (Map.Entry ent : fieldToReader.entrySet()) { + for (Map.Entry ent : tvFieldToReader.entrySet()) { String fieldName = ent.getKey(); Terms vector = ent.getValue().getTermVector(docID, fieldName); if (vector != null) { Index: lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java (revision 1291880) +++ lucene/core/src/test/org/apache/lucene/index/TestParallelTermEnum.java (working copy) @@ -24,178 +24,87 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.store.Directory; import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; public class TestParallelTermEnum extends LuceneTestCase { - private AtomicReader ir1; - private AtomicReader ir2; - private Directory rd1; - private Directory rd2; - - @Override - public void setUp() throws Exception { - super.setUp(); - Document doc; - rd1 = newDirectory(); - IndexWriter iw1 = new IndexWriter(rd1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); + private AtomicReader ir1; + private AtomicReader ir2; + private Directory rd1; + private Directory rd2; + + @Override + public void setUp() throws Exception { + super.setUp(); + Document doc; + rd1 = newDirectory(); + IndexWriter iw1 = new IndexWriter(rd1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); - doc = new Document(); - doc.add(newField("field1", "the quick brown fox jumps", TextField.TYPE_STORED)); - doc.add(newField("field2", "the quick brown fox jumps", TextField.TYPE_STORED)); - doc.add(newField("field4", "", TextField.TYPE_UNSTORED)); - iw1.addDocument(doc); + doc = new Document(); + doc.add(newField("field1", "the quick brown fox jumps", TextField.TYPE_STORED)); + doc.add(newField("field2", "the quick brown fox jumps", TextField.TYPE_STORED)); + iw1.addDocument(doc); - iw1.close(); - rd2 = newDirectory(); - IndexWriter iw2 = new IndexWriter(rd2, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); + iw1.close(); + rd2 = newDirectory(); + IndexWriter iw2 = new IndexWriter(rd2, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); - doc = new Document(); - doc.add(newField("field0", "", TextField.TYPE_UNSTORED)); - doc.add(newField("field1", "the fox jumps over the lazy dog", TextField.TYPE_STORED)); - doc.add(newField("field3", "the fox jumps over the lazy dog", TextField.TYPE_STORED)); - iw2.addDocument(doc); + doc = new Document(); + doc.add(newField("field1", "the fox jumps over the lazy dog", TextField.TYPE_STORED)); + doc.add(newField("field3", "the fox jumps over the lazy dog", TextField.TYPE_STORED)); + iw2.addDocument(doc); - iw2.close(); + iw2.close(); - this.ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(rd1)); - this.ir2 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(rd2)); - } + this.ir1 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(rd1)); + this.ir2 = SlowCompositeReaderWrapper.wrap(DirectoryReader.open(rd2)); + } - @Override - public void tearDown() throws Exception { - ir1.close(); - ir2.close(); - rd1.close(); - rd2.close(); - super.tearDown(); + @Override + public void tearDown() throws Exception { + ir1.close(); + ir2.close(); + rd1.close(); + rd2.close(); + super.tearDown(); + } + + private void checkTerms(Terms terms, Bits liveDocs, String... termsList) throws IOException { + assertNotNull(terms); + final TermsEnum te = terms.iterator(null); + + for (String t : termsList) { + BytesRef b = te.next(); + assertNotNull(b); + assertEquals(t, b.utf8ToString()); + DocsEnum td = _TestUtil.docs(random, te, liveDocs, null, false); + assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); + assertEquals(0, td.docID()); + assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); } + assertNull(te.next()); + } - public void test1() throws IOException { - ParallelAtomicReader pr = new ParallelAtomicReader(ir1, ir2); + public void test1() throws IOException { + ParallelAtomicReader pr = new ParallelAtomicReader(ir1, ir2); - Bits liveDocs = pr.getLiveDocs(); + Bits liveDocs = pr.getLiveDocs(); - FieldsEnum fe = pr.fields().iterator(); + FieldsEnum fe = pr.fields().iterator(); - String f = fe.next(); - assertEquals("field0", f); - f = fe.next(); - assertEquals("field1", f); + String f = fe.next(); + assertEquals("field1", f); + checkTerms(fe.terms(), liveDocs, "brown", "fox", "jumps", "quick", "the"); - Terms terms = fe.terms(); - TermsEnum te = terms.iterator(null); + f = fe.next(); + assertEquals("field2", f); + checkTerms(fe.terms(), liveDocs, "brown", "fox", "jumps", "quick", "the"); - assertEquals("brown", te.next().utf8ToString()); - DocsEnum td = _TestUtil.docs(random, te, liveDocs, null, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); + f = fe.next(); + assertEquals("field3", f); + checkTerms(fe.terms(), liveDocs, "dog", "fox", "jumps", "lazy", "over", "the"); - assertEquals("fox", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("jumps", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("quick", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("the", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertNull(te.next()); - f = fe.next(); - assertEquals("field2", f); - terms = fe.terms(); - assertNotNull(terms); - te = terms.iterator(null); - - assertEquals("brown", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("fox", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("jumps", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("quick", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("the", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertNull(te.next()); - f = fe.next(); - assertEquals("field3", f); - terms = fe.terms(); - assertNotNull(terms); - te = terms.iterator(null); - - assertEquals("dog", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("fox", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("jumps", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("lazy", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("over", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertEquals("the", te.next().utf8ToString()); - td = _TestUtil.docs(random, te, liveDocs, td, false); - assertTrue(td.nextDoc() != DocsEnum.NO_MORE_DOCS); - assertEquals(0, td.docID()); - assertEquals(td.nextDoc(), DocsEnum.NO_MORE_DOCS); - - assertNull(te.next()); - } + assertNull(fe.next()); + } }