Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1475961) +++ lucene/CHANGES.txt (working copy) @@ -46,6 +46,10 @@ if you had a 64-bit JVM without compressed OOPS: IBM J9, or Oracle with large heap/explicitly disabled. (Mike McCandless, Uwe Schindler, Robert Muir) +* LUCENE-4953: Fixed ParallelCompositeReader to inform ReaderClosedListeners of + its synthetic subreaders. FieldCaches keyed on the atomic childs will purged + earlier and FC insanity prevented. (Mike McCandless, Uwe Schindler) + Optimizations * LUCENE-4938: Don't use an unnecessarily large priority queue in IndexSearcher Index: lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java (revision 1475961) +++ lucene/core/src/java/org/apache/lucene/index/ParallelAtomicReader.java (working copy) @@ -27,6 +27,7 @@ import java.util.TreeMap; import org.apache.lucene.util.Bits; +import org.apache.lucene.index.ParallelCompositeReader.CloseMode; /** An {@link AtomicReader} which reads multiple, parallel indexes. Each index @@ -53,7 +54,7 @@ private final AtomicReader[] parallelReaders, storedFieldsReaders; private final Set completeReaderSet = Collections.newSetFromMap(new IdentityHashMap()); - private final boolean closeSubReaders; + private final CloseMode closeMode; private final int maxDoc, numDocs; private final boolean hasDeletions; private final SortedMap fieldToReader = new TreeMap(); @@ -68,14 +69,19 @@ /** Create a ParallelAtomicReader based on the provided * readers. */ public ParallelAtomicReader(boolean closeSubReaders, AtomicReader... readers) throws IOException { - this(closeSubReaders, readers, readers); + this(closeSubReaders ? CloseMode.CLOSE_SUBREADERS : CloseMode.DECREF_SUBREADERS, readers, readers); } /** Expert: create a ParallelAtomicReader based on the provided * readers and storedFieldReaders; when a document is * loaded, only storedFieldsReaders will be used. */ public ParallelAtomicReader(boolean closeSubReaders, AtomicReader[] readers, AtomicReader[] storedFieldsReaders) throws IOException { - this.closeSubReaders = closeSubReaders; + this(closeSubReaders ? CloseMode.CLOSE_SUBREADERS : CloseMode.DECREF_SUBREADERS, readers, storedFieldsReaders); + } + + /** Private ctor for {@link ParallelCompositeReader} to create synthetic readers without incref or close. */ + ParallelAtomicReader(CloseMode closeMode, AtomicReader[] readers, AtomicReader[] storedFieldsReaders) throws IOException { + this.closeMode = closeMode; if (readers.length == 0 && storedFieldsReaders.length > 0) throw new IllegalArgumentException("There must be at least one main reader if storedFieldsReaders are used."); this.parallelReaders = readers.clone(); @@ -132,7 +138,7 @@ // do this finally so any Exceptions occurred before don't affect refcounts: for (AtomicReader reader : completeReaderSet) { - if (!closeSubReaders) { + if (closeMode == CloseMode.DECREF_SUBREADERS) { reader.incRef(); } reader.registerParentReader(this); @@ -241,10 +247,12 @@ @Override protected synchronized void doClose() throws IOException { + if (closeMode == CloseMode.IGNORE_SUBREADERS) + return; IOException ioe = null; for (AtomicReader reader : completeReaderSet) { try { - if (closeSubReaders) { + if (closeMode == CloseMode.CLOSE_SUBREADERS) { reader.close(); } else { reader.decRef(); Index: lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java (revision 1475961) +++ lucene/core/src/java/org/apache/lucene/index/ParallelCompositeReader.java (working copy) @@ -48,7 +48,9 @@ * it might happen that the segment structure of your index is no longer predictable. */ public final class ParallelCompositeReader extends BaseCompositeReader { - private final boolean closeSubReaders; + static enum CloseMode { CLOSE_SUBREADERS, DECREF_SUBREADERS, IGNORE_SUBREADERS }; + + private final CloseMode closeMode; private final Set completeReaderSet = Collections.newSetFromMap(new IdentityHashMap()); @@ -61,22 +63,28 @@ /** Create a ParallelCompositeReader based on the provided * readers. */ public ParallelCompositeReader(boolean closeSubReaders, CompositeReader... readers) throws IOException { - this(closeSubReaders, readers, readers); + this(closeSubReaders ? CloseMode.CLOSE_SUBREADERS : CloseMode.DECREF_SUBREADERS, readers, readers); } /** Expert: create a ParallelCompositeReader based on the provided * readers and storedFieldReaders; when a document is * loaded, only storedFieldsReaders will be used. */ - public ParallelCompositeReader(boolean closeSubReaders, CompositeReader[] readers, CompositeReader[] storedFieldReaders) throws IOException { + public ParallelCompositeReader(boolean closeSubReaders, CompositeReader[] readers, CompositeReader[] storedFieldsReaders) throws IOException { + this(closeSubReaders ? CloseMode.CLOSE_SUBREADERS : CloseMode.DECREF_SUBREADERS, readers, storedFieldsReaders); + } + + /** Private ctor for {@link ParallelCompositeReader} to create synthetic readers without incref or close. */ + ParallelCompositeReader(CloseMode closeMode, CompositeReader[] readers, CompositeReader[] storedFieldReaders) throws IOException { super(prepareSubReaders(readers, storedFieldReaders)); - this.closeSubReaders = closeSubReaders; + this.closeMode = closeMode; Collections.addAll(completeReaderSet, readers); Collections.addAll(completeReaderSet, storedFieldReaders); // do this finally so any Exceptions occurred before don't affect refcounts: - if (!closeSubReaders) { - for (CompositeReader reader : completeReaderSet) { + for (CompositeReader reader : completeReaderSet) { + if (closeMode == CloseMode.DECREF_SUBREADERS) { reader.incRef(); } + reader.registerParentReader(this); } } @@ -112,10 +120,7 @@ for (int j = 0; j < storedFieldsReaders.length; j++) { storedSubs[j] = (AtomicReader) storedFieldsReaders[j].getSequentialSubReaders().get(i); } - // we simply enable closing of subReaders, to prevent incRefs on subReaders - // -> for synthetic subReaders, close() is never - // called by our doClose() - subReaders[i] = new ParallelAtomicReader(true, atomicSubs, storedSubs); + subReaders[i] = new ParallelAtomicReader(CloseMode.IGNORE_SUBREADERS, atomicSubs, storedSubs); } else { assert firstSubReaders.get(i) instanceof CompositeReader; final CompositeReader[] compositeSubs = new CompositeReader[readers.length]; @@ -126,9 +131,7 @@ for (int j = 0; j < storedFieldsReaders.length; j++) { storedSubs[j] = (CompositeReader) storedFieldsReaders[j].getSequentialSubReaders().get(i); } - // we simply enable closing of subReaders, to prevent incRefs on subReaders - // -> for synthetic subReaders, close() is never called by our doClose() - subReaders[i] = new ParallelCompositeReader(true, compositeSubs, storedSubs); + subReaders[i] = new ParallelCompositeReader(CloseMode.IGNORE_SUBREADERS, compositeSubs, storedSubs); } } return subReaders; @@ -170,10 +173,21 @@ @Override protected synchronized void doClose() throws IOException { + if (closeMode == CloseMode.IGNORE_SUBREADERS) + return; IOException ioe = null; + // Notify readerClosedListeners of readers we created in prepareReaders: + for (IndexReader reader : getSequentialSubReaders()) { + try { + reader.close(); + } catch (IOException e) { + if (ioe == null) ioe = e; + } + } + // close subreaders / decRef them for (final CompositeReader reader : completeReaderSet) { try { - if (closeSubReaders) { + if (closeMode == CloseMode.CLOSE_SUBREADERS) { reader.close(); } else { reader.decRef(); Index: lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java (revision 1475961) +++ lucene/core/src/test/org/apache/lucene/index/TestParallelCompositeReader.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexReader.ReaderClosedListener; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; @@ -277,6 +278,33 @@ dir2.close(); } + public void testReaderClosedListener() throws Exception { + Directory dir1 = getDir1(random()); + CompositeReader ir1 = DirectoryReader.open(dir1); + + // with overlapping + ParallelCompositeReader pr = new ParallelCompositeReader(false, + new CompositeReader[] {ir1}, + new CompositeReader[] {ir1}); + + final int[] listenerClosedCount = new int[1]; + + assertEquals(3, pr.leaves().size()); + + for(AtomicReaderContext cxt : pr.leaves()) { + cxt.reader().addReaderClosedListener(new ReaderClosedListener() { + @Override + public void onClose(IndexReader reader) { + listenerClosedCount[0]++; + } + }); + } + pr.close(); + ir1.close(); + assertEquals(3, listenerClosedCount[0]); + dir1.close(); + } + private void queryTest(Query query) throws IOException { ScoreDoc[] parallelHits = parallel.search(query, null, 1000).scoreDocs; ScoreDoc[] singleHits = single.search(query, null, 1000).scoreDocs;