Index: lucene/src/test/org/apache/lucene/TestExternalCodecs.java =================================================================== --- lucene/src/test/org/apache/lucene/TestExternalCodecs.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/TestExternalCodecs.java (working copy) @@ -58,6 +58,10 @@ public void testPerFieldCodec() throws Exception { final int NUM_DOCS = atLeast(173); + if (VERBOSE) { + System.out.println("TEST: NUM_DOCS=" + NUM_DOCS); + } + MockDirectoryWrapper dir = newDirectory(); dir.setCheckIndexOnClose(false); // we use a custom codec provider IndexWriter w = new IndexWriter( @@ -101,7 +105,14 @@ System.out.println("\nTEST: now delete 2nd doc"); } w.deleteDocuments(new Term("id", "44")); + + if (VERBOSE) { + System.out.println("\nTEST: now force merge"); + } w.forceMerge(1); + if (VERBOSE) { + System.out.println("\nTEST: now open reader"); + } r = IndexReader.open(w, true); assertEquals(NUM_DOCS-2, r.maxDoc()); assertEquals(NUM_DOCS-2, r.numDocs()); @@ -112,6 +123,9 @@ assertEquals(0, s.search(new TermQuery(new Term("id", "77")), 1).totalHits); assertEquals(0, s.search(new TermQuery(new Term("id", "44")), 1).totalHits); + if (VERBOSE) { + System.out.println("\nTEST: now close NRT reader"); + } r.close(); w.close(); Index: lucene/src/test/org/apache/lucene/index/TestDoc.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDoc.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/index/TestDoc.java (working copy) @@ -198,8 +198,8 @@ private SegmentInfo merge(Directory dir, SegmentInfo si1, SegmentInfo si2, String merged, boolean useCompoundFile) throws Exception { IOContext context = newIOContext(random); - SegmentReader r1 = SegmentReader.get(si1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, context); - SegmentReader r2 = SegmentReader.get(si2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, context); + SegmentReader r1 = new SegmentReader(si1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, context); + SegmentReader r2 = new SegmentReader(si2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, context); final Codec codec = Codec.getDefault(); SegmentMerger merger = new SegmentMerger(InfoStream.getDefault(), si1.dir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, merged, MergeState.CheckAbort.NONE, null, new FieldInfos(new FieldInfos.FieldNumberBiMap()), codec, context); @@ -226,7 +226,7 @@ private void printSegment(PrintWriter out, SegmentInfo si) throws Exception { - SegmentReader reader = SegmentReader.get(si, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + SegmentReader reader = new SegmentReader(si, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); for (int i = 0; i < reader.numDocs(); i++) out.println(reader.document(i)); Index: lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (working copy) @@ -64,7 +64,7 @@ SegmentInfo info = writer.newestSegment(); writer.close(); //After adding the document, we should be able to read it back in - SegmentReader reader = SegmentReader.get(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + SegmentReader reader = new SegmentReader(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); assertTrue(reader != null); Document doc = reader.document(0); assertTrue(doc != null); @@ -94,7 +94,7 @@ // test that the norms are not present in the segment if // omitNorms is true - for (FieldInfo fi : reader.core.fieldInfos) { + for (FieldInfo fi : reader.fieldInfos()) { if (fi.isIndexed) { assertTrue(fi.omitNorms == !reader.hasNorms(fi.name)); } @@ -125,7 +125,7 @@ writer.commit(); SegmentInfo info = writer.newestSegment(); writer.close(); - SegmentReader reader = SegmentReader.get(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + SegmentReader reader = new SegmentReader(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, MultiFields.getLiveDocs(reader), "repeated", new BytesRef("repeated")); @@ -197,7 +197,7 @@ writer.commit(); SegmentInfo info = writer.newestSegment(); writer.close(); - SegmentReader reader = SegmentReader.get(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + SegmentReader reader = new SegmentReader(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, reader.getLiveDocs(), "f1", new BytesRef("a")); assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS); @@ -241,7 +241,7 @@ writer.commit(); SegmentInfo info = writer.newestSegment(); writer.close(); - SegmentReader reader = SegmentReader.get(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + SegmentReader reader = new SegmentReader(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); DocsAndPositionsEnum termPositions = reader.termPositionsEnum(reader.getLiveDocs(), "preanalyzed", new BytesRef("term1")); assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS); Index: lucene/src/test/org/apache/lucene/index/TestAddIndexes.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (working copy) @@ -443,6 +443,9 @@ setMergePolicy(newLogMergePolicy(4)) ); + if (VERBOSE) { + System.out.println("\nTEST: now addIndexes"); + } writer.addIndexes(aux, new MockDirectoryWrapper(random, new RAMDirectory(aux, newIOContext(random)))); assertEquals(1020, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); Index: lucene/src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReader.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -717,36 +717,6 @@ dir.close(); } - // LUCENE-1579: Ensure that on a cloned reader, segments - // reuse the doc values arrays in FieldCache - public void testFieldCacheReuseAfterClone() throws Exception { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); - Document doc = new Document(); - doc.add(newField("number", "17", StringField.TYPE_UNSTORED)); - writer.addDocument(doc); - writer.close(); - - // Open reader - IndexReader r = getOnlySegmentReader(IndexReader.open(dir)); - final int[] ints = FieldCache.DEFAULT.getInts(r, "number", false); - assertEquals(1, ints.length); - assertEquals(17, ints[0]); - - // Clone reader - IndexReader r2 = (IndexReader) r.clone(); - r.close(); - assertTrue(r2 != r); - final int[] ints2 = FieldCache.DEFAULT.getInts(r2, "number", false); - r2.close(); - - assertEquals(1, ints2.length); - assertEquals(17, ints2[0]); - assertTrue(ints == ints2); - - dir.close(); - } - // LUCENE-1579: Ensure that on a reopened reader, that any // shared segments reuse the doc values arrays in // FieldCache Index: lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java (working copy) @@ -56,7 +56,7 @@ public void testTermDocs(int indexDivisor) throws IOException { //After adding the document, we should be able to read it back in - SegmentReader reader = SegmentReader.get(info, indexDivisor, newIOContext(random)); + SegmentReader reader = new SegmentReader(info, indexDivisor, newIOContext(random)); assertTrue(reader != null); assertEquals(indexDivisor, reader.getTermInfosIndexDivisor()); @@ -79,7 +79,7 @@ public void testBadSeek(int indexDivisor) throws IOException { { //After adding the document, we should be able to read it back in - SegmentReader reader = SegmentReader.get(info, indexDivisor, newIOContext(random)); + SegmentReader reader = new SegmentReader(info, indexDivisor, newIOContext(random)); assertTrue(reader != null); DocsEnum termDocs = _TestUtil.docs(random, reader, "textField2", @@ -93,7 +93,7 @@ } { //After adding the document, we should be able to read it back in - SegmentReader reader = SegmentReader.get(info, indexDivisor, newIOContext(random)); + SegmentReader reader = new SegmentReader(info, indexDivisor, newIOContext(random)); assertTrue(reader != null); DocsEnum termDocs = _TestUtil.docs(random, reader, "junk", Index: lucene/src/test/org/apache/lucene/index/TestMultiReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestMultiReader.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/index/TestMultiReader.java (working copy) @@ -38,8 +38,8 @@ IndexReader reader; sis.read(dir); - SegmentReader reader1 = SegmentReader.get(sis.info(0), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); - SegmentReader reader2 = SegmentReader.get(sis.info(1), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + SegmentReader reader1 = new SegmentReader(sis.info(0), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + SegmentReader reader2 = new SegmentReader(sis.info(1), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); readers[0] = reader1; readers[1] = reader2; assertTrue(reader1 != null); Index: lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (working copy) @@ -24,7 +24,6 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.InfoStream; @@ -54,8 +53,8 @@ SegmentInfo info1 = DocHelper.writeDoc(random, merge1Dir, doc1); DocHelper.setupDoc(doc2); SegmentInfo info2 = DocHelper.writeDoc(random, merge2Dir, doc2); - reader1 = SegmentReader.get(info1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); - reader2 = SegmentReader.get(info2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + reader1 = new SegmentReader(info1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + reader2 = new SegmentReader(info2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); } @Override @@ -86,9 +85,9 @@ assertTrue(docsMerged == 2); final FieldInfos fieldInfos = mergeState.fieldInfos; //Should be able to open a new SegmentReader against the new directory - SegmentReader mergedReader = SegmentReader.getRW(new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, + SegmentReader mergedReader = new SegmentReader(new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, codec, fieldInfos), - true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); + IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); assertTrue(mergedReader != null); assertTrue(mergedReader.numDocs() == 2); Document newDoc1 = mergedReader.document(0); Index: lucene/src/test/org/apache/lucene/index/TestTieredMergePolicy.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestTieredMergePolicy.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/index/TestTieredMergePolicy.java (working copy) @@ -132,6 +132,10 @@ assertEquals(numDocs, r.numDocs()); r.close(); + if (VERBOSE) { + System.out.println("\nTEST: delete doc"); + } + w.deleteDocuments(new Term("id", ""+(42+17))); r = w.getReader(); Index: lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderClone.java (working copy) @@ -1,86 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.TextField; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; - -/** - * Tests cloning multiple types of readers, modifying the liveDocs and norms - * and verifies copy on write semantics of the liveDocs and norms is - * implemented properly - */ -public class TestIndexReaderClone extends LuceneTestCase { - - public void testDirectoryReader() throws Exception { - final Directory dir = createIndex(0); - performDefaultTests(IndexReader.open(dir)); - dir.close(); - } - - public void testMultiReader() throws Exception { - final Directory dir1 = createIndex(0); - final IndexReader r1 = IndexReader.open(dir1); - final Directory dir2 = createIndex(0); - final IndexReader r2 = IndexReader.open(dir2); - final MultiReader mr = new MultiReader(r1, r2); - performDefaultTests(mr); - dir1.close(); - dir2.close(); - } - - public void testParallelReader() throws Exception { - final Directory dir1 = createIndex(0); - final IndexReader r1 = IndexReader.open(dir1); - final Directory dir2 = createIndex(1); - final IndexReader r2 = IndexReader.open(dir2); - final ParallelReader pr = new ParallelReader(); - pr.add(r1); - pr.add(r2); - performDefaultTests(pr); - dir1.close(); - dir2.close(); - } - - private Directory createIndex(int no) throws Exception { - final Directory dir = newDirectory(); - IndexWriter w = new IndexWriter( - dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setMergePolicy(newLogMergePolicy(false)) - ); - Document doc = new Document(); - doc.add(newField("field"+no, "yes it's stored", TextField.TYPE_STORED)); - w.addDocument(doc); - w.close(); - return dir; - } - - private void performDefaultTests(IndexReader r1) throws Exception { - IndexReader r2 = (IndexReader) r1.clone(); - assertTrue(r1 != r2); - TestIndexReader.assertIndexEquals(r1, r2); - r1.close(); - r2.close(); - TestIndexReaderReopen.assertReaderClosed(r1, true, true); - TestIndexReaderReopen.assertReaderClosed(r2, true, true); - } -} Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -1128,8 +1128,7 @@ writer2.close(); IndexReader r1 = IndexReader.open(dir2); - IndexReader r2 = (IndexReader) r1.clone(); - writer.addIndexes(r1, r2); + writer.addIndexes(r1, r1); writer.close(); IndexReader r3 = IndexReader.open(dir); @@ -1137,7 +1136,6 @@ r3.close(); r1.close(); - r2.close(); dir2.close(); dir.close(); Index: lucene/src/test/org/apache/lucene/index/TestSegmentReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (revision 1220696) +++ lucene/src/test/org/apache/lucene/index/TestSegmentReader.java (working copy) @@ -41,7 +41,7 @@ dir = newDirectory(); DocHelper.setupDoc(testDoc); SegmentInfo info = DocHelper.writeDoc(random, dir, testDoc); - reader = SegmentReader.get(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.READ); + reader = new SegmentReader(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.READ); } @Override @@ -73,50 +73,6 @@ } } - public void testDelete() throws IOException { - Document docToDelete = new Document(); - DocHelper.setupDoc(docToDelete); - SegmentInfo info = DocHelper.writeDoc(random, dir, docToDelete); - SegmentReader deleteReader = SegmentReader.getRW(info, true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); - assertNotNull(deleteReader); - assertEquals(1, deleteReader.numDocs()); - final Object combKey = deleteReader.getCombinedCoreAndDeletesKey(); - final Object coreKey = deleteReader.getCoreCacheKey(); - assertNotNull(combKey); - assertNotNull(coreKey); - assertNotSame(combKey, coreKey); - - SegmentReader clone1 = (SegmentReader) deleteReader.clone(); - assertSame(coreKey, clone1.getCoreCacheKey()); - assertSame(combKey, clone1.getCombinedCoreAndDeletesKey()); - - deleteReader.deleteDocument(0); - final Object newCombKey = deleteReader.getCombinedCoreAndDeletesKey(); - assertNotNull(newCombKey); - assertNotSame(combKey, newCombKey); - assertSame(coreKey, deleteReader.getCoreCacheKey()); - assertFalse(deleteReader.getLiveDocs().get(0)); - assertTrue(deleteReader.hasDeletions()); - assertTrue(deleteReader.numDocs() == 0); - - SegmentReader clone2 = (SegmentReader) deleteReader.clone(); - assertSame(coreKey, clone2.getCoreCacheKey()); - assertSame(newCombKey, clone2.getCombinedCoreAndDeletesKey()); - assertFalse(clone2.getLiveDocs().get(0)); - assertTrue(clone2.hasDeletions()); - assertEquals(0, clone2.numDocs()); - clone2.close(); - - assertSame(coreKey, clone1.getCoreCacheKey()); - assertSame(combKey, clone1.getCombinedCoreAndDeletesKey()); - assertNull(clone1.getLiveDocs()); - assertFalse(clone1.hasDeletions()); - assertEquals(1, clone2.numDocs()); - clone1.close(); - - deleteReader.close(); - } - public void testGetFieldNameVariations() { Collection result = reader.getFieldNames(IndexReader.FieldOption.ALL); assertTrue(result != null); Index: lucene/src/java/org/apache/lucene/index/ParallelReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/ParallelReader.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/ParallelReader.java (working copy) @@ -205,16 +205,6 @@ return fields; } - @Override - public synchronized Object clone() { - // doReopen calls ensureOpen - try { - return doReopen(true); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - /** * Tries to reopen the subreaders. *
@@ -236,11 +226,6 @@ */ @Override protected synchronized IndexReader doOpenIfChanged() throws CorruptIndexException, IOException { - // doReopen calls ensureOpen - return doReopen(false); - } - - private IndexReader doReopen(boolean doClone) throws CorruptIndexException, IOException { ensureOpen(); boolean reopened = false; @@ -251,16 +236,11 @@ try { for (final IndexReader oldReader : readers) { IndexReader newReader = null; - if (doClone) { - newReader = (IndexReader) oldReader.clone(); + newReader = IndexReader.openIfChanged(oldReader); + if (newReader != null) { reopened = true; } else { - newReader = IndexReader.openIfChanged(oldReader); - if (newReader != null) { - reopened = true; - } else { - newReader = oldReader; - } + newReader = oldReader; } newReaders.add(newReader); } Index: lucene/src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentReader.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -23,7 +23,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.store.Directory; import org.apache.lucene.codecs.PerDocProducer; @@ -39,27 +38,21 @@ /** * @lucene.experimental */ -public final class SegmentReader extends IndexReader implements Cloneable { - private final boolean readOnly; +public final class SegmentReader extends IndexReader { - private SegmentInfo si; + private final SegmentInfo si; private final ReaderContext readerContext = new AtomicReaderContext(this); - final CloseableThreadLocal fieldsReaderLocal = new FieldsReaderLocal(); - final CloseableThreadLocal termVectorsLocal = new CloseableThreadLocal(); + private final CloseableThreadLocal fieldsReaderLocal = new FieldsReaderLocal(); + private final CloseableThreadLocal termVectorsLocal = new CloseableThreadLocal(); - volatile BitVector liveDocs = null; - volatile Object combinedCoreAndDeletesKey; - AtomicInteger liveDocsRef = null; - boolean hasChanges = false; + private final BitVector liveDocs; - // TODO: remove deletions from SR - private int pendingDeleteCount; - private boolean rollbackHasChanges = false; - private SegmentInfo rollbackSegmentInfo; - private int rollbackPendingDeleteCount; - // end TODO + // Normally set to si.docCount - si.delDocCount, unless we + // were created as an NRT reader from IW, in which case IW + // tells us the docCount: + private final int numDocs; - SegmentCoreReaders core; + private final SegmentCoreReaders core; /** * Sets the initial value @@ -75,56 +68,69 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static SegmentReader get(SegmentInfo si, int termInfosIndexDivisor, IOContext context) throws CorruptIndexException, IOException { - return get(true, si, true, termInfosIndexDivisor, context); - } - - // TODO: remove deletions from SR - static SegmentReader getRW(SegmentInfo si, boolean doOpenStores, int termInfosIndexDivisor, IOContext context) throws CorruptIndexException, IOException { - return get(false, si, doOpenStores, termInfosIndexDivisor, context); - } - - /** - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - private static SegmentReader get(boolean readOnly, - SegmentInfo si, - boolean doOpenStores, - int termInfosIndexDivisor, - IOContext context) - throws CorruptIndexException, IOException { - - SegmentReader instance = new SegmentReader(readOnly, si); + public SegmentReader(SegmentInfo si, int termInfosIndexDivisor, IOContext context) throws IOException { + this.si = si; boolean success = false; try { - instance.core = new SegmentCoreReaders(instance, si.dir, si, context, termInfosIndexDivisor); - if (doOpenStores) { - instance.core.openDocStores(si); + core = new SegmentCoreReaders(this, si.dir, si, context, termInfosIndexDivisor); + if (si.hasDeletions()) { + // NOTE: the bitvector is stored using the regular directory, not cfs + liveDocs = new BitVector(directory(), si.getDelFileName(), new IOContext(IOContext.READ, true)); + } else { + assert si.getDelCount() == 0; + liveDocs = null; } - instance.loadLiveDocs(context); + numDocs = si.docCount - si.getDelCount(); + assert checkLiveCounts(false); success = true; } finally { - // With lock-less commits, it's entirely possible (and // fine) to hit a FileNotFound exception above. In // this case, we want to explicitly close any subset // of things that were opened so that we don't have to // wait for a GC to do so. if (!success) { - instance.doClose(); + doClose(); } } - return instance; } - private SegmentReader(boolean readOnly, SegmentInfo si) { - this.readOnly = readOnly; + // Create new SegmentReader sharing core from a previous + // SegmentReader and loading new live docs from a new + // deletes file. Used by openIfChanged. + SegmentReader(SegmentInfo si, SegmentReader parent, IOContext context) throws IOException { + assert si.dir == parent.getSegmentInfo().dir; this.si = si; + + // It's no longer possible to unDeleteAll, so, we can + // only be created if we have deletions: + assert si.hasDeletions(); + + // ... but load our own deleted docs: + liveDocs = new BitVector(si.dir, si.getDelFileName(), new IOContext(IOContext.READ, true)); + numDocs = si.docCount - si.getDelCount(); + assert checkLiveCounts(false); + + // We share core w/ parent: + parent.core.incRef(); + core = parent.core; } - void openDocStores() throws IOException { - core.openDocStores(si); + // Create new SegmentReader sharing core from a previous + // SegmentReader and using the provided in-memory + // liveDocs. Used by IndexWriter to provide a new NRT + // reader: + SegmentReader(SegmentReader parent, BitVector liveDocs, int numDocs) throws IOException { + this.si = parent.si; + parent.core.incRef(); + this.core = parent.core; + + assert liveDocs != null; + this.liveDocs = liveDocs; + + this.numDocs = numDocs; + + assert checkLiveCounts(true); } @Override @@ -133,123 +139,37 @@ return liveDocs; } - private boolean checkLiveCounts() throws IOException { - final int recomputedCount = liveDocs.getRecomputedCount(); - // First verify BitVector is self consistent: - assert liveDocs.count() == recomputedCount : "live count=" + liveDocs.count() + " vs recomputed count=" + recomputedCount; - - assert si.getDelCount() == si.docCount - recomputedCount : - "delete count mismatch: info=" + si.getDelCount() + " vs BitVector=" + (si.docCount-recomputedCount); - - // Verify # deletes does not exceed maxDoc for this - // segment: - assert si.getDelCount() <= maxDoc() : - "delete count mismatch: " + recomputedCount + ") exceeds max doc (" + maxDoc() + ") for segment " + si.name; - - return true; - } - - private void loadLiveDocs(IOContext context) throws IOException { - // NOTE: the bitvector is stored using the regular directory, not cfs - if (si.hasDeletions()) { - liveDocs = new BitVector(directory(), si.getDelFileName(), new IOContext(context, true)); - liveDocsRef = new AtomicInteger(1); - assert checkLiveCounts(); + private boolean checkLiveCounts(boolean isNRT) throws IOException { + if (liveDocs != null) { if (liveDocs.size() != si.docCount) { throw new CorruptIndexException("document count mismatch: deleted docs count " + liveDocs.size() + " vs segment doc count " + si.docCount + " segment=" + si.name); } - } else { - assert si.getDelCount() == 0; - } - // we need a key reflecting actual deletes (if existent or not): - combinedCoreAndDeletesKey = new Object(); - } - /** Clones are always in readOnly mode */ - @Override - public final synchronized Object clone() { - try { - return reopenSegment(si, true); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - // used by DirectoryReader: - synchronized SegmentReader reopenSegment(SegmentInfo si, boolean doClone) throws CorruptIndexException, IOException { - ensureOpen(); - boolean deletionsUpToDate = (this.si.hasDeletions() == si.hasDeletions()) - && (!si.hasDeletions() || this.si.getDelFileName().equals(si.getDelFileName())); + final int recomputedCount = liveDocs.getRecomputedCount(); + // Verify BitVector is self consistent: + assert liveDocs.count() == recomputedCount : "live count=" + liveDocs.count() + " vs recomputed count=" + recomputedCount; - // if we're cloning we need to run through the reopenSegment logic - // also if both old and new readers aren't readonly, we clone to avoid sharing modifications - if (deletionsUpToDate && !doClone && readOnly) { - return null; - } + // Verify our docCount matches: + assert numDocs == recomputedCount : + "delete count mismatch: numDocs=" + numDocs + " vs BitVector=" + (si.docCount-recomputedCount); - // When cloning, the incoming SegmentInfos should not - // have any changes in it: - assert !doClone || (deletionsUpToDate); - - // clone reader - SegmentReader clone = new SegmentReader(true, si); - - boolean success = false; - try { - core.incRef(); - clone.core = core; - clone.pendingDeleteCount = pendingDeleteCount; - clone.combinedCoreAndDeletesKey = combinedCoreAndDeletesKey; - - if (doClone) { - if (liveDocs != null) { - liveDocsRef.incrementAndGet(); - clone.liveDocs = liveDocs; - clone.liveDocsRef = liveDocsRef; - } - } else { - if (!deletionsUpToDate) { - // load deleted docs - assert clone.liveDocs == null; - clone.loadLiveDocs(IOContext.READ); - } else if (liveDocs != null) { - liveDocsRef.incrementAndGet(); - clone.liveDocs = liveDocs; - clone.liveDocsRef = liveDocsRef; - } - } - success = true; - } finally { - if (!success) { - // An exception occurred during reopen, we have to decRef the norms - // that we incRef'ed already and close singleNormsStream and FieldsReader - clone.decRef(); - } + assert isNRT || si.docCount - si.getDelCount() == recomputedCount : + "si.docCount=" + si.docCount + "si.getDelCount()=" + si.getDelCount() + " recomputedCount=" + recomputedCount; } - - return clone; + + return true; } /** @lucene.internal */ public StoredFieldsReader getFieldsReader() { return fieldsReaderLocal.get(); } - + @Override protected void doClose() throws IOException { - if (hasChanges) { - doCommit(); - } - + //System.out.println("SR.close seg=" + si); termVectorsLocal.close(); fieldsReaderLocal.close(); - - if (liveDocs != null) { - liveDocsRef.decrementAndGet(); - // null so if an app hangs on to us we still free most ram - liveDocs = null; - } - if (core != null) { core.decRef(); } @@ -410,10 +330,9 @@ @Override public String toString() { final StringBuilder buffer = new StringBuilder(); - if (hasChanges) { - buffer.append('*'); - } - buffer.append(si.toString(core.dir, pendingDeleteCount)); + // SegmentInfo.toString takes dir and number of + // *pending* deletions; so we reverse compute that here: + buffer.append(si.toString(core.dir, si.docCount - numDocs - si.getDelCount())); return buffer.toString(); } @@ -437,10 +356,6 @@ return si; } - void setSegmentInfo(SegmentInfo info) { - si = info; - } - /** Returns the directory this index resides in. */ @Override public Directory directory() { @@ -460,7 +375,7 @@ @Override public Object getCombinedCoreAndDeletesKey() { - return combinedCoreAndDeletesKey; + return this; } @Override @@ -479,103 +394,6 @@ } /** - * Clones the deleteDocs BitVector. May be overridden by subclasses. New and experimental. - * @param bv BitVector to clone - * @return New BitVector - */ - // TODO: remove deletions from SR - BitVector cloneDeletedDocs(BitVector bv) { - ensureOpen(); - return (BitVector)bv.clone(); - } - - // TODO: remove deletions from SR - void doCommit() throws IOException { - assert hasChanges; - startCommit(); - boolean success = false; - try { - commitChanges(); - success = true; - } finally { - if (!success) { - rollbackCommit(); - } - } - } - - // TODO: remove deletions from SR - private void startCommit() { - rollbackSegmentInfo = (SegmentInfo) si.clone(); - rollbackHasChanges = hasChanges; - rollbackPendingDeleteCount = pendingDeleteCount; - } - - // TODO: remove deletions from SR - private void rollbackCommit() { - si.reset(rollbackSegmentInfo); - hasChanges = rollbackHasChanges; - pendingDeleteCount = rollbackPendingDeleteCount; - } - - // TODO: remove deletions from SR - private synchronized void commitChanges() throws IOException { - si.advanceDelGen(); - - assert liveDocs.length() == si.docCount; - - // We can write directly to the actual name (vs to a - // .tmp & renaming it) because the file is not live - // until segments file is written: - final String delFileName = si.getDelFileName(); - boolean success = false; - try { - liveDocs.write(directory(), delFileName, IOContext.DEFAULT); - success = true; - } finally { - if (!success) { - try { - directory().deleteFile(delFileName); - } catch (Throwable t) { - // suppress this so we keep throwing the - // original exception - } - } - } - si.setDelCount(si.getDelCount()+pendingDeleteCount); - pendingDeleteCount = 0; - assert (maxDoc()-liveDocs.count()) == si.getDelCount(): "delete count mismatch during commit: info=" + si.getDelCount() + " vs BitVector=" + (maxDoc()-liveDocs.count()); - hasChanges = false; - } - - // TODO: remove deletions from SR - synchronized void deleteDocument(int docNum) throws IOException { - if (readOnly) - throw new UnsupportedOperationException("this SegmentReader is read only"); - hasChanges = true; - if (liveDocs == null) { - liveDocs = new BitVector(maxDoc()); - liveDocs.setAll(); - liveDocsRef = new AtomicInteger(1); - } - // there is more than 1 SegmentReader with a reference to this - // liveDocs BitVector so decRef the current liveDocsRef, - // clone the BitVector, create a new liveDocsRef - if (liveDocsRef.get() > 1) { - AtomicInteger oldRef = liveDocsRef; - liveDocs = cloneDeletedDocs(liveDocs); - liveDocsRef = new AtomicInteger(1); - oldRef.decrementAndGet(); - } - // we need a key reflecting actual deletes (if existent or not): - combinedCoreAndDeletesKey = new Object(); - // liveDocs are now dirty: - if (liveDocs.getAndClear(docNum)) { - pendingDeleteCount++; - } - } - - /** * Called when the shared core for this SegmentReader * is closed. *

Index: lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java =================================================================== --- lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (working copy) @@ -211,22 +211,25 @@ // Lock order: IW -> BD -> RP assert readerPool.infoIsLive(info); - final SegmentReader reader = readerPool.get(info, false, IOContext.READ); + final IndexWriter.ReadersAndLiveDocs rld = readerPool.get(info, true); + final SegmentReader reader = rld.getReader(IOContext.READ); int delCount = 0; final boolean segAllDeletes; try { if (coalescedDeletes != null) { //System.out.println(" del coalesced"); - delCount += applyTermDeletes(coalescedDeletes.termsIterable(), reader); - delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), reader); + delCount += applyTermDeletes(coalescedDeletes.termsIterable(), rld, reader); + delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), rld, reader); } //System.out.println(" del exact"); // Don't delete by Term here; DocumentsWriterPerThread // already did that on flush: - delCount += applyQueryDeletes(packet.queriesIterable(), reader); - segAllDeletes = reader.numDocs() == 0; + delCount += applyQueryDeletes(packet.queriesIterable(), rld, reader); + final int fullDelCount = rld.info.getDelCount() + rld.pendingDeleteCount; + assert fullDelCount <= rld.info.docCount; + segAllDeletes = fullDelCount == rld.info.docCount; } finally { - readerPool.release(reader, IOContext.Context.READ); + readerPool.release(reader, false); } anyNewDeletes |= delCount > 0; @@ -238,7 +241,7 @@ } if (infoStream.isEnabled("BD")) { - infoStream.message("BD", "seg=" + info + " segGen=" + segGen + " segDeletes=[" + packet + "]; coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] delCount=" + delCount + (segAllDeletes ? " 100% deleted" : "")); + infoStream.message("BD", "seg=" + info + " segGen=" + segGen + " segDeletes=[" + packet + "]; coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : "")); } if (coalescedDeletes == null) { @@ -260,15 +263,18 @@ if (coalescedDeletes != null) { // Lock order: IW -> BD -> RP assert readerPool.infoIsLive(info); - SegmentReader reader = readerPool.get(info, false, IOContext.READ); + final IndexWriter.ReadersAndLiveDocs rld = readerPool.get(info, true); + final SegmentReader reader = rld.getReader(IOContext.READ); int delCount = 0; final boolean segAllDeletes; try { - delCount += applyTermDeletes(coalescedDeletes.termsIterable(), reader); - delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), reader); - segAllDeletes = reader.numDocs() == 0; + delCount += applyTermDeletes(coalescedDeletes.termsIterable(), rld, reader); + delCount += applyQueryDeletes(coalescedDeletes.queriesIterable(), rld, reader); + final int fullDelCount = rld.info.getDelCount() + rld.pendingDeleteCount; + assert fullDelCount <= rld.info.docCount; + segAllDeletes = fullDelCount == rld.info.docCount; } finally { - readerPool.release(reader, IOContext.Context.READ); + readerPool.release(reader, false); } anyNewDeletes |= delCount > 0; @@ -280,7 +286,7 @@ } if (infoStream.isEnabled("BD")) { - infoStream.message("BD", "seg=" + info + " segGen=" + segGen + " coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] delCount=" + delCount + (segAllDeletes ? " 100% deleted" : "")); + infoStream.message("BD", "seg=" + info + " segGen=" + segGen + " coalesced deletes=[" + (coalescedDeletes == null ? "null" : coalescedDeletes) + "] newDelCount=" + delCount + (segAllDeletes ? " 100% deleted" : "")); } } info.setBufferedDeletesGen(nextGen); @@ -348,7 +354,7 @@ } // Delete by Term - private synchronized long applyTermDeletes(Iterable termsIter, SegmentReader reader) throws IOException { + private synchronized long applyTermDeletes(Iterable termsIter, IndexWriter.ReadersAndLiveDocs rld, SegmentReader reader) throws IOException { long delCount = 0; Fields fields = reader.fields(); if (fields == null) { @@ -362,7 +368,9 @@ DocsEnum docs = null; assert checkDeleteTerm(null); - + + boolean any = false; + //System.out.println(Thread.currentThread().getName() + " del terms reader=" + reader); for (Term term : termsIter) { // Since we visit terms sorted, we gain performance @@ -387,7 +395,7 @@ // System.out.println(" term=" + term); if (termsEnum.seekExact(term.bytes(), false)) { - DocsEnum docsEnum = termsEnum.docs(reader.getLiveDocs(), docs, false); + DocsEnum docsEnum = termsEnum.docs(rld.liveDocs, docs, false); //System.out.println("BDS: got docsEnum=" + docsEnum); if (docsEnum != null) { @@ -396,14 +404,19 @@ //System.out.println(Thread.currentThread().getName() + " del term=" + term + " doc=" + docID); if (docID == DocsEnum.NO_MORE_DOCS) { break; + } + // NOTE: there is no limit check on the docID + // when deleting by Term (unlike by Query) + // because on flush we apply all Term deletes to + // each segment. So all Term deleting here is + // against prior segments: + if (!any) { + rld.writableLiveDocs(); + any = true; } - reader.deleteDocument(docID); - // TODO: we could/should change - // reader.deleteDocument to return boolean - // true if it did in fact delete, because here - // we could be deleting an already-deleted doc - // which makes this an upper bound: - delCount++; + if (rld.delete(docID)) { + delCount++; + } } } } @@ -422,9 +435,10 @@ } // Delete by query - private static long applyQueryDeletes(Iterable queriesIter, SegmentReader reader) throws IOException { + private static long applyQueryDeletes(Iterable queriesIter, IndexWriter.ReadersAndLiveDocs rld, SegmentReader reader) throws IOException { long delCount = 0; final AtomicReaderContext readerContext = (AtomicReaderContext) reader.getTopReaderContext(); + boolean any = false; for (QueryAndLimit ent : queriesIter) { Query query = ent.query; int limit = ent.limit; @@ -434,13 +448,18 @@ if (it != null) { while(true) { int doc = it.nextDoc(); - if (doc >= limit) + if (doc >= limit) { break; + } - reader.deleteDocument(doc); - // as we use getLiveDocs() to filter out already deleted documents, - // we only delete live documents, so the counting is right: - delCount++; + if (!any) { + rld.writableLiveDocs(); + any = true; + } + + if (rld.delete(doc)) { + delCount++; + } } } } Index: lucene/src/java/org/apache/lucene/index/SegmentInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfos.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/SegmentInfos.java (working copy) @@ -362,7 +362,8 @@ final SegmentInfo info = it.next(); if (info.getDelCount() == info.docCount) { it.remove(); - segmentSet.remove(info); + final boolean didRemove = segmentSet.remove(info); + assert didRemove; } } assert segmentSet.size() == segments.size(); @@ -995,8 +996,11 @@ } } + // the rest of the segments in list are duplicates, so don't remove from map, only list! + segments.subList(newSegIdx, segments.size()).clear(); + // Either we found place to insert segment, or, we did - // not, but only because all segments we merged became + // not, but only because all segments we merged becamee // deleted while we are merging, in which case it should // be the case that the new segment is also all deleted, // we insert it at the beginning if it should not be dropped: @@ -1004,9 +1008,6 @@ segments.add(0, merge.info); } - // the rest of the segments in list are duplicates, so don't remove from map, only list! - segments.subList(newSegIdx, segments.size()).clear(); - // update the Set if (!dropSegment) { segmentSet.add(merge.info); @@ -1112,5 +1113,4 @@ return -1; } } - } Index: lucene/src/java/org/apache/lucene/index/CheckIndex.java =================================================================== --- lucene/src/java/org/apache/lucene/index/CheckIndex.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/CheckIndex.java (working copy) @@ -17,19 +17,6 @@ * limitations under the License. */ -import org.apache.lucene.codecs.BlockTreeTermsReader; -import org.apache.lucene.codecs.Codec; -import org.apache.lucene.document.FieldType; // for javadocs -import org.apache.lucene.search.DocIdSetIterator; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.DocValues.SortedSource; -import org.apache.lucene.index.DocValues.Source; - import java.io.File; import java.io.IOException; import java.io.PrintStream; @@ -41,16 +28,19 @@ import java.util.List; import java.util.Map; +import org.apache.lucene.codecs.BlockTreeTermsReader; +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldType; // for javadocs +import org.apache.lucene.index.DocValues.SortedSource; +import org.apache.lucene.index.DocValues.Source; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; -import org.apache.lucene.document.Document; - -import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CommandLineUtil; @@ -534,18 +524,18 @@ } if (infoStream != null) infoStream.print(" test: open reader........."); - reader = SegmentReader.get(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.DEFAULT); + reader = new SegmentReader(info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.DEFAULT); segInfoStat.openReaderPassed = true; final int numDocs = reader.numDocs(); toLoseDocCount = numDocs; if (reader.hasDeletions()) { - if (reader.liveDocs.count() != info.docCount - info.getDelCount()) { - throw new RuntimeException("delete count mismatch: info=" + (info.docCount - info.getDelCount()) + " vs reader=" + reader.liveDocs.count()); + if (reader.numDocs() != info.docCount - info.getDelCount()) { + throw new RuntimeException("delete count mismatch: info=" + (info.docCount - info.getDelCount()) + " vs reader=" + reader.numDocs()); } - if ((info.docCount-reader.liveDocs.count()) > reader.maxDoc()) { - throw new RuntimeException("too many deleted docs: maxDoc()=" + reader.maxDoc() + " vs del count=" + (info.docCount-reader.liveDocs.count())); + if ((info.docCount-reader.numDocs()) > reader.maxDoc()) { + throw new RuntimeException("too many deleted docs: maxDoc()=" + reader.maxDoc() + " vs del count=" + (info.docCount-reader.numDocs())); } if (info.docCount - numDocs != info.getDelCount()) { throw new RuntimeException("delete count mismatch: info=" + info.getDelCount() + " vs reader=" + (info.docCount - numDocs)); Index: lucene/src/java/org/apache/lucene/index/BaseMultiReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/BaseMultiReader.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/BaseMultiReader.java (working copy) @@ -26,7 +26,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.ReaderUtil; -abstract class BaseMultiReader extends IndexReader implements Cloneable { +abstract class BaseMultiReader extends IndexReader { protected final R[] subReaders; protected final int[] starts; // 1st docno for each segment private final ReaderContext topLevelContext; @@ -64,9 +64,6 @@ protected abstract IndexReader doOpenIfChanged() throws CorruptIndexException, IOException; @Override - public abstract Object clone(); - - @Override public Bits getLiveDocs() { throw new UnsupportedOperationException("please use MultiFields.getLiveDocs, or wrap your IndexReader with SlowMultiReaderWrapper, if you really need a top level Bits liveDocs"); } Index: lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java =================================================================== --- lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (working copy) @@ -349,7 +349,7 @@ try { synchronized(this) { if (verbose()) { - message(" consider merge " + merge.segString(dir)); + message(" consider merge " + writer.segString(merge.segments)); } // OK to spawn a new merge thread to handle this @@ -457,7 +457,7 @@ tWriter.mergeInit(merge); updateMergeThreads(); if (verbose()) { - message(" merge thread: do another merge " + merge.segString(dir)); + message(" merge thread: do another merge " + tWriter.segString(merge.segments)); } } else { break; @@ -492,15 +492,22 @@ @Override public String toString() { MergePolicy.OneMerge merge = getRunningMerge(); - if (merge == null) + if (merge == null) { merge = startMerge; - return "merge thread: " + merge.segString(dir); + } + try { + return "merge thread: " + tWriter.segString(merge.segments); + } catch (IOException ioe) { + throw new RuntimeException(ioe); + } } } /** Called when an exception is hit in a background merge * thread */ protected void handleMergeException(Throwable exc) { + //System.out.println("MERGE EXC:"); + //exc.printStackTrace(System.out); try { // When an exception is hit during merge, IndexWriter // removes any partial files and then allows another Index: lucene/src/java/org/apache/lucene/index/MultiReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/MultiReader.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/MultiReader.java (working copy) @@ -61,19 +61,6 @@ @Override protected synchronized IndexReader doOpenIfChanged() throws CorruptIndexException, IOException { - return doReopen(false); - } - - @Override - public synchronized Object clone() { - try { - return doReopen(true); - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - private IndexReader doReopen(boolean doClone) throws CorruptIndexException, IOException { ensureOpen(); boolean changed = false; @@ -82,17 +69,12 @@ boolean success = false; try { for (int i = 0; i < subReaders.length; i++) { - if (doClone) { - newSubReaders[i] = (IndexReader) subReaders[i].clone(); + final IndexReader newSubReader = IndexReader.openIfChanged(subReaders[i]); + if (newSubReader != null) { + newSubReaders[i] = newSubReader; changed = true; } else { - final IndexReader newSubReader = IndexReader.openIfChanged(subReaders[i]); - if (newSubReader != null) { - newSubReaders[i] = newSubReader; - changed = true; - } else { - newSubReaders[i] = subReaders[i]; - } + newSubReaders[i] = subReaders[i]; } } success = true; Index: lucene/src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DirectoryReader.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/DirectoryReader.java (working copy) @@ -62,7 +62,7 @@ IOException prior = null; boolean success = false; try { - readers[i] = SegmentReader.get(sis.info(i), termInfosIndexDivisor, IOContext.READ); + readers[i] = new SegmentReader(sis.info(i), termInfosIndexDivisor, IOContext.READ); success = true; } catch(IOException ex) { prior = ex; @@ -94,7 +94,8 @@ try { final SegmentInfo info = infos.info(i); assert info.dir == dir; - final SegmentReader reader = writer.readerPool.getReadOnlyClone(info, IOContext.READ); + final IndexWriter.ReadersAndLiveDocs rld = writer.readerPool.get(info, true); + final SegmentReader reader = rld.getReadOnlyClone(IOContext.READ); if (reader.numDocs() > 0 || writer.getKeepFullyDeletedSegments()) { readers.add(reader); infosUpto++; @@ -116,7 +117,7 @@ /** This constructor is only used for {@link #doOpenIfChanged()} */ static DirectoryReader open(Directory directory, IndexWriter writer, SegmentInfos infos, SegmentReader[] oldReaders, - boolean doClone, int termInfosIndexDivisor) throws IOException { + int termInfosIndexDivisor) throws IOException { // we put the old SegmentReaders in a map, that allows us // to lookup a reader using its segment name final Map segmentReaders = new HashMap(); @@ -151,24 +152,21 @@ SegmentReader newReader; if (newReaders[i] == null || infos.info(i).getUseCompoundFile() != newReaders[i].getSegmentInfo().getUseCompoundFile()) { - // We should never see a totally new segment during cloning - assert !doClone; - // this is a new reader; in case we hit an exception we can close it safely - newReader = SegmentReader.get(infos.info(i), termInfosIndexDivisor, IOContext.READ); + newReader = new SegmentReader(infos.info(i), termInfosIndexDivisor, IOContext.READ); readerShared[i] = false; newReaders[i] = newReader; } else { - newReader = newReaders[i].reopenSegment(infos.info(i), doClone); - if (newReader == null) { - // this reader will be shared between the old and the new one, - // so we must incRef it + if (newReaders[i].getSegmentInfo().getDelGen() == infos.info(i).getDelGen()) { + // No change; this reader will be shared between + // the old and the new one, so we must incRef + // it: readerShared[i] = true; newReaders[i].incRef(); } else { readerShared[i] = false; - // Steal ref returned to us by reopenSegment: - newReaders[i] = newReader; + // Steal the ref returned by SegmentReader ctor: + newReaders[i] = new SegmentReader(infos.info(i), newReaders[i], IOContext.READ); } } success = true; @@ -224,16 +222,6 @@ } @Override - public final synchronized Object clone() { - try { - DirectoryReader newReader = doOpenIfChanged((SegmentInfos) segmentInfos.clone(), true, writer); - return newReader; - } catch (Exception ex) { - throw new RuntimeException(ex); - } - } - - @Override protected final IndexReader doOpenIfChanged() throws CorruptIndexException, IOException { return doOpenIfChanged(null); } @@ -302,13 +290,13 @@ protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { final SegmentInfos infos = new SegmentInfos(); infos.read(directory, segmentFileName); - return doOpenIfChanged(infos, false, null); + return doOpenIfChanged(infos, null); } }.run(commit); } - private synchronized DirectoryReader doOpenIfChanged(SegmentInfos infos, boolean doClone, IndexWriter writer) throws CorruptIndexException, IOException { - return DirectoryReader.open(directory, writer, infos, subReaders, doClone, termInfosIndexDivisor); + private synchronized DirectoryReader doOpenIfChanged(SegmentInfos infos, IndexWriter writer) throws CorruptIndexException, IOException { + return DirectoryReader.open(directory, writer, infos, subReaders, termInfosIndexDivisor); } /** Version number when this IndexReader was opened. */ Index: lucene/src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexReader.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -71,7 +71,7 @@ IndexReader instance; use your own (non-Lucene) objects instead. */ -public abstract class IndexReader implements Cloneable,Closeable { +public abstract class IndexReader implements Closeable { /** * A custom listener that's invoked when the IndexReader @@ -150,6 +150,8 @@ /** Expert: returns the current refCount for this reader */ public final int getRefCount() { + // NOTE: don't ensureOpen, so that callers can see + // refCount is 0 (reader is closed) return refCount.get(); } @@ -495,8 +497,8 @@ * internal state). */ @Override - public synchronized Object clone() { - throw new UnsupportedOperationException("This reader does not implement clone()"); + public final Object clone() { + throw new UnsupportedOperationException("IndexReaders do not implement clone() (they are readOnly)"); } /** Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; @@ -391,348 +392,405 @@ return r; } - /** Holds shared SegmentReader instances. IndexWriter uses - * SegmentReaders for 1) applying deletes, 2) doing - * merges, 3) handing out a real-time reader. This pool - * reuses instances of the SegmentReaders in all these - * places if it is in "near real-time mode" (getReader() - * has been called on this instance). */ + // This class inherits all sync from IW: + class ReadersAndLiveDocs { + // Not final because we replace (clone) when we need to + // change it and it's been shared: + public final SegmentInfo info; - class ReaderPool { - - final class SegmentCacheKey { - public final SegmentInfo si; - public final IOContext.Context context; - - public SegmentCacheKey(SegmentInfo segInfo, IOContext.Context context) { - assert context == IOContext.Context.MERGE || context == IOContext.Context.READ; - this.si = segInfo; - this.context = context; + // Set once (null, and then maybe set, and never set again): + private SegmentReader reader; + + // TODO: it's sometimes wasteful that we hold open two + // separate SRs (one for merging one for + // reading)... maybe just use a single SR? The gains of + // not loading the terms index (for merging in the + // non-NRT case) are far less now... and if the app has + // any deletes it'll open real readers anyway. + + // Set once (null, and then maybe set, and never set again): + private SegmentReader mergeReader; + + // Holds the current shared (readable and writable + // liveDocs). This is null when there are no deleted + // docs, and it's copy-on-write (cloned whenever we need + // to change it but it's been shared to an external NRT + // reader). + public BitVector liveDocs; + + // How many further deletions we've done against + // liveDocs vs when we loaded it or last wrote it: + public int pendingDeleteCount; + + // True if the current liveDocs is referenced by an + // external NRT reader: + public boolean shared; + + public ReadersAndLiveDocs(SegmentInfo info) { + this.info = info; + shared = true; + } + + // Returns false if we are the only remaining refs of + // this reader: + public synchronized boolean anyOutsideRefs(SegmentReader sr) { + int myRefCounts = 0; + if (sr == reader) { + myRefCounts++; } - - @Override - public int hashCode() { - return si.hashCode() + context.hashCode(); + if (sr == mergeReader) { + myRefCounts++; } + final int rc = sr.getRefCount(); + assert rc >= myRefCounts; + return rc > myRefCounts; + } - @Override - public String toString() { - return "SegmentCacheKey(" + si + "," + context + ")"; + // Returns true if any reader remains + public synchronized boolean removeReader(SegmentReader sr, boolean drop) throws IOException { + if (sr == reader) { + //System.out.println(" non-merge reader"); + reader.decRef(); + reader = null; } + + if (sr == mergeReader) { + //System.out.println(" merge reader"); + mergeReader.decRef(); + mergeReader = null; + if (drop && reader != null) { + //System.out.println(" also release normal reader rc=" + rld.reader.getRefCount()); + reader.decRef(); + reader = null; + } + } - @Override - public boolean equals(Object _other) { - if (!(_other instanceof SegmentCacheKey)) { - return false; + return reader != null || mergeReader != null; + } + + // Called only from assert + private boolean countsMatch() { + try { + if (liveDocs == null) { + assert pendingDeleteCount == 0; + } else { + assert liveDocs.count() == info.docCount - info.getDelCount() - pendingDeleteCount : + "liveDocs.count()=" + liveDocs.count() + " info.docCount=" + info.docCount + " info.delCount=" + info.getDelCount() + " pendingDelCount=" + pendingDeleteCount; } - final SegmentCacheKey other = (SegmentCacheKey) _other; - return si.equals(other.si) && context == other.context; + } catch (AssertionError ae) { + System.out.println("FAIL: seg=" + info + " liveDocs=" + liveDocs); + new Throwable().printStackTrace(System.out); + throw ae; } + return true; } - private final Map readerMap = new HashMap(); + // Get reader for searching/deleting + public synchronized SegmentReader getReader(IOContext context) throws IOException { + //System.out.println(" livedocs=" + rld.liveDocs); - /** Forcefully clear changes for the specified segments. This is called on successful merge. */ - synchronized void clear(List infos) throws IOException { - if (infos == null) { - for (Map.Entry ent: readerMap.entrySet()) { - ent.getValue().hasChanges = false; + if (reader == null) { + reader = new SegmentReader(info, config.getReaderTermsIndexDivisor(), context); + if (liveDocs == null) { + liveDocs = (BitVector) reader.getLiveDocs(); } - } else { - for (final SegmentInfo info: infos) { - final SegmentReader r = readerMap.get(new SegmentCacheKey(info, IOContext.Context.MERGE)); - if (r != null) { - r.hasChanges = false; + //System.out.println("ADD seg=" + rld.info + " isMerge=" + isMerge + " " + readerMap.size() + " in pool"); + } + + // Ref for caller + reader.incRef(); + return reader; + } + + // Get reader for merging (does not load the terms + // index): + public synchronized SegmentReader getMergeReader(IOContext context) throws IOException { + //System.out.println(" livedocs=" + rld.liveDocs); + + if (mergeReader == null) { + + if (reader != null) { + // Just use the already opened non-merge reader + // for merging. In the NRT case this saves us + // pointless double-open: + //System.out.println("PROMOTE non-merge reader seg=" + rld.info); + reader.incRef(); + mergeReader = reader; + } else { + mergeReader = new SegmentReader(info, -1, context); + if (liveDocs == null) { + liveDocs = (BitVector) mergeReader.getLiveDocs(); } } } - } - // used only by asserts - public synchronized boolean infoIsLive(SegmentInfo info) { - return infoIsLive(info, ""); + // Ref for caller + mergeReader.incRef(); + return mergeReader; } - public synchronized boolean infoIsLive(SegmentInfo info, String message) { - int idx = segmentInfos.indexOf(info); - assert idx != -1: "info=" + info + " isn't live: " + message; - assert segmentInfos.info(idx) == info: "info=" + info + " doesn't match live info in segmentInfos: " + message; - return true; + public synchronized boolean delete(int docID) { + assert liveDocs != null; + assert docID >= 0 && docID < liveDocs.length(); + final boolean didDelete = liveDocs.getAndClear(docID); + if (didDelete) { + pendingDeleteCount++; + //System.out.println(" new del seg=" + info + " docID=" + docID + " pendingDelCount=" + pendingDeleteCount + " totDelCount=" + (info.docCount-liveDocs.count())); + } + return didDelete; } - public synchronized SegmentInfo mapToLive(SegmentInfo info) { - int idx = segmentInfos.indexOf(info); - if (idx != -1) { - info = segmentInfos.info(idx); + public synchronized void dropReaders() throws IOException { + if (reader != null) { + //System.out.println(" pool.drop info=" + info + " rc=" + reader.getRefCount()); + reader.decRef(); + reader = null; } - return info; + if (mergeReader != null) { + //System.out.println(" pool.drop info=" + info + " merge rc=" + mergeReader.getRefCount()); + mergeReader.decRef(); + mergeReader = null; + } } /** - * Release the segment reader (i.e. decRef it and close if there - * are no more references. - * @return true if this release altered the index (eg - * the SegmentReader had pending changes to del docs and - * was closed). Caller must call checkpoint() if so. - * @param sr - * @throws IOException + * Returns a ref to a clone. NOTE: this clone is not + * enrolled in the pool, so you should simply close() + * it when you're done (ie, do not call release()). */ - public synchronized boolean release(SegmentReader sr, IOContext.Context context) throws IOException { - return release(sr, false, context); + public synchronized SegmentReader getReadOnlyClone(IOContext context) throws IOException { + if (reader == null) { + getReader(context).decRef(); + assert reader != null; + } + assert countsMatch(); + shared = true; + if (liveDocs != null) { + return new SegmentReader(reader, liveDocs, info.docCount - info.getDelCount() - pendingDeleteCount); + } else { + reader.incRef(); + return reader; + } } - /** - * Release the segment reader (i.e. decRef it and close if there - * are no more references. - * @return true if this release altered the index (eg - * the SegmentReader had pending changes to del docs and - * was closed). Caller must call checkpoint() if so. - * @param sr - * @throws IOException - */ - public synchronized boolean release(SegmentReader sr, boolean drop) throws IOException { - final SegmentCacheKey cacheKey = new SegmentCacheKey(sr.getSegmentInfo(), IOContext.Context.READ); - final SegmentReader other = readerMap.get(cacheKey); - if (sr == other) { - return release(sr, drop, IOContext.Context.READ); + public synchronized void writableLiveDocs() { + assert Thread.holdsLock(IndexWriter.this); + //System.out.println("writableLivedocs seg=" + info + " liveDocs=" + liveDocs + " shared=" + shared); + if (shared) { + // Copy on write: this means we've cloned a + // SegmentReader sharing the current liveDocs + // instance; must now make a private clone so we can + // change it: + if (liveDocs == null) { + //System.out.println("create BV seg=" + info); + liveDocs = new BitVector(info.docCount); + liveDocs.setAll(); + } else { + liveDocs = (BitVector) liveDocs.clone(); + } + shared = false; } else { - assert sr == readerMap.get(new SegmentCacheKey(sr.getSegmentInfo(), IOContext.Context.MERGE)); - return release(sr, drop, IOContext.Context.MERGE); + assert liveDocs != null; } } + public synchronized BitVector getReadOnlyLiveDocs() { + //System.out.println("getROLiveDocs seg=" + info); + assert Thread.holdsLock(IndexWriter.this); + shared = true; + assert countsMatch(); + //if (liveDocs != null) { + //System.out.println(" liveCount=" + liveDocs.count()); + //} + return liveDocs; + } + + // Commit live docs to the directory (writes new + // _X_N.del files); returns true if it wrote the file + // and false if there were no new deletes to write: + public synchronized boolean writeLiveDocs(Directory dir) throws IOException { + //System.out.println("rld.writeLiveDocs seg=" + info + " pendingDelCount=" + pendingDeleteCount); + if (pendingDeleteCount != 0) { + // We have new deletes + assert liveDocs.length() == info.docCount; + + // Save in case we need to rollback on failure: + final SegmentInfo sav = (SegmentInfo) info.clone(); + info.advanceDelGen(); + + // We can write directly to the actual name (vs to a + // .tmp & renaming it) because the file is not live + // until segments file is written: + final String delFileName = info.getDelFileName(); + boolean success = false; + try { + liveDocs.write(dir, delFileName, IOContext.DEFAULT); + success = true; + } finally { + if (!success) { + info.reset(sav); + try { + dir.deleteFile(delFileName); + } catch (Throwable t) { + // Suppress this so we keep throwing the + // original exception + } + } + } + assert (info.docCount - liveDocs.count()) == info.getDelCount() + pendingDeleteCount: + "delete count mismatch during commit: seg=" + info + " info.delCount=" + info.getDelCount() + " vs BitVector=" + (info.docCount-liveDocs.count() + " pendingDelCount=" + pendingDeleteCount); + info.setDelCount(info.getDelCount() + pendingDeleteCount); + pendingDeleteCount = 0; + return true; + } else { + return false; + } + } + + @Override + public String toString() { + return "SegmentLiveDocs(seg=" + info + " pendingDeleteCount=" + pendingDeleteCount + " shared=" + shared + ")"; + } + } + + /** Holds shared SegmentReader instances. IndexWriter uses + * SegmentReaders for 1) applying deletes, 2) doing + * merges, 3) handing out a real-time reader. This pool + * reuses instances of the SegmentReaders in all these + * places if it is in "near real-time mode" (getReader() + * has been called on this instance). */ + + class ReaderPool { + + private final Map readerMap = new HashMap(); + + // used only by asserts + public synchronized boolean infoIsLive(SegmentInfo info) { + int idx = segmentInfos.indexOf(info); + assert idx != -1: "info=" + info + " isn't live"; + assert segmentInfos.info(idx) == info: "info=" + info + " doesn't match live info in segmentInfos"; + return true; + } + /** * Release the segment reader (i.e. decRef it and close if there - * are no more references. - * @return true if this release altered the index (eg - * the SegmentReader had pending changes to del docs and - * was closed). Caller must call checkpoint() if so. + * are no more references). If drop is true then we + * remove this entry from the pool. * @param sr * @throws IOException */ - public synchronized boolean release(SegmentReader sr, boolean drop, IOContext.Context context) throws IOException { - - SegmentCacheKey cacheKey = new SegmentCacheKey(sr.getSegmentInfo(), context); - - final boolean pooled = readerMap.containsKey(cacheKey); - - assert !pooled || readerMap.get(cacheKey) == sr; - + public synchronized void release(SegmentReader sr, boolean drop) throws IOException { // Drop caller's ref; for an external reader (not // pooled), this decRef will close it + //System.out.println("pool.release seg=" + sr.getSegmentInfo() + " rc=" + sr.getRefCount() + " drop=" + drop); sr.decRef(); - if (pooled && (drop || (!poolReaders && sr.getRefCount() == 1))) { + final ReadersAndLiveDocs rld = readerMap.get(sr.getSegmentInfo()); - // We invoke deleter.checkpoint below, so we must be - // sync'd on IW if there are changes: - assert !sr.hasChanges || Thread.holdsLock(IndexWriter.this); + if (rld != null && (drop || (!poolReaders && !rld.anyOutsideRefs(sr)))) { // Discard (don't save) changes when we are dropping // the reader; this is used only on the sub-readers - // after a successful merge. - final boolean hasChanges; - if (drop) { - hasChanges = sr.hasChanges = false; - } else { - hasChanges = sr.hasChanges; - } + // after a successful merge. If deletes had + // accumulated on those sub-readers while the merge + // is running, by now we have carried forward those + // deletes onto the newly merged segment, so we can + // discard them on the sub-readers: - // Drop our ref -- this will commit any pending - // changes to the dir - sr.close(); - - // We are the last ref to this reader; since we're - // not pooling readers, we release it: - readerMap.remove(cacheKey); - - if (drop && context == IOContext.Context.MERGE) { - // Also drop the READ reader if present: we don't - // need its deletes since they've been carried - // over to the merged segment - cacheKey = new SegmentCacheKey(sr.getSegmentInfo(), IOContext.Context.READ); - SegmentReader sr2 = readerMap.get(cacheKey); - if (sr2 != null) { - readerMap.remove(cacheKey); - sr2.hasChanges = false; - sr2.close(); + if (!drop) { + if (rld.writeLiveDocs(directory)) { + assert infoIsLive(sr.getSegmentInfo()); + // Must checkpoint w/ deleter, because we just + // created created new _X_N.del file. + deleter.checkpoint(segmentInfos, false); } } - return hasChanges; + if (!rld.removeReader(sr, drop)) { + //System.out.println("DROP seg=" + rld.info + " " + readerMap.size() + " in pool"); + readerMap.remove(sr.getSegmentInfo()); + } } - - return false; } - public synchronized void drop(List infos) throws IOException { - drop(infos, IOContext.Context.READ); - drop(infos, IOContext.Context.MERGE); - } - - public synchronized void drop(List infos, IOContext.Context context) throws IOException { - for(SegmentInfo info : infos) { - drop(info, context); - } - } - - public synchronized void drop(SegmentInfo info) throws IOException { - drop(info, IOContext.Context.READ); - drop(info, IOContext.Context.MERGE); - } - - public synchronized void dropAll() throws IOException { - for(SegmentReader reader : readerMap.values()) { - reader.hasChanges = false; - - // NOTE: it is allowed that this decRef does not - // actually close the SR; this can happen when a - // near real-time reader using this SR is still open - reader.decRef(); - } - readerMap.clear(); - } - - public synchronized void drop(SegmentInfo info, IOContext.Context context) throws IOException { - final SegmentReader sr; - if ((sr = readerMap.remove(new SegmentCacheKey(info, context))) != null) { - sr.hasChanges = false; - readerMap.remove(new SegmentCacheKey(info, context)); - sr.close(); - } - } - /** Remove all our references to readers, and commits * any pending changes. */ - synchronized void close() throws IOException { - // We invoke deleter.checkpoint below, so we must be - // sync'd on IW: - assert Thread.holdsLock(IndexWriter.this); - - for(Map.Entry ent : readerMap.entrySet()) { - - SegmentReader sr = ent.getValue(); - if (sr.hasChanges) { - assert infoIsLive(sr.getSegmentInfo(), "key=" + ent.getKey()); - sr.doCommit(); - - // Must checkpoint w/ deleter, because this - // segment reader will have created new _X_N.del - // file. + synchronized void dropAll(boolean doSave) throws IOException { + final Iterator> it = readerMap.entrySet().iterator(); + while(it.hasNext()) { + final ReadersAndLiveDocs rld = it.next().getValue(); + //System.out.println("pool.dropAll: seg=" + rld.info); + if (doSave && rld.writeLiveDocs(directory)) { + assert infoIsLive(rld.info); + // Must checkpoint w/ deleter, because we just + // created created new _X_N.del file. deleter.checkpoint(segmentInfos, false); } - // NOTE: it is allowed that this decRef does not - // actually close the SR; this can happen when a + // Important to remove as-we-go, not with .clear() + // in the end, in case we hit an exception; + // otherwise we could over-decref if close() is + // called again: + it.remove(); + + // NOTE: it is allowed that these decRefs do not + // actually close the SRs; this happens when a // near real-time reader is kept open after the - // IndexWriter instance is closed - sr.decRef(); + // IndexWriter instance is closed: + rld.dropReaders(); } + assert readerMap.size() == 0; + } - readerMap.clear(); + public synchronized void drop(SegmentInfo info) throws IOException { + final ReadersAndLiveDocs rld = readerMap.remove(info); + if (rld != null) { + rld.dropReaders(); + } } /** - * Commit all segment reader in the pool. + * Commit live docs changes for the segment readers for + * the provided infos. + * * @throws IOException */ - synchronized void commit(SegmentInfos infos) throws IOException { - - // We invoke deleter.checkpoint below, so we must be - // sync'd on IW: - assert Thread.holdsLock(IndexWriter.this); - + public synchronized void commit(SegmentInfos infos) throws IOException { for (SegmentInfo info : infos) { - final SegmentReader sr = readerMap.get(new SegmentCacheKey(info, IOContext.Context.READ)); - if (sr != null && sr.hasChanges) { + final ReadersAndLiveDocs rld = readerMap.get(info); + if (rld != null && rld.writeLiveDocs(directory)) { assert infoIsLive(info); - sr.doCommit(); - // Must checkpoint w/ deleter, because this - // segment reader will have created new _X_N.del - // file. + // Must checkpoint w/ deleter, because we just + // created created new _X_N.del file. deleter.checkpoint(segmentInfos, false); } } } - public synchronized SegmentReader getReadOnlyClone(SegmentInfo info, IOContext context) throws IOException { - return getReadOnlyClone(info, true, context); - } - /** - * Returns a ref to a clone. NOTE: this clone is not - * enrolled in the pool, so you should simply close() - * it when you're done (ie, do not call release()). - */ - public synchronized SegmentReader getReadOnlyClone(SegmentInfo info, boolean doOpenStores, IOContext context) throws IOException { - SegmentReader sr = get(info, doOpenStores, context); - try { - return (SegmentReader) sr.clone(); // cloning is always readOnly - } finally { - sr.decRef(); - } - } - - public synchronized SegmentReader get(SegmentInfo info, IOContext context) throws IOException { - return get(info, true, context); - } - - /** - * Obtain a SegmentReader from the readerPool. The reader - * must be returned by calling {@link #release(SegmentReader)} - * @see #release(SegmentReader) - * @param info - * @param doOpenStores + * Obtain a ReadersAndLiveDocs instance from the + * readerPool. If getReader is true, you must later call + * {@link #release(SegmentReader)}. * @throws IOException */ - public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, IOContext context) throws IOException { + public synchronized ReadersAndLiveDocs get(SegmentInfo info, boolean create) { - SegmentCacheKey cacheKey = new SegmentCacheKey(info, context.context); - SegmentReader sr = readerMap.get(cacheKey); - if (sr == null) { - // TODO: we may want to avoid doing this while - // synchronized - // Returns a ref, which we xfer to readerMap: - sr = SegmentReader.getRW(info, doOpenStores, context.context == IOContext.Context.MERGE ? -1 : config.getReaderTermsIndexDivisor(), context); + assert info.dir == directory; - if (info.dir == directory) { - // Only pool if reader is not external - readerMap.put(cacheKey, sr); + ReadersAndLiveDocs rld = readerMap.get(info); + //System.out.println("rld.get seg=" + info + " poolReaders=" + poolReaders); + if (rld == null) { + //System.out.println(" new rld"); + if (!create) { + return null; } - } else { - if (doOpenStores) { - sr.openDocStores(); - } + rld = new ReadersAndLiveDocs(info); + readerMap.put(info, rld); } - - // Return a ref to our caller - if (info.dir == directory) { - // Only incRef if we pooled (reader is not external) - sr.incRef(); - } - return sr; + return rld; } + } - // Returns a ref - public synchronized SegmentReader getIfExists(SegmentInfo info) throws IOException { - SegmentReader sr = getIfExists(info, IOContext.Context.READ); - if (sr == null) { - sr = getIfExists(info, IOContext.Context.MERGE); - } - return sr; - } - - // Returns a ref - public synchronized SegmentReader getIfExists(SegmentInfo info, IOContext.Context context) throws IOException { - SegmentCacheKey cacheKey = new SegmentCacheKey(info, context); - SegmentReader sr = readerMap.get(cacheKey); - if (sr != null) { - sr.incRef(); - } - return sr; - } - } - /** * Obtain the number of deleted docs for a pooled reader. * If the reader isn't being pooled, the segmentInfo's @@ -740,21 +798,13 @@ */ public int numDeletedDocs(SegmentInfo info) throws IOException { ensureOpen(false); - SegmentReader reader = readerPool.getIfExists(info); - try { - if (reader != null) { - // the pulled reader could be from an in-flight merge - // while the info we see has already new applied deletes after a commit - // we max out the deletes since deletes never shrink - return Math.max(info.getDelCount(), reader.numDeletedDocs()); - } else { - return info.getDelCount(); - } - } finally { - if (reader != null) { - readerPool.release(reader, false); - } + int delCount = info.getDelCount(); + + final ReadersAndLiveDocs rld = readerPool.get(info, false); + if (rld != null) { + delCount += rld.pendingDeleteCount; } + return delCount; } /** @@ -1089,7 +1139,7 @@ // used by assert below final DocumentsWriter oldWriter = docWriter; synchronized(this) { - readerPool.close(); + readerPool.dropAll(true); docWriter = null; deleter.close(); } @@ -1994,11 +2044,11 @@ // them: deleter.checkpoint(segmentInfos, false); deleter.refresh(); + + // Don't bother saving any changes in our segmentInfos + readerPool.dropAll(false); } - // Don't bother saving any changes in our segmentInfos - readerPool.clear(null); - lastCommitChangeCount = changeCount; success = true; @@ -2054,7 +2104,7 @@ deleter.refresh(); // Don't bother saving any changes in our segmentInfos - readerPool.dropAll(); + readerPool.dropAll(false); // Mark that the index has changed ++changeCount; @@ -2079,7 +2129,7 @@ // Abort all pending & running merges: for (final MergePolicy.OneMerge merge : pendingMerges) { if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "now abort pending merge " + merge.segString(directory)); + infoStream.message("IW", "now abort pending merge " + segString(merge.segments)); } merge.abort(); mergeFinish(merge); @@ -2088,7 +2138,7 @@ for (final MergePolicy.OneMerge merge : runningMerges) { if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "now abort running merge " + merge.segString(directory)); + infoStream.message("IW", "now abort running merge " + segString(merge.segments)); } merge.abort(); } @@ -2289,7 +2339,7 @@ nextGen = bufferedDeletesStream.getNextGen(); } if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "publish sets newSegment delGen=" + nextGen); + infoStream.message("IW", "publish sets newSegment delGen=" + nextGen + " seg=" + newSegment); } newSegment.setBufferedDeletesGen(nextGen); segmentInfos.add(newSegment); @@ -2918,9 +2968,7 @@ // it once it's done: if (!mergingSegments.contains(info)) { segmentInfos.remove(info); - if (readerPool != null) { - readerPool.drop(info); - } + readerPool.drop(info); } } checkpoint(); @@ -2950,7 +2998,7 @@ return docWriter.getNumDocs(); } - private void ensureValidMerge(MergePolicy.OneMerge merge) throws IOException { + private synchronized void ensureValidMerge(MergePolicy.OneMerge merge) throws IOException { for(SegmentInfo info : merge.segments) { if (!segmentInfos.contains(info)) { throw new MergePolicy.MergeException("MergePolicy selected a segment (" + info.name + ") that is not in the current index " + segString(), directory); @@ -2967,47 +3015,41 @@ * saves the resulting deletes file (incrementing the * delete generation for merge.info). If no deletes were * flushed, no new deletes file is saved. */ - synchronized private void commitMergedDeletes(MergePolicy.OneMerge merge, SegmentReader mergedReader) throws IOException { + synchronized private ReadersAndLiveDocs commitMergedDeletes(MergePolicy.OneMerge merge) throws IOException { assert testPoint("startCommitMergeDeletes"); final List sourceSegments = merge.segments; if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "commitMergeDeletes " + merge.segString(directory)); + infoStream.message("IW", "commitMergeDeletes " + segString(merge.segments)); } // Carefully merge deletes that occurred after we // started merging: int docUpto = 0; - int delCount = 0; long minGen = Long.MAX_VALUE; + // Lazy init (only when we find a delete to carry over): + ReadersAndLiveDocs mergedDeletes = null; + for(int i=0; i < sourceSegments.size(); i++) { SegmentInfo info = sourceSegments.get(i); minGen = Math.min(info.getBufferedDeletesGen(), minGen); final int docCount = info.docCount; final BitVector prevLiveDocs = merge.readerLiveDocs.get(i); final BitVector currentLiveDocs; - { - final SegmentReader currentReader = readerPool.getIfExists(info, IOContext.Context.READ); - if (currentReader != null) { - currentLiveDocs = (BitVector) currentReader.getLiveDocs(); - readerPool.release(currentReader, false, IOContext.Context.READ); - } else { - assert readerPool.infoIsLive(info); - if (info.hasDeletions()) { - currentLiveDocs = new BitVector(directory, - info.getDelFileName(), - new IOContext(IOContext.Context.READ)); - } else { - currentLiveDocs = null; - } - } - } + ReadersAndLiveDocs rld = readerPool.get(info, false); + // We enrolled in mergeInit: + assert rld != null; + currentLiveDocs = rld.liveDocs; if (prevLiveDocs != null) { + // If we had deletions on starting the merge we must + // still have deletions now: + assert currentLiveDocs != null; + // There were deletes on this segment when the merge // started. The merge has collapsed away those // deletes, but, if new deletes were flushed since @@ -3024,8 +3066,11 @@ assert !currentLiveDocs.get(j); } else { if (!currentLiveDocs.get(j)) { - mergedReader.deleteDocument(docUpto); - delCount++; + if (mergedDeletes == null) { + mergedDeletes = readerPool.get(merge.info, true); + mergedDeletes.writableLiveDocs(); + } + mergedDeletes.delete(docUpto); } docUpto++; } @@ -3039,8 +3084,11 @@ // does: for(int j=0; j 0; - // If new deletes were applied while we were merging // (which happens if eg commit() or getReader() is // called during our merge), then it better be the case // that the delGen has increased for all our merged // segments: - assert !mergedReader.hasChanges || minGen > mergedReader.getSegmentInfo().getBufferedDeletesGen(); + assert mergedDeletes == null || minGen > merge.info.getBufferedDeletesGen(); - mergedReader.getSegmentInfo().setBufferedDeletesGen(minGen); + merge.info.setBufferedDeletesGen(minGen); + + return mergedDeletes; } - synchronized private boolean commitMerge(MergePolicy.OneMerge merge, SegmentReader mergedReader) throws IOException { + synchronized private boolean commitMerge(MergePolicy.OneMerge merge) throws IOException { assert testPoint("startCommitMerge"); @@ -3073,7 +3127,7 @@ } if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "commitMerge: " + merge.segString(directory) + " index=" + segString()); + infoStream.message("IW", "commitMerge: " + segString(merge.segments) + " index=" + segString()); } assert merge.registerDone; @@ -3086,13 +3140,15 @@ // abort this merge if (merge.isAborted()) { if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "commitMerge: skipping merge " + merge.segString(directory) + ": it was aborted"); + infoStream.message("IW", "commitMerge: skip: it was aborted"); } return false; } - commitMergedDeletes(merge, mergedReader); + final ReadersAndLiveDocs mergedDeletes = commitMergedDeletes(merge); + assert mergedDeletes == null || mergedDeletes.pendingDeleteCount != 0; + // If the doc store we are using has been closed and // is in now compound format (but wasn't when we // started), then we will switch to the compound @@ -3100,36 +3156,46 @@ assert !segmentInfos.contains(merge.info); - final boolean allDeleted = mergedReader.numDocs() == 0; + final boolean allDeleted = merge.segments.size() == 0 || + merge.info.docCount == 0 || + (mergedDeletes != null && + mergedDeletes.pendingDeleteCount == merge.info.docCount); - if (allDeleted) { - if (infoStream.isEnabled("IW")) { + if (infoStream.isEnabled("IW")) { + if (allDeleted) { infoStream.message("IW", "merged segment " + merge.info + " is 100% deleted" + (keepFullyDeletedSegments ? "" : "; skipping insert")); } } final boolean dropSegment = allDeleted && !keepFullyDeletedSegments; + + // If we merged no segments then we better be dropping + // the new segment: + assert merge.segments.size() > 0 || dropSegment; + + assert merge.info.docCount != 0 || keepFullyDeletedSegments || dropSegment; + segmentInfos.applyMergeChanges(merge, dropSegment); if (dropSegment) { readerPool.drop(merge.info); + } else { + if (mergedDeletes != null && !poolReaders) { + mergedDeletes.writeLiveDocs(directory); + readerPool.drop(merge.info); + } } + // Must note the change to segmentInfos so any commits + // in-flight don't lose it: + checkpoint(); + if (infoStream.isEnabled("IW")) { infoStream.message("IW", "after commit: " + segString()); } closeMergeReaders(merge, false); - // Must note the change to segmentInfos so any commits - // in-flight don't lose it: - checkpoint(); - - // If the merged segments had pending changes, clear - // them so that they don't bother writing them to - // disk, updating SegmentInfo, etc.: - readerPool.clear(merge.segments); - if (merge.maxNumSegments != -1) { // cascade the forceMerge: if (!segmentsToMerge.containsKey(merge.info)) { @@ -3143,7 +3209,7 @@ final private void handleMergeException(Throwable t, MergePolicy.OneMerge merge) throws IOException { if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "handleMergeException: merge=" + merge.segString(directory) + " exc=" + t); + infoStream.message("IW", "handleMergeException: merge=" + segString(merge.segments) + " exc=" + t); } // Set the exception on the merge, so if @@ -3184,7 +3250,6 @@ boolean success = false; final long t0 = System.currentTimeMillis(); - //System.out.println(Thread.currentThread().getName() + ": merge start: size=" + (merge.estimatedMergeBytes/1024./1024.) + " MB\n merge=" + merge.segString(directory) + "\n idx=" + segString()); try { try { @@ -3192,7 +3257,7 @@ mergeInit(merge); if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "now merge\n merge=" + merge.segString(directory) + "\n index=" + segString()); + infoStream.message("IW", "now merge\n merge=" + segString(merge.segments) + "\n index=" + segString()); } mergeMiddle(merge); @@ -3245,12 +3310,14 @@ * returned. */ final synchronized boolean registerMerge(MergePolicy.OneMerge merge) throws MergePolicy.MergeAbortedException, IOException { - if (merge.registerDone) + if (merge.registerDone) { return true; + } + assert merge.segments.size() > 0; if (stopMerges) { merge.abort(); - throw new MergePolicy.MergeAbortedException("merge is aborted: " + merge.segString(directory)); + throw new MergePolicy.MergeAbortedException("merge is aborted: " + segString(merge.segments)); } boolean isExternal = false; @@ -3274,7 +3341,7 @@ pendingMerges.add(merge); if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "add merge to pendingMerges: " + merge.segString(directory) + " [total " + pendingMerges.size() + " pending]"); + infoStream.message("IW", "add merge to pendingMerges: " + segString(merge.segments) + " [total " + pendingMerges.size() + " pending]"); } merge.mergeGen = mergeGen; @@ -3352,6 +3419,11 @@ // names. merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, new FieldInfos(globalFieldNumberMap)); + // TODO: in the non-pool'd case this is somewhat + // wasteful, because we open these readers, close them, + // and then open them again for merging. Maybe we + // could pre-pool them somehow in that case... + // Lock order: IW -> BD final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments); @@ -3369,10 +3441,8 @@ mergingSegments.remove(info); merge.segments.remove(info); } + readerPool.drop(info); } - if (readerPool != null) { - readerPool.drop(result.allDeleted); - } checkpoint(); } @@ -3457,13 +3527,12 @@ final int numSegments = merge.readers.size(); Throwable th = null; - boolean anyChanges = false; boolean drop = !suppressExceptions; for (int i = 0; i < numSegments; i++) { if (merge.readers.get(i) != null) { try { - anyChanges |= readerPool.release(merge.readers.get(i), drop, IOContext.Context.MERGE); + readerPool.release(merge.readers.get(i), drop); } catch (Throwable t) { if (th == null) { th = t; @@ -3473,10 +3542,6 @@ } } - if (suppressExceptions && anyChanges) { - checkpoint(); - } - // If any error occured, throw it. if (!suppressExceptions && th != null) { if (th instanceof IOException) throw (IOException) th; @@ -3486,27 +3551,6 @@ } } - private synchronized BitVector getLiveDocsClone(SegmentInfo info, SegmentReader other) throws IOException { - final SegmentReader delReader = readerPool.getIfExists(info, IOContext.Context.READ); - BitVector liveDocs; - if (delReader != null) { - liveDocs = (BitVector) delReader.getLiveDocs(); - readerPool.release(delReader, false, IOContext.Context.READ); - if (liveDocs != null) { - // We clone the del docs because other - // deletes may come in while we're merging. We - // need frozen deletes while merging, and then - // we carry over any new deletions in - // commitMergedDeletes. - liveDocs = (BitVector) liveDocs.clone(); - } - } else { - liveDocs = (BitVector) other.getLiveDocs(); - } - - return liveDocs; - } - /** Does the actual (time-consuming) work of the merge, * but without holding synchronized lock on IndexWriter * instance */ @@ -3528,7 +3572,7 @@ payloadProcessorProvider, merge.info.getFieldInfos(), codec, context); if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "merging " + merge.segString(directory) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors()); + infoStream.message("IW", "merging " + segString(merge.segments) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors()); } merge.readers = new ArrayList(); @@ -3546,17 +3590,37 @@ // Hold onto the "live" reader; we will use this to // commit merged deletes - final SegmentReader reader = readerPool.get(info, context); + final ReadersAndLiveDocs rld = readerPool.get(info, true); + final SegmentReader reader = rld.getMergeReader(context); + assert reader != null; // Carefully pull the most recent live docs: - final BitVector liveDocs = getLiveDocsClone(info, reader); + final BitVector liveDocs; + synchronized(this) { + // Must sync to ensure BufferedDeletesStream + // cannot change liveDocs/pendingDeleteCount while + // we pull a copy: + liveDocs = rld.getReadOnlyLiveDocs(); + if (infoStream.isEnabled("IW")) { + if (rld.pendingDeleteCount != 0) { + infoStream.message("IW", "seg=" + info + " delCount=" + info.getDelCount() + " pendingDelCount=" + rld.pendingDeleteCount); + } else if (info.getDelCount() != 0) { + infoStream.message("IW", "seg=" + info + " delCount=" + info.getDelCount()); + } else { + infoStream.message("IW", "seg=" + info + " no deletes"); + } + } + } + merge.readerLiveDocs.add(liveDocs); merge.readers.add(reader); if (liveDocs == null || liveDocs.count() > 0) { merger.add(reader, liveDocs); totDocCount += liveDocs == null ? reader.maxDoc() : liveDocs.count(); + } else { + //System.out.println(" skip seg: fully deleted"); } segUpto++; } @@ -3651,41 +3715,23 @@ final IndexReaderWarmer mergedSegmentWarmer = config.getMergedSegmentWarmer(); - // TODO: in the non-realtime case, we may want to only - // keep deletes (it's costly to open entire reader - // when we just need deletes) - - final boolean loadDocStores; - if (mergedSegmentWarmer != null) { - // Load terms index & doc stores so the segment - // warmer can run searches, load documents/term - // vectors - loadDocStores = true; - } else { - loadDocStores = false; + if (poolReaders && mergedSegmentWarmer != null) { + final ReadersAndLiveDocs rld = readerPool.get(merge.info, true); + final SegmentReader sr = rld.getReader(IOContext.READ); + try { + mergedSegmentWarmer.warm(sr); + } finally { + synchronized(this) { + readerPool.release(sr, false); + } + } } // Force READ context because we merge deletes onto // this reader: - final SegmentReader mergedReader = readerPool.get(merge.info, loadDocStores, new IOContext(IOContext.Context.READ)); - try { - if (poolReaders && mergedSegmentWarmer != null) { - mergedSegmentWarmer.warm(mergedReader); - } - - if (!commitMerge(merge, mergedReader)) { - // commitMerge will return false if this merge was aborted - return 0; - } - } finally { - synchronized(this) { - if (readerPool.release(mergedReader, IOContext.Context.READ)) { - // Must checkpoint after releasing the - // mergedReader since it may have written a new - // deletes file: - checkpoint(); - } - } + if (!commitMerge(merge)) { + // commitMerge will return false if this merge was aborted + return 0; } success = true; @@ -3742,22 +3788,7 @@ /** @lucene.internal */ public synchronized String segString(SegmentInfo info) throws IOException { StringBuilder buffer = new StringBuilder(); - SegmentReader reader = readerPool.getIfExists(info); - try { - if (reader != null) { - buffer.append(reader.toString()); - } else { - buffer.append(info.toString(directory, 0)); - if (info.dir != directory) { - buffer.append("**"); - } - } - } finally { - if (reader != null) { - readerPool.release(reader, false); - } - } - return buffer.toString(); + return info.toString(info.dir, numDeletedDocs(info) - info.getDelCount()); } private synchronized void doWait() { Index: lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (working copy) @@ -450,8 +450,10 @@ public void checkpoint(SegmentInfos segmentInfos, boolean isCommit) throws IOException { assert locked(); + assert Thread.holdsLock(writer); + if (infoStream.isEnabled("IFD")) { - infoStream.message("IFD", "now checkpoint \"" + segmentInfos.toString(directory) + "\" [" + segmentInfos.size() + " segments " + "; isCommit = " + isCommit + "]"); + infoStream.message("IFD", "now checkpoint \"" + writer.segString(segmentInfos) + "\" [" + segmentInfos.size() + " segments " + "; isCommit = " + isCommit + "]"); } // Try again now to delete any previously un-deletable Index: lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (revision 1220696) +++ lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (working copy) @@ -62,10 +62,10 @@ private final SegmentReader owner; - StoredFieldsReader fieldsReaderOrig; - TermVectorsReader termVectorsReaderOrig; - CompoundFileDirectory cfsReader; - CompoundFileDirectory storeCFSReader; + final StoredFieldsReader fieldsReaderOrig; + final TermVectorsReader termVectorsReaderOrig; + final CompoundFileDirectory cfsReader; + final CompoundFileDirectory storeCFSReader; final Set coreClosedListeners = new MapBackedSet(new ConcurrentHashMap()); @@ -88,6 +88,8 @@ if (si.getUseCompoundFile()) { cfsReader = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); dir0 = cfsReader; + } else { + cfsReader = null; } cfsDir = dir0; si.loadFieldInfos(cfsDir, false); // prevent opening the CFS to load fieldInfos @@ -104,6 +106,38 @@ // kinda jaky to assume the codec handles the case of no norms file at all gracefully?! norms = codec.normsFormat().normsReader(cfsDir, si, fieldInfos, context, dir); perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState); + + final Directory storeDir; + if (si.getDocStoreOffset() != -1) { + if (si.getDocStoreIsCompoundFile()) { + storeCFSReader = new CompoundFileDirectory(dir, + IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), + context, false); + storeDir = storeCFSReader; + assert storeDir != null; + } else { + storeCFSReader = null; + storeDir = dir; + assert storeDir != null; + } + } else if (si.getUseCompoundFile()) { + storeDir = cfsReader; + storeCFSReader = null; + assert storeDir != null; + } else { + storeDir = dir; + storeCFSReader = null; + assert storeDir != null; + } + + fieldsReaderOrig = si.getCodec().storedFieldsFormat().fieldsReader(storeDir, si, fieldInfos, context); + + if (si.getHasVectors()) { // open term vector files only as needed + termVectorsReaderOrig = si.getCodec().termVectorsFormat().vectorsReader(storeDir, si, fieldInfos, context); + } else { + termVectorsReaderOrig = null; + } + success = true; } finally { if (!success) { @@ -135,6 +169,7 @@ } synchronized void decRef() throws IOException { + //System.out.println("core.decRef seg=" + owner.getSegmentInfo() + " rc=" + ref); if (ref.decrementAndGet() == 0) { IOUtils.close(fields, perDocProducer, termVectorsReaderOrig, fieldsReaderOrig, cfsReader, storeCFSReader, norms); @@ -144,46 +179,6 @@ } } - synchronized void openDocStores(SegmentInfo si) throws IOException { - - assert si.name.equals(segment); - - if (fieldsReaderOrig == null) { - final Directory storeDir; - if (si.getDocStoreOffset() != -1) { - if (si.getDocStoreIsCompoundFile()) { - assert storeCFSReader == null; - storeCFSReader = new CompoundFileDirectory(dir, - IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), - context, false); - storeDir = storeCFSReader; - assert storeDir != null; - } else { - storeDir = dir; - assert storeDir != null; - } - } else if (si.getUseCompoundFile()) { - // In some cases, we were originally opened when CFS - // was not used, but then we are asked to open doc - // stores after the segment has switched to CFS - if (cfsReader == null) { - cfsReader = new CompoundFileDirectory(dir,IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); - } - storeDir = cfsReader; - assert storeDir != null; - } else { - storeDir = dir; - assert storeDir != null; - } - - fieldsReaderOrig = si.getCodec().storedFieldsFormat().fieldsReader(storeDir, si, fieldInfos, context); - - if (si.getHasVectors()) { // open term vector files only as needed - termVectorsReaderOrig = si.getCodec().termVectorsFormat().vectorsReader(storeDir, si, fieldInfos, context); - } - } - } - @Override public String toString() { return "SegmentCoreReader(owner=" + owner + ")"; Index: lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java =================================================================== --- lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java (revision 1220696) +++ lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java (working copy) @@ -65,6 +65,8 @@ protected List computeTestMethods() { if (testMethods != null) return testMethods; + + System.out.println("COMPUTE TEST METHODS: " + this); Random r = new Random(runnerSeed);