Index: lucene/core/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (revision 1564833) +++ lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -4643,6 +4643,14 @@ } + synchronized void incRefDeleter(SegmentInfos segmentInfos) throws IOException { + deleter.incRef(segmentInfos, false); + } + + synchronized void decRefDeleter(SegmentInfos segmentInfos) throws IOException { + deleter.decRef(segmentInfos); + } + private boolean processEvents(boolean triggerMerge, boolean forcePurge) throws IOException { return processEvents(eventQueue, triggerMerge, forcePurge); } @@ -4680,4 +4688,5 @@ */ void process(IndexWriter writer, boolean triggerMerge, boolean clearBuffers) throws IOException; } + } Index: lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (revision 1564833) +++ lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (working copy) @@ -107,6 +107,9 @@ writer.readerPool.release(rld); } } + + writer.incRefDeleter(segmentInfos); + StandardDirectoryReader result = new StandardDirectoryReader(dir, readers.toArray(new SegmentReader[readers.size()]), writer, segmentInfos, applyAllDeletes); @@ -362,6 +365,8 @@ } if (writer != null) { + writer.decRefDeleter(segmentInfos); + // Since we just closed, writer may now be able to // delete unused files: writer.deletePendingFiles(); Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (revision 1564833) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterReader.java (working copy) @@ -67,7 +67,7 @@ } public void testAddCloseOpen() throws IOException { - Directory dir1 = newDirectory(); + Directory dir1 = getAssertNoDeletesDirectory(newDirectory()); IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); IndexWriter writer = new IndexWriter(dir1, iwc); @@ -116,7 +116,7 @@ public void testUpdateDocument() throws Exception { boolean doFullMerge = true; - Directory dir1 = newDirectory(); + Directory dir1 = getAssertNoDeletesDirectory(newDirectory()); IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); if (iwc.getMaxBufferedDocs() < 20) { iwc.setMaxBufferedDocs(20); @@ -230,7 +230,7 @@ public void testAddIndexes() throws Exception { boolean doFullMerge = false; - Directory dir1 = newDirectory(); + Directory dir1 = getAssertNoDeletesDirectory(newDirectory()); IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); if (iwc.getMaxBufferedDocs() < 20) { iwc.setMaxBufferedDocs(20); @@ -285,7 +285,7 @@ public void testAddIndexes2() throws Exception { boolean doFullMerge = false; - Directory dir1 = newDirectory(); + Directory dir1 = getAssertNoDeletesDirectory(newDirectory()); IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); // create a 2nd index @@ -315,7 +315,7 @@ public void testDeleteFromIndexWriter() throws Exception { boolean doFullMerge = true; - Directory dir1 = newDirectory(); + Directory dir1 = getAssertNoDeletesDirectory(newDirectory()); IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); // create the index createIndexNoClose(!doFullMerge, "index1", writer); @@ -365,7 +365,8 @@ final int numIter = 2; int numDirs = 3; - Directory mainDir = newDirectory(); + Directory mainDir = getAssertNoDeletesDirectory(newDirectory()); + IndexWriter mainWriter = new IndexWriter(mainDir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy())); _TestUtil.reduceOpenFiles(mainWriter); @@ -518,7 +519,7 @@ * IW.getReader */ public void doTestIndexWriterReopenSegment(boolean doFullMerge) throws Exception { - Directory dir1 = newDirectory(); + Directory dir1 = getAssertNoDeletesDirectory(newDirectory()); IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); IndexReader r1 = writer.getReader(); assertEquals(0, r1.maxDoc()); @@ -608,7 +609,7 @@ public void testMergeWarmer() throws Exception { - Directory dir1 = newDirectory(); + Directory dir1 = getAssertNoDeletesDirectory(newDirectory()); // Enroll warmer MyWarmer warmer = new MyWarmer(); IndexWriter writer = new IndexWriter( @@ -647,7 +648,7 @@ } public void testAfterCommit() throws Exception { - Directory dir1 = newDirectory(); + Directory dir1 = getAssertNoDeletesDirectory(newDirectory()); IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())).setMergeScheduler(new ConcurrentMergeScheduler())); writer.commit(); @@ -679,7 +680,7 @@ // Make sure reader remains usable even if IndexWriter closes public void testAfterClose() throws Exception { - Directory dir1 = newDirectory(); + Directory dir1 = getAssertNoDeletesDirectory(newDirectory()); IndexWriter writer = new IndexWriter(dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random()))); // create the index @@ -707,7 +708,7 @@ // Stress test reopen during addIndexes public void testDuringAddIndexes() throws Exception { - Directory dir1 = newDirectory(); + Directory dir1 = getAssertNoDeletesDirectory(newDirectory()); final IndexWriter writer = new IndexWriter( dir1, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random())). @@ -792,6 +793,13 @@ dir1.close(); } + private Directory getAssertNoDeletesDirectory(Directory directory) { + if (directory instanceof MockDirectoryWrapper) { + ((MockDirectoryWrapper)directory).setAssertNoDeleteOpenFile(true); + } + return directory; + } + // Stress test reopen during add/delete public void testDuringAddDelete() throws Exception { Directory dir1 = newDirectory(); @@ -1011,7 +1019,7 @@ } public void testReopenAfterNoRealChange() throws Exception { - Directory d = newDirectory(); + Directory d = getAssertNoDeletesDirectory(newDirectory()); IndexWriter w = new IndexWriter( d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); @@ -1050,7 +1058,7 @@ public void testNRTOpenExceptions() throws Exception { // LUCENE-5262: test that several failed attempts to obtain an NRT reader // don't leak file handles. - MockDirectoryWrapper dir = newMockDirectory(); + MockDirectoryWrapper dir = (MockDirectoryWrapper) getAssertNoDeletesDirectory(newMockDirectory()); final AtomicBoolean shouldFail = new AtomicBoolean(); dir.failOn(new MockDirectoryWrapper.Failure() { @Override @@ -1104,7 +1112,7 @@ /** Make sure if all we do is open NRT reader against * writer, we don't see merge starvation. */ public void testTooManySegments() throws Exception { - Directory dir = newDirectory(); + Directory dir = getAssertNoDeletesDirectory(newDirectory()); // Don't use newIndexWriterConfig, because we need a // "sane" mergePolicy: IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); Index: lucene/core/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java (revision 1564833) +++ lucene/core/src/test/org/apache/lucene/index/TestNRTReaderWithThreads.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; public class TestNRTReaderWithThreads extends LuceneTestCase { @@ -30,6 +31,9 @@ public void testIndexing() throws Exception { Directory mainDir = newDirectory(); + if (mainDir instanceof MockDirectoryWrapper) { + ((MockDirectoryWrapper)mainDir).setAssertNoDeleteOpenFile(true); + } IndexWriter writer = new IndexWriter( mainDir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())). Index: lucene/core/src/test/org/apache/lucene/index/TestNRTThreads.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestNRTThreads.java (revision 1564833) +++ lucene/core/src/test/org/apache/lucene/index/TestNRTThreads.java (working copy) @@ -21,6 +21,7 @@ import java.util.concurrent.ExecutorService; import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; @@ -54,7 +55,7 @@ } r.close(); writer.commit(); - final Set openDeletedFiles = dir.getOpenDeletedFiles(); + final Set openDeletedFiles = ((MockDirectoryWrapper) dir).getOpenDeletedFiles(); if (openDeletedFiles.size() > 0) { System.out.println("OBD files: " + openDeletedFiles); } @@ -80,7 +81,7 @@ r.close(); //System.out.println("numDocs=" + r.numDocs() + " openDelFileCount=" + dir.openDeleteFileCount()); - final Set openDeletedFiles = dir.getOpenDeletedFiles(); + final Set openDeletedFiles = ((MockDirectoryWrapper) dir).getOpenDeletedFiles(); if (openDeletedFiles.size() > 0) { System.out.println("OBD files: " + openDeletedFiles); } @@ -88,6 +89,13 @@ assertFalse("saw non-zero open-but-deleted count", anyOpenDelFiles); } + + @Override + protected Directory getDirectory(Directory in) { + assert in instanceof MockDirectoryWrapper; + ((MockDirectoryWrapper) in).setAssertNoDeleteOpenFile(true); + return in; + } @Override protected void doAfterWriter(ExecutorService es) throws Exception { Index: lucene/core/src/test/org/apache/lucene/index/TestStressIndexing.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestStressIndexing.java (revision 1564833) +++ lucene/core/src/test/org/apache/lucene/index/TestStressIndexing.java (working copy) @@ -165,6 +165,10 @@ */ public void testStressIndexAndSearching() throws Exception { Directory directory = newDirectory(); + if (directory instanceof MockDirectoryWrapper) { + ((MockDirectoryWrapper) directory).setAssertNoUnrefencedFilesOnClose(true); + } + runStressTest(directory, new ConcurrentMergeScheduler()); directory.close(); } Index: lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java (revision 1564833) +++ lucene/test-framework/src/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java (working copy) @@ -37,6 +37,7 @@ import org.apache.lucene.search.SortField; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.Bits; @@ -61,7 +62,7 @@ protected final AtomicInteger delCount = new AtomicInteger(); protected final AtomicInteger packCount = new AtomicInteger(); - protected MockDirectoryWrapper dir; + protected Directory dir; protected IndexWriter writer; private static class SubDocs { @@ -436,8 +437,10 @@ Random random = new Random(random().nextLong()); final LineFileDocs docs = new LineFileDocs(random, true); final File tempDir = _TestUtil.getTempDir(testName); - dir = newMockFSDirectory(tempDir); // some subclasses rely on this being MDW - dir.setCheckIndexOnClose(false); // don't double-checkIndex, we do it ourselves. + dir = getDirectory(newMockFSDirectory(tempDir)); // some subclasses rely on this being MDW + if (dir instanceof BaseDirectoryWrapper) { + ((BaseDirectoryWrapper) dir).setCheckIndexOnClose(false); // don't double-checkIndex, we do it ourselves. + } final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setInfoStream(new FailOnNonBulkMergesInfoStream()); Index: lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (revision 1564833) +++ lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (working copy) @@ -70,6 +70,7 @@ double randomIOExceptionRateOnOpen; Random randomState; boolean noDeleteOpenFile = true; + boolean assertNoDeleteOpenFile = false; boolean preventDoubleWrite = true; boolean trackDiskUsage = false; boolean wrapLockFactory = true; @@ -316,9 +317,22 @@ public void setNoDeleteOpenFile(boolean value) { this.noDeleteOpenFile = value; } + public boolean getNoDeleteOpenFile() { return noDeleteOpenFile; } + + /** + * Trip a test assert if there is an attempt + * to delete an open file. + */ + public void setAssertNoDeleteOpenFile(boolean value) { + this.assertNoDeleteOpenFile = value; + } + + public boolean getAssertNoDeleteOpenFile() { + return assertNoDeleteOpenFile; + } /** * If 0.0, no exceptions will be thrown. Else this should @@ -410,10 +424,16 @@ if (unSyncedFiles.contains(name)) unSyncedFiles.remove(name); - if (!forced && noDeleteOpenFile) { + if (!forced && (noDeleteOpenFile || assertNoDeleteOpenFile)) { if (openFiles.containsKey(name)) { openFilesDeleted.add(name); - throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true); + IOException ex = fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true); + if (!assertNoDeleteOpenFile) { + throw ex; + } else { + ex.printStackTrace(); + assert false : "MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"; + } } else { openFilesDeleted.remove(name); } @@ -448,8 +468,12 @@ throw new IOException("file \"" + name + "\" was already written to"); } } - if (noDeleteOpenFile && openFiles.containsKey(name)) { - throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite"); + if ((noDeleteOpenFile || assertNoDeleteOpenFile) && openFiles.containsKey(name)) { + if (!assertNoDeleteOpenFile) { + throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite"); + } else { + assert false : "MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite"; + } } if (crashed) { @@ -612,7 +636,7 @@ openFiles = new HashMap(); openFilesDeleted = new HashSet(); } - if (noDeleteOpenFile && openFiles.size() > 0) { + if (openFiles.size() > 0) { // print the first one as its very verbose otherwise Exception cause = null; Iterator stacktraces = openFileHandles.values().iterator(); @@ -620,10 +644,19 @@ cause = stacktraces.next(); // RuntimeException instead of IOException because // super() does not throw IOException currently: - throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause); + if (!assertNoDeleteOpenFile) { + throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause); + } else { + cause.printStackTrace(); + assert false : "MockDirectoryWrapper: cannot close: there are still open files: " + openFiles; + } } - if (noDeleteOpenFile && openLocks.size() > 0) { - throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks); + if ((noDeleteOpenFile || assertNoDeleteOpenFile) && openLocks.size() > 0) { + if (!assertNoDeleteOpenFile) { + throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks); + } else { + assert false : "MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks; + } } isOpen = false;