Index: lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (revision 1466627) +++ lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (working copy) @@ -67,6 +67,7 @@ // Max actual bytes used. This is set by MockRAMOutputStream: long maxUsedSize; double randomIOExceptionRate; + double randomIOExceptionRateOnOpen; Random randomState; boolean noDeleteOpenFile = true; boolean preventDoubleWrite = true; @@ -322,27 +323,54 @@ public void setRandomIOExceptionRate(double rate) { randomIOExceptionRate = rate; } + public double getRandomIOExceptionRate() { return randomIOExceptionRate; } + /** + * If 0.0, no exceptions will be thrown during openInput + * and createOutput. Else this should + * be a double 0.0 - 1.0 and we will randomly throw an + * IOException in openInput and createOutput with + * this probability. + */ + public void setRandomIOExceptionRateOnOpen(double rate) { + randomIOExceptionRateOnOpen = rate; + } + + public double getRandomIOExceptionRateOnOpen() { + return randomIOExceptionRateOnOpen; + } + void maybeThrowIOException() throws IOException { maybeThrowIOException(null); } void maybeThrowIOException(String message) throws IOException { - if (randomIOExceptionRate > 0.0) { - int number = Math.abs(randomState.nextInt() % 1000); - if (number < randomIOExceptionRate*1000) { - if (LuceneTestCase.VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception" + (message == null ? "" : " (" + message + ")")); - new Throwable().printStackTrace(System.out); - } - throw new IOException("a random IOException" + (message == null ? "" : "(" + message + ")")); + if (randomState.nextDouble() < randomIOExceptionRate) { + if (LuceneTestCase.VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception" + (message == null ? "" : " (" + message + ")")); + new Throwable().printStackTrace(System.out); } + throw new IOException("a random IOException" + (message == null ? "" : "(" + message + ")")); } } + void maybeThrowIOExceptionOnOpen() throws IOException { + if (randomState.nextDouble() < randomIOExceptionRateOnOpen) { + if (LuceneTestCase.VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception during open"); + new Throwable().printStackTrace(System.out); + } + if (randomState.nextBoolean()) { + throw new IOException("a random IOException"); + } else { + throw new FileNotFoundException("a random IOException"); + } + } + } + @Override public synchronized void deleteFile(String name) throws IOException { maybeYield(); @@ -403,22 +431,28 @@ @Override public synchronized IndexOutput createOutput(String name, IOContext context) throws IOException { + maybeThrowDeterministicException(); + maybeThrowIOExceptionOnOpen(); maybeYield(); if (failOnCreateOutput) { maybeThrowDeterministicException(); } - if (crashed) + if (crashed) { throw new IOException("cannot createOutput after crash"); + } init(); synchronized(this) { - if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen")) + if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen")) { throw new IOException("file \"" + name + "\" was already written to"); + } } - if (noDeleteOpenFile && openFiles.containsKey(name)) + if (noDeleteOpenFile && openFiles.containsKey(name)) { throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite"); + } - if (crashed) + if (crashed) { throw new IOException("cannot createOutput after crash"); + } unSyncedFiles.add(name); createdFiles.add(name); @@ -428,9 +462,9 @@ RAMFile existing = ramdir.fileMap.get(name); // Enforce write once: - if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite) + if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite) { throw new IOException("file " + name + " already exists"); - else { + } else { if (existing!=null) { ramdir.sizeInBytes.getAndAdd(-existing.sizeInBytes); existing.directory = null; @@ -484,6 +518,8 @@ @Override public synchronized IndexInput openInput(String name, IOContext context) throws IOException { + maybeThrowDeterministicException(); + maybeThrowIOExceptionOnOpen(); maybeYield(); if (failOnOpenInput) { maybeThrowDeterministicException(); @@ -587,8 +623,11 @@ if (noDeleteOpenFile && openLocks.size() > 0) { throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks); } + isOpen = false; if (getCheckIndexOnClose()) { + randomIOExceptionRate = 0.0; + randomIOExceptionRateOnOpen = 0.0; if (indexPossiblyExists()) { if (LuceneTestCase.VERBOSE) { System.out.println("\nNOTE: MockDirectoryWrapper: now crash"); @@ -793,7 +832,7 @@ } } } - + @Override public synchronized String[] listAll() throws IOException { maybeYield(); Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java (revision 0) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java (working copy) @@ -0,0 +1,158 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.PrintStreamInfoStream; +import org.apache.lucene.util._TestUtil; + +public class TestIndexWriterOutOfFileDescriptors extends LuceneTestCase { + public void test() throws Exception { + MockDirectoryWrapper dir = newMockFSDirectory(_TestUtil.getTempDir("TestIndexWriterOutOfFileDescriptors")); + dir.setPreventDoubleWrite(false); + double rate = random().nextDouble()*0.01; + //System.out.println("rate=" + rate); + dir.setRandomIOExceptionRateOnOpen(rate); + int iters = atLeast(20); + LineFileDocs docs = new LineFileDocs(random()); + IndexReader r = null; + DirectoryReader r2 = null; + boolean any = false; + MockDirectoryWrapper dirCopy = null; + int lastNumDocs = 0; + for(int iter=0;iter files = new HashSet(); + for (String file : dir.listAll()) { + dir.copy(dirCopy, file, file, IOContext.DEFAULT); + files.add(file); + } + dirCopy.sync(files); + // Have IW kiss the dir so we remove any leftover + // files ... we can easily have leftover files at + // the time we take a copy because we are holding + // open a reader: + new IndexWriter(dirCopy, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))).close(); + dirCopy.setRandomIOExceptionRate(rate); + dir.setRandomIOExceptionRateOnOpen(rate); + } + } + + if (r2 != null) { + r2.close(); + } + if (r != null) { + r.close(); + dirCopy.close(); + } + dir.close(); + } +} Property changes on: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (revision 1466627) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (working copy) @@ -40,7 +40,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; Index: lucene/core/src/test/org/apache/lucene/index/TestTransactions.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestTransactions.java (revision 1466627) +++ lucene/core/src/test/org/apache/lucene/index/TestTransactions.java (working copy) @@ -181,13 +181,27 @@ @Override public void doWork() throws Throwable { - IndexReader r1, r2; + IndexReader r1=null, r2=null; synchronized(lock) { - r1 = DirectoryReader.open(dir1); - r2 = DirectoryReader.open(dir2); + try { + r1 = DirectoryReader.open(dir1); + r2 = DirectoryReader.open(dir2); + } catch (IOException e) { + if (!e.getMessage().contains("on purpose")) { + throw e; + } + if (r1 != null) { + r1.close(); + } + if (r2 != null) { + r2.close(); + } + return; + } } - if (r1.numDocs() != r2.numDocs()) + if (r1.numDocs() != r2.numDocs()) { throw new RuntimeException("doc counts differ: r1=" + r1.numDocs() + " r2=" + r2.numDocs()); + } r1.close(); r2.close(); } Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 1466627) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (working copy) @@ -1496,7 +1496,7 @@ if (doFail && name.startsWith("segments_")) { StackTraceElement[] trace = new Exception().getStackTrace(); for (int i = 0; i < trace.length; i++) { - if ("indexExists".equals(trace[i].getMethodName())) { + if ("read".equals(trace[i].getMethodName())) { throw new UnsupportedOperationException("expected UOE"); } } @@ -1516,8 +1516,8 @@ new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, null)); fail("should have gotten a UOE"); } catch (UnsupportedOperationException expected) { - } + uoe.doFail = false; d.close(); } Index: lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (revision 1466627) +++ lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (working copy) @@ -914,19 +914,6 @@ dir.close(); } - // LUCENE-2812 - public void testIndexExists() throws Exception { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); - writer.addDocument(new Document()); - writer.prepareCommit(); - assertFalse(DirectoryReader.indexExists(dir)); - writer.commit(); - writer.close(); - assertTrue(DirectoryReader.indexExists(dir)); - dir.close(); - } - // Make sure totalTermFreq works correctly in the terms // dict cache public void testTotalTermFreqCached() throws Exception { Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1466627) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -52,7 +52,9 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; @@ -2115,5 +2117,52 @@ } } - + + // LUCENE-2727/LUCENE-2812/LUCENE-4738: + public void testCorruptFirstCommit() throws Exception { + for(int i=0;i<6;i++) { + BaseDirectoryWrapper dir = newDirectory(); + dir.createOutput("segments_0", IOContext.DEFAULT).close(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + int mode = i/2; + if (mode == 0) { + iwc.setOpenMode(OpenMode.CREATE); + } else if (mode == 1) { + iwc.setOpenMode(OpenMode.APPEND); + } else if (mode == 2) { + iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); + } + + if (VERBOSE) { + System.out.println("\nTEST: i=" + i); + } + + try { + if ((i & 1) == 0) { + new IndexWriter(dir, iwc).close(); + } else { + new IndexWriter(dir, iwc).rollback(); + } + if (mode != 0) { + fail("expected exception"); + } + } catch (IOException ioe) { + // OpenMode.APPEND should throw an exception since no + // index exists: + if (mode == 0) { + // Unexpected + throw ioe; + } + } + + if (VERBOSE) { + System.out.println(" at close: " + Arrays.toString(dir.listAll())); + } + + if (mode != 0) { + dir.setCheckIndexOnClose(false); + } + dir.close(); + } + } } Index: lucene/core/src/java/org/apache/lucene/store/Directory.java =================================================================== --- lucene/core/src/java/org/apache/lucene/store/Directory.java (revision 1466627) +++ lucene/core/src/java/org/apache/lucene/store/Directory.java (working copy) @@ -184,7 +184,7 @@ * 
    * Directory to; // the directory to copy to
    * for (String file : dir.listAll()) {
-   *   dir.copy(to, file, newFile); // newFile can be either file, or a new name
+   *   dir.copy(to, file, newFile, IOContext.DEFAULT); // newFile can be either file, or a new name
    * }
    *
*

Index: lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 1466627) +++ lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (working copy) @@ -123,7 +123,7 @@ * @throws IOException if there is a low-level IO error */ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, - InfoStream infoStream, IndexWriter writer) throws IOException { + InfoStream infoStream, IndexWriter writer, boolean initialIndexExists) throws IOException { this.infoStream = infoStream; this.writer = writer; @@ -209,7 +209,7 @@ } } - if (currentCommitPoint == null && currentSegmentsFile != null) { + if (currentCommitPoint == null && currentSegmentsFile != null && initialIndexExists) { // We did not in fact see the segments_N file // corresponding to the segmentInfos that was passed // in. Yet, it must exist, because our caller holds @@ -221,7 +221,7 @@ try { sis.read(directory, currentSegmentsFile); } catch (IOException e) { - throw new CorruptIndexException("failed to locate current segments_N file"); + throw new CorruptIndexException("failed to locate current segments_N file \"" + currentSegmentsFile + "\""); } if (infoStream.isEnabled("IFD")) { infoStream.message("IFD", "forced open of current segments file " + segmentInfos.getSegmentsFileName()); Index: lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (revision 1466627) +++ lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (working copy) @@ -23,9 +23,9 @@ import java.util.Collections; import java.util.List; -import org.apache.lucene.index.SegmentInfos.FindSegmentsFile; import org.apache.lucene.search.SearcherManager; // javadocs import org.apache.lucene.store.Directory; +import org.apache.lucene.store.NoSuchDirectoryException; /** DirectoryReader is an implementation of {@link CompositeReader} that can read indexes in a {@link Directory}. @@ -314,34 +314,39 @@ } /** - * Returns true if an index exists at the specified directory. + * Returns true if an index likely exists at + * the specified directory. Note that if a corrupt index + * exists, or if an index in the process of committing * @param directory the directory to check for an index * @return true if an index exists; false otherwise */ - public static boolean indexExists(Directory directory) { + public static boolean indexExists(Directory directory) throws IOException { + // LUCENE-2812, LUCENE-2727, LUCENE-4738: this logic will + // return true in cases that should arguably be false, + // such as only IW.prepareCommit has been called, or a + // corrupt first commit, but it's too deadly to make + // this logic "smarter" and risk accidentally returning + // false due to various cases like file description + // exhaustion, access denited, etc., because in that + // case IndexWriter may delete the entire index. It's + // safer to err towards "index exists" than try to be + // smart about detecting not-yet-fully-committed or + // corrupt indices. This means that IndexWriter will + // throw an exception on such indices and the app must + // resolve the situation manually: + String[] files; try { - new FindSegmentsFile(directory) { - @Override - protected Object doBody(String segmentFileName) throws IOException { - try { - new SegmentInfos().read(directory, segmentFileName); - } catch (FileNotFoundException ex) { - if (!directory.fileExists(segmentFileName)) { - throw ex; - } - /* this is ok - we might have run into a access exception here. - * or even worse like on LUCENE-4870 this is triggered due to - * too many open files on the system. In that case we rather report - * a false positive here since wrongly returning false from indexExist - * can cause data loss since IW relies on this.*/ - } - return null; - } - }.run(); - return true; - } catch (IOException ioe) { + files = directory.listAll(); + } catch (NoSuchDirectoryException nsde) { + // Directory does not exist --> no index exists return false; } + for(String file : files) { + if (file.startsWith(IndexFileNames.SEGMENTS)) { + return true; + } + } + return false; } /** Index: lucene/core/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (revision 1466627) +++ lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -659,6 +659,8 @@ // IndexFormatTooOldException. segmentInfos = new SegmentInfos(); + boolean initialIndexExists = true; + if (create) { // Try to read first. This is to allow create // against an index that's currently open for @@ -669,6 +671,7 @@ segmentInfos.clear(); } catch (IOException e) { // Likely this means it's a fresh directory + initialIndexExists = false; } // Record that we have a change (zero out all @@ -709,7 +712,8 @@ synchronized(this) { deleter = new IndexFileDeleter(directory, config.getIndexDeletionPolicy(), - segmentInfos, infoStream, this); + segmentInfos, infoStream, this, + initialIndexExists); } if (deleter.startingCommitDeleted) {