Index: lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (revision 1466217) +++ lucene/test-framework/src/java/org/apache/lucene/store/MockDirectoryWrapper.java (working copy) @@ -322,6 +322,7 @@ public void setRandomIOExceptionRate(double rate) { randomIOExceptionRate = rate; } + public double getRandomIOExceptionRate() { return randomIOExceptionRate; } @@ -338,7 +339,11 @@ System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception" + (message == null ? "" : " (" + message + ")")); new Throwable().printStackTrace(System.out); } - throw new IOException("a random IOException" + (message == null ? "" : "(" + message + ")")); + if (randomState.nextBoolean()) { + throw new IOException("a random IOException" + (message == null ? "" : "(" + message + ")")); + } else { + throw new FileNotFoundException("a random IOException" + (message == null ? "" : "(" + message + ")")); + } } } } @@ -403,22 +408,27 @@ @Override public synchronized IndexOutput createOutput(String name, IOContext context) throws IOException { + maybeThrowAnyException(); maybeYield(); if (failOnCreateOutput) { maybeThrowDeterministicException(); } - if (crashed) + if (crashed) { throw new IOException("cannot createOutput after crash"); + } init(); synchronized(this) { - if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen")) + if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen")) { throw new IOException("file \"" + name + "\" was already written to"); + } } - if (noDeleteOpenFile && openFiles.containsKey(name)) + if (noDeleteOpenFile && openFiles.containsKey(name)) { throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite"); + } - if (crashed) + if (crashed) { throw new IOException("cannot createOutput after crash"); + } unSyncedFiles.add(name); createdFiles.add(name); @@ -428,9 +438,9 @@ RAMFile existing = ramdir.fileMap.get(name); // Enforce write once: - if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite) + if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite) { throw new IOException("file " + name + " already exists"); - else { + } else { if (existing!=null) { ramdir.sizeInBytes.getAndAdd(-existing.sizeInBytes); existing.directory = null; @@ -484,6 +494,7 @@ @Override public synchronized IndexInput openInput(String name, IOContext context) throws IOException { + maybeThrowAnyException(); maybeYield(); if (failOnOpenInput) { maybeThrowDeterministicException(); @@ -589,6 +600,7 @@ } isOpen = false; if (getCheckIndexOnClose()) { + randomIOExceptionRate = 0.0; if (indexPossiblyExists()) { if (LuceneTestCase.VERBOSE) { System.out.println("\nNOTE: MockDirectoryWrapper: now crash"); @@ -793,6 +805,16 @@ } } } + + /** + * might throw a deterministic exception, if any have been registered + * with failOn, and might throw an IOException, if random IO Exceptions + * are turned on with setRandomIOExceptionRate. + */ + synchronized void maybeThrowAnyException() throws IOException { + maybeThrowDeterministicException(); + maybeThrowIOException(); + } @Override public synchronized String[] listAll() throws IOException { Index: lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (revision 1466217) +++ lucene/test-framework/src/java/org/apache/lucene/util/_TestUtil.java (working copy) @@ -54,12 +54,12 @@ import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.index.AtomicReader; import org.apache.lucene.index.AtomicReaderContext; -import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.CheckIndex.Status.DocValuesStatus; import org.apache.lucene.index.CheckIndex.Status.FieldNormStatus; import org.apache.lucene.index.CheckIndex.Status.StoredFieldStatus; import org.apache.lucene.index.CheckIndex.Status.TermIndexStatus; import org.apache.lucene.index.CheckIndex.Status.TermVectorStatus; +import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.ConcurrentMergeScheduler; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; @@ -85,6 +85,7 @@ import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.MockDirectoryWrapper; import org.junit.Assert; import com.carrotsearch.randomizedtesting.RandomizedContext; import com.carrotsearch.randomizedtesting.generators.RandomInts; @@ -210,11 +211,25 @@ } public static CheckIndex.Status checkIndex(Directory dir, boolean crossCheckTermVectors) throws IOException { + // hack alert: turn off random IOExceptions temporarily + double randomExceptionRateSave = 0.0D; + if (dir instanceof MockDirectoryWrapper) { + MockDirectoryWrapper m = (MockDirectoryWrapper) dir; + randomExceptionRateSave = m.getRandomIOExceptionRate(); + m.setRandomIOExceptionRate(0.0D); + } + ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); CheckIndex checker = new CheckIndex(dir); checker.setCrossCheckTermVectors(crossCheckTermVectors); checker.setInfoStream(new PrintStream(bos, false, "UTF-8"), false); CheckIndex.Status indexStatus = checker.checkIndex(null); + + // restore the random IOException rate + if (randomExceptionRateSave > 0.0D) { + ((MockDirectoryWrapper)dir).setRandomIOExceptionRate(randomExceptionRateSave); + } + if (indexStatus == null || indexStatus.clean == false) { System.out.println("CheckIndex failed"); System.out.println(bos.toString("UTF-8")); Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java (revision 0) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java (working copy) @@ -0,0 +1,158 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.PrintStreamInfoStream; +import org.apache.lucene.util._TestUtil; + +public class TestIndexWriterOutOfFileDescriptors extends LuceneTestCase { + public void test() throws Exception { + MockDirectoryWrapper dir = newMockFSDirectory(_TestUtil.getTempDir("TestIndexWriterOutOfFileDescriptors")); + dir.setPreventDoubleWrite(false); + double rate = random().nextDouble()*0.01; + //System.out.println("rate=" + rate); + dir.setRandomIOExceptionRate(rate); + int iters = atLeast(20); + LineFileDocs docs = new LineFileDocs(random()); + IndexReader r = null; + DirectoryReader r2 = null; + boolean any = false; + MockDirectoryWrapper dirCopy = null; + int lastNumDocs = 0; + for(int iter=0;iter files = new HashSet(); + for (String file : dir.listAll()) { + dir.copy(dirCopy, file, file, IOContext.DEFAULT); + files.add(file); + } + dirCopy.sync(files); + // Have IW kiss the dir so we remove any leftover + // files ... we can easily have leftover files at + // the time we take a copy because we are holding + // open a reader: + new IndexWriter(dirCopy, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))).close(); + dirCopy.setRandomIOExceptionRate(rate); + dir.setRandomIOExceptionRate(rate); + } + } + + if (r2 != null) { + r2.close(); + } + if (r != null) { + r.close(); + dirCopy.close(); + } + dir.close(); + } +} Property changes on: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOutOfFileDescriptors.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1466217) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -52,7 +52,9 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.BaseDirectoryWrapper; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; @@ -2115,5 +2117,52 @@ } } - + + // LUCENE-2727/LUCENE-2812/LUCENE-4738: + public void testCorruptFirstCommit() throws Exception { + for(int i=0;i<6;i++) { + BaseDirectoryWrapper dir = newDirectory(); + dir.createOutput("segments_0", IOContext.DEFAULT).close(); + IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())); + int mode = i/2; + if (mode == 0) { + iwc.setOpenMode(OpenMode.CREATE); + } else if (mode == 1) { + iwc.setOpenMode(OpenMode.APPEND); + } else if (mode == 2) { + iwc.setOpenMode(OpenMode.CREATE_OR_APPEND); + } + + if (VERBOSE) { + System.out.println("\nTEST: i=" + i); + } + + try { + if ((i & 1) == 0) { + new IndexWriter(dir, iwc).close(); + } else { + new IndexWriter(dir, iwc).rollback(); + } + if (mode != 0) { + fail("expected exception"); + } + } catch (IOException ioe) { + // OpenMode.APPEND should throw an exception since no + // index exists: + if (mode == 0) { + // Unexpected + throw ioe; + } + } + + if (VERBOSE) { + System.out.println(" at close: " + Arrays.toString(dir.listAll())); + } + + if (mode != 0) { + dir.setCheckIndexOnClose(false); + } + dir.close(); + } + } } Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (revision 1466217) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (working copy) @@ -358,7 +358,10 @@ if (VERBOSE) { System.out.println(" now test readers"); } - + + // Turn off random exceptions for verify: + dir.setRandomIOExceptionRate(0.0); + // Finally, verify index is not corrupt, and, if // we succeeded, we see all docs added, and if we // failed, we see either all docs or no docs added Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 1466217) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (working copy) @@ -1496,7 +1496,7 @@ if (doFail && name.startsWith("segments_")) { StackTraceElement[] trace = new Exception().getStackTrace(); for (int i = 0; i < trace.length; i++) { - if ("indexExists".equals(trace[i].getMethodName())) { + if ("read".equals(trace[i].getMethodName())) { throw new UnsupportedOperationException("expected UOE"); } } @@ -1516,8 +1516,8 @@ new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, null)); fail("should have gotten a UOE"); } catch (UnsupportedOperationException expected) { - } + uoe.doFail = false; d.close(); } Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (revision 1466217) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (working copy) @@ -40,7 +40,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.Bits; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -568,6 +567,9 @@ dir.setRandomIOExceptionRate(randomIOExceptionRate); dir.setMaxSizeInBytes(maxSizeInBytes); + // turn off random IOExceptions for the verification step + dir.setRandomIOExceptionRate(0.0D); + // Finally, verify index is not corrupt, and, if // we succeeded, we see all docs changed, and if // we failed, we see either all docs or no docs Index: lucene/core/src/test/org/apache/lucene/index/TestTransactions.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestTransactions.java (revision 1466217) +++ lucene/core/src/test/org/apache/lucene/index/TestTransactions.java (working copy) @@ -181,13 +181,27 @@ @Override public void doWork() throws Throwable { - IndexReader r1, r2; + IndexReader r1=null, r2=null; synchronized(lock) { - r1 = DirectoryReader.open(dir1); - r2 = DirectoryReader.open(dir2); + try { + r1 = DirectoryReader.open(dir1); + r2 = DirectoryReader.open(dir2); + } catch (IOException e) { + if (!e.getMessage().contains("on purpose")) { + throw e; + } + if (r1 != null) { + r1.close(); + } + if (r2 != null) { + r2.close(); + } + return; + } } - if (r1.numDocs() != r2.numDocs()) + if (r1.numDocs() != r2.numDocs()) { throw new RuntimeException("doc counts differ: r1=" + r1.numDocs() + " r2=" + r2.numDocs()); + } r1.close(); r2.close(); } Index: lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (revision 1466217) +++ lucene/core/src/test/org/apache/lucene/index/TestDirectoryReader.java (working copy) @@ -914,19 +914,6 @@ dir.close(); } - // LUCENE-2812 - public void testIndexExists() throws Exception { - Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))); - writer.addDocument(new Document()); - writer.prepareCommit(); - assertFalse(DirectoryReader.indexExists(dir)); - writer.commit(); - writer.close(); - assertTrue(DirectoryReader.indexExists(dir)); - dir.close(); - } - // Make sure totalTermFreq works correctly in the terms // dict cache public void testTotalTermFreqCached() throws Exception { Index: lucene/core/src/java/org/apache/lucene/store/Directory.java =================================================================== --- lucene/core/src/java/org/apache/lucene/store/Directory.java (revision 1466217) +++ lucene/core/src/java/org/apache/lucene/store/Directory.java (working copy) @@ -184,7 +184,7 @@ *
    * Directory to; // the directory to copy to
    * for (String file : dir.listAll()) {
-   *   dir.copy(to, file, newFile); // newFile can be either file, or a new name
+   *   dir.copy(to, file, newFile, IOContext.DEFAULT); // newFile can be either file, or a new name
    * }
    * 
*

Index: lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (revision 1466217) +++ lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java (working copy) @@ -23,7 +23,6 @@ import java.util.Collections; import java.util.List; -import org.apache.lucene.index.SegmentInfos.FindSegmentsFile; import org.apache.lucene.search.SearcherManager; // javadocs import org.apache.lucene.store.Directory; @@ -318,30 +317,27 @@ * @param directory the directory to check for an index * @return true if an index exists; false otherwise */ - public static boolean indexExists(Directory directory) { - try { - new FindSegmentsFile(directory) { - @Override - protected Object doBody(String segmentFileName) throws IOException { - try { - new SegmentInfos().read(directory, segmentFileName); - } catch (FileNotFoundException ex) { - if (!directory.fileExists(segmentFileName)) { - throw ex; - } - /* this is ok - we might have run into a access exception here. - * or even worse like on LUCENE-4870 this is triggered due to - * too many open files on the system. In that case we rather report - * a false positive here since wrongly returning false from indexExist - * can cause data loss since IW relies on this.*/ - } - return null; - } - }.run(); - return true; - } catch (IOException ioe) { - return false; + public static boolean indexExists(Directory directory) throws IOException { + // LUCENE-2812, LUCENE-2727, LUCENE-4738: this logic will + // return true in cases that should arguably be false, + // such as only IW.prepareCommit has been called, or a + // corrupt first commit, but it's too deadly to make + // this logic "smarter" and risk accidentally returning + // false due to various cases like file description + // exhaustion, access denited, etc., because in that + // case IndexWriter may delete the entire index. It's + // safer to err towards "index exists" than try to be + // smart about detecting not-yet-fully-committed or + // corrupt indices. This means that IndexWriter will + // throw an exception on such indices and the app must + // resolve the situation manually: + for(String file : directory.listAll()) { + if (file.startsWith(IndexFileNames.SEGMENTS)) { + return true; + } } + + return false; } /** Index: lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 1466217) +++ lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (working copy) @@ -123,7 +123,7 @@ * @throws IOException if there is a low-level IO error */ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, - InfoStream infoStream, IndexWriter writer) throws IOException { + InfoStream infoStream, IndexWriter writer, boolean loadFirstCommit) throws IOException { this.infoStream = infoStream; this.writer = writer; @@ -209,7 +209,7 @@ } } - if (currentCommitPoint == null && currentSegmentsFile != null) { + if (currentCommitPoint == null && currentSegmentsFile != null && loadFirstCommit) { // We did not in fact see the segments_N file // corresponding to the segmentInfos that was passed // in. Yet, it must exist, because our caller holds @@ -221,7 +221,7 @@ try { sis.read(directory, currentSegmentsFile); } catch (IOException e) { - throw new CorruptIndexException("failed to locate current segments_N file"); + throw new CorruptIndexException("failed to locate current segments_N file \"" + currentSegmentsFile + "\""); } if (infoStream.isEnabled("IFD")) { infoStream.message("IFD", "forced open of current segments file " + segmentInfos.getSegmentsFileName()); Index: lucene/core/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (revision 1466217) +++ lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -659,6 +659,8 @@ // IndexFormatTooOldException. segmentInfos = new SegmentInfos(); + boolean loadFirstCommit = true; + if (create) { // Try to read first. This is to allow create // against an index that's currently open for @@ -669,6 +671,7 @@ segmentInfos.clear(); } catch (IOException e) { // Likely this means it's a fresh directory + loadFirstCommit = false; } // Record that we have a change (zero out all @@ -709,7 +712,8 @@ synchronized(this) { deleter = new IndexFileDeleter(directory, config.getIndexDeletionPolicy(), - segmentInfos, infoStream, this); + segmentInfos, infoStream, this, + loadFirstCommit); } if (deleter.startingCommitDeleted) {