Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1427595) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -1023,9 +1023,16 @@ w = new IndexWriter(dir, conf); Document doc = new Document(); + Field idField = newStringField("id", "", Field.Store.NO); + doc.add(idField); doc.add(newField("field", "some text contents", storedTextType)); for(int i=0;i<100;i++) { - w.addDocument(doc); + idField.setStringValue(Integer.toString(i)); + if (i%2 == 0) { + w.updateDocument(new Term("id", idField.stringValue()), doc); + } else { + w.addDocument(doc); + } if (i%10 == 0) { w.commit(); } @@ -1046,7 +1053,7 @@ allowInterrupt = true; } } catch (ThreadInterruptedException re) { - if (true || VERBOSE) { + if (VERBOSE) { System.out.println("TEST: got interrupt"); re.printStackTrace(System.out); } @@ -1114,6 +1121,8 @@ final int numInterrupts = atLeast(300); int i = 0; while(i < numInterrupts) { + // TODO: would be nice to also sometimes interrupt the + // CMS merge threads too ... Thread.sleep(10); if (t.allowInterrupt) { i++; Index: lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java =================================================================== --- lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java (revision 1427595) +++ lucene/core/src/java/org/apache/lucene/codecs/lucene40/BitVector.java (working copy) @@ -26,6 +26,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.MutableBits; /** Optimized implementation of a vector of bits. This is more-or-less like @@ -227,6 +228,7 @@ public final void write(Directory d, String name, IOContext context) throws IOException { assert !(d instanceof CompoundFileDirectory); IndexOutput output = d.createOutput(name, context); + boolean success = false; try { output.writeInt(-2); CodecUtil.writeHeader(output, CODEC, VERSION_CURRENT); @@ -237,8 +239,18 @@ writeBits(output); } assert verifyCount(); + success = true; } finally { - output.close(); + if (!success) { + IOUtils.closeWhileHandlingException(output); + // nocommit do we really need to do this here....? + // this is taken care of higher up ...? and if it's + // not ... what about codecs that write del files + // themselves (eg ST)? + d.deleteFile(name); + } else { + IOUtils.close(output); + } } } Index: lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java (revision 1427595) +++ lucene/core/src/java/org/apache/lucene/index/ReadersAndLiveDocs.java (working copy) @@ -182,16 +182,28 @@ // NOTE: removes callers ref public synchronized void dropReaders() throws IOException { - if (reader != null) { - //System.out.println(" pool.drop info=" + info + " rc=" + reader.getRefCount()); - reader.decRef(); - reader = null; + // TODO: can we somehow use IOUtils here...? problem is + // we are calling .decRef not .close)... + try { + if (reader != null) { + //System.out.println(" pool.drop info=" + info + " rc=" + reader.getRefCount()); + try { + reader.decRef(); + } finally { + reader = null; + } + } + } finally { + if (mergeReader != null) { + //System.out.println(" pool.drop info=" + info + " merge rc=" + mergeReader.getRefCount()); + try { + mergeReader.decRef(); + } finally { + mergeReader = null; + } + } } - if (mergeReader != null) { - //System.out.println(" pool.drop info=" + info + " merge rc=" + mergeReader.getRefCount()); - mergeReader.decRef(); - mergeReader = null; - } + decRef(); } @@ -275,10 +287,20 @@ // We can write directly to the actual name (vs to a // .tmp & renaming it) because the file is not live // until segments file is written: - info.info.getCodec().liveDocsFormat().writeLiveDocs((MutableBits)liveDocs, dir, info, pendingDeleteCount, IOContext.DEFAULT); + boolean success = false; + try { + info.info.getCodec().liveDocsFormat().writeLiveDocs((MutableBits)liveDocs, dir, info, pendingDeleteCount, IOContext.DEFAULT); + success = true; + } finally { + if (!success) { + // Advance only the nextWriteDelGen so that a 2nd + // attempt to write will write to a new file + info.advanceNextWriteDelGen(); + } + } // If we hit an exc in the line above (eg disk full) - // then info remains pointing to the previous + // then info's delGen remains pointing to the previous // (successfully written) del docs: info.advanceDelGen(); info.setDelCount(info.getDelCount() + pendingDeleteCount); Index: lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (revision 1427595) +++ lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (working copy) @@ -518,6 +518,8 @@ /** Called when an exception is hit in a background merge * thread */ protected void handleMergeException(Throwable exc) { + System.out.println(Thread.currentThread().getName() + ": merge exc:"); + exc.printStackTrace(System.out); try { // When an exception is hit during merge, IndexWriter // removes any partial files and then allows another Index: lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java (revision 1427595) +++ lucene/core/src/java/org/apache/lucene/index/SegmentInfoPerCommit.java (working copy) @@ -40,6 +40,10 @@ // are no deletes yet): private long delGen; + // Normally 1+delGen, unless an exception was hit on last + // attempt to write: + private long nextWriteDelGen; + private volatile long sizeInBytes = -1; /** Sole constructor. @@ -52,17 +56,27 @@ this.info = info; this.delCount = delCount; this.delGen = delGen; + if (delGen == -1) { + nextWriteDelGen = 1; + } else { + nextWriteDelGen = delGen+1; + } } + /** Called when we succeed in writing deletes */ void advanceDelGen() { - if (delGen == -1) { - delGen = 1; - } else { - delGen++; - } + delGen = nextWriteDelGen; + nextWriteDelGen = delGen+1; sizeInBytes = -1; } + /** Called if there was an exception while writing + * deletes, so that we don't try to write to the same + * file more than once. */ + void advanceNextWriteDelGen() { + nextWriteDelGen++; + } + /** Returns total size in bytes of all files for this * segment. */ public long sizeInBytes() throws IOException { @@ -126,11 +140,7 @@ * of the live docs file. */ public long getNextDelGen() { - if (delGen == -1) { - return 1; - } else { - return delGen + 1; - } + return nextWriteDelGen; } /** @@ -169,6 +179,7 @@ @Override public SegmentInfoPerCommit clone() { + // nocommit do we need to pass down nextWriteDelGen!? return new SegmentInfoPerCommit(info, delCount, delGen); } } Index: lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (revision 1427595) +++ lucene/codecs/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (working copy) @@ -29,8 +29,8 @@ import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.Terms; @@ -40,6 +40,7 @@ import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.IntsRef; import org.apache.lucene.util.OpenBitSet; import org.apache.lucene.util.StringHelper; @@ -67,10 +68,17 @@ final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD; public SimpleTextFieldsReader(SegmentReadState state) throws IOException { + fieldInfos = state.fieldInfos; in = state.dir.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context); - - fieldInfos = state.fieldInfos; - fields = readFields(in.clone()); + boolean success = false; + try { + fields = readFields(in.clone()); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(this); + } + } } private TreeMap readFields(IndexInput in) throws IOException {