Index: lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java --- lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java Sat May 28 08:38:45 2011 -0400 +++ lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java Sat May 28 12:36:07 2011 -0400 @@ -269,7 +269,7 @@ in = cache.openInput(fileName); in.copyBytes(out, in.length()); } finally { - IOUtils.closeSafely(in, out); + IOUtils.closeSafely(false, in, out); } synchronized(this) { cache.deleteFile(fileName); Index: lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java --- lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java Sat May 28 12:36:07 2011 -0400 @@ -219,7 +219,7 @@ } catch (IOException e) { priorException = e; } finally { - IOUtils.closeSafely(priorException, os); + IOUtils.closeSafelyPriorEx(priorException, os); } } Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java --- lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java Sat May 28 12:36:07 2011 -0400 @@ -84,19 +84,44 @@ @Override public void abort() { - for(int i=0;i extensions) { Index: lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java --- lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java Sat May 28 12:36:07 2011 -0400 @@ -143,8 +143,11 @@ @Override public void close() throws IOException { - write(END); - newline(); - out.close(); + try { + write(END); + newline(); + } finally { + out.close(); + } } } Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java --- lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java Sat May 28 12:36:07 2011 -0400 @@ -33,6 +33,7 @@ import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.util.IOUtils; /** @lucene.experimental */ public final class StandardPostingsWriter extends PostingsWriterBase { @@ -42,8 +43,8 @@ final static int VERSION_START = 0; final static int VERSION_CURRENT = VERSION_START; - final IndexOutput freqOut; - final IndexOutput proxOut; + IndexOutput freqOut; + IndexOutput proxOut; final DefaultSkipListWriter skipListWriter; /** Expert: The fraction of TermDocs entries stored in skip tables, * used to accelerate {@link DocsEnum#advance(int)}. Larger values result in @@ -85,31 +86,35 @@ public StandardPostingsWriter(SegmentWriteState state) throws IOException { this(state, DEFAULT_SKIP_INTERVAL); } + public StandardPostingsWriter(SegmentWriteState state, int skipInterval) throws IOException { - super(); this.skipInterval = skipInterval; this.skipMinimum = skipInterval; /* set to the same for now */ //this.segment = state.segmentName; String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.FREQ_EXTENSION); freqOut = state.directory.createOutput(fileName); - - if (state.fieldInfos.hasProx()) { - // At least one field does not omit TF, so create the - // prox file - fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION); - proxOut = state.directory.createOutput(fileName); - } else { - // Every field omits TF so we will write no prox file - proxOut = null; + boolean success = false; + try { + if (state.fieldInfos.hasProx()) { + // At least one field does not omit TF, so create the + // prox file + fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION); + proxOut = state.directory.createOutput(fileName); + } else { + // Every field omits TF so we will write no prox file + proxOut = null; + } + + totalNumDocs = state.numDocs; + + skipListWriter = new DefaultSkipListWriter(skipInterval, maxSkipLevels, + state.numDocs, freqOut, proxOut); + success = true; + } finally { + if (!success) { + IOUtils.closeSafely(true, freqOut, proxOut); + } } - - totalNumDocs = state.numDocs; - - skipListWriter = new DefaultSkipListWriter(skipInterval, - maxSkipLevels, - state.numDocs, - freqOut, - proxOut); } @Override @@ -267,12 +272,6 @@ @Override public void close() throws IOException { - try { - freqOut.close(); - } finally { - if (proxOut != null) { - proxOut.close(); - } - } + IOUtils.closeSafely(false, freqOut, proxOut); } } Index: lucene/src/java/org/apache/lucene/store/Directory.java --- lucene/src/java/org/apache/lucene/store/Directory.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/java/org/apache/lucene/store/Directory.java Sat May 28 12:36:07 2011 -0400 @@ -208,7 +208,7 @@ } catch (IOException ioe) { priorException = ioe; } finally { - IOUtils.closeSafely(priorException, os, is); + IOUtils.closeSafelyPriorEx(priorException, os, is); } } Index: lucene/src/java/org/apache/lucene/util/IOUtils.java --- lucene/src/java/org/apache/lucene/util/IOUtils.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/java/org/apache/lucene/util/IOUtils.java Sat May 28 12:36:07 2011 -0400 @@ -19,6 +19,7 @@ import java.io.Closeable; import java.io.IOException; +import java.util.Iterator; /** @lucene.internal */ public final class IOUtils { @@ -46,45 +47,114 @@ * @param priorException null or an exception that will be rethrown after method completion * @param objects objects to call close() on */ - public static void closeSafely(E priorException, Closeable... objects) throws E, IOException { - IOException firstIOE = null; + public static void closeSafelyPriorEx(E priorException, Closeable... objects) throws E, IOException { + Throwable th = null; for (Closeable object : objects) { try { - if (object != null) + if (object != null) { object.close(); - } catch (IOException ioe) { - if (firstIOE == null) - firstIOE = ioe; + } + } catch (Throwable t) { + if (th == null) { + th = t; + } } } - if (priorException != null) + if (priorException != null) { throw priorException; - else if (firstIOE != null) - throw firstIOE; + } else if (th != null) { + if (th instanceof IOException) throw (IOException) th; + if (th instanceof RuntimeException) throw (RuntimeException) th; + if (th instanceof Error) throw (Error) th; + throw new RuntimeException(th); + } + } + + /** @see #closeSafelyPriorEx(Exception, Closeable...) */ + public static void closeSafelyPriorExIterable(E priorException, Iterable objects) throws E, IOException { + Throwable th = null; + + for (Closeable object : objects) { + try { + if (object != null) { + object.close(); + } + } catch (Throwable t) { + if (th == null) { + th = t; + } + } + } + + if (priorException != null) { + throw priorException; + } else if (th != null) { + if (th instanceof IOException) throw (IOException) th; + if (th instanceof RuntimeException) throw (RuntimeException) th; + if (th instanceof Error) throw (Error) th; + throw new RuntimeException(th); + } } /** - *

Closes all given Closeables, suppressing all thrown exceptions. Some of the Closeables - * may be null, they are ignored. After everything is closed, method either throws the first of suppressed exceptions, - * or completes normally.

- * @param objects objects to call close() on + * Closes all given Closeables, suppressing all thrown exceptions. + * Some of the Closeables may be null, they are ignored. After + * everything is closed, and if {@code suppressExceptions} is {@code false}, + * method either throws the first of suppressed exceptions, or completes + * normally. + * + * @param suppressExceptions + * if true then exceptions that occur during close() are suppressed + * @param objects + * objects to call close() on */ - public static void closeSafely(Closeable... objects) throws IOException { - IOException firstIOE = null; + public static void closeSafely(boolean suppressExceptions, Closeable... objects) throws IOException { + Throwable th = null; for (Closeable object : objects) { try { - if (object != null) + if (object != null) { object.close(); - } catch (IOException ioe) { - if (firstIOE == null) - firstIOE = ioe; + } + } catch (Throwable t) { + if (th == null) + th = t; } } - if (firstIOE != null) - throw firstIOE; + if (th != null && !suppressExceptions) { + if (th instanceof IOException) throw (IOException) th; + if (th instanceof RuntimeException) throw (RuntimeException) th; + if (th instanceof Error) throw (Error) th; + throw new RuntimeException(th); + } } + + /** + * @see #closeSafely(boolean, Closeable...) + */ + public static void closeSafelyIterable(boolean suppressExceptions, Iterable objects) throws IOException { + Throwable th = null; + + for (Closeable object : objects) { + try { + if (object != null) { + object.close(); + } + } catch (Throwable t) { + if (th == null) + th = t; + } + } + + if (th != null && !suppressExceptions) { + if (th instanceof IOException) throw (IOException) th; + if (th instanceof RuntimeException) throw (RuntimeException) th; + if (th instanceof Error) throw (Error) th; + throw new RuntimeException(th); + } + } + } Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java --- lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java Sat May 28 12:36:07 2011 -0400 @@ -44,6 +44,7 @@ import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.store.*; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; /** * A silly test codec to verify core support for fixed @@ -97,15 +98,25 @@ @Override public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { - return new FixedIntBlockIndexOutput(dir.createOutput(fileName), blockSize) { - @Override - protected void flushBlock() throws IOException { - for(int i=0;i= 0; - out.writeVInt(buffer[i]); + IndexOutput out = dir.createOutput(fileName); + boolean success = false; + try { + FixedIntBlockIndexOutput ret = new FixedIntBlockIndexOutput(out, blockSize) { + @Override + protected void flushBlock() throws IOException { + for(int i=0;i= 0; + out.writeVInt(buffer[i]); + } } + }; + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeSafely(true, out); } - }; + } } } Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java --- lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java Sat May 28 12:36:07 2011 -0400 @@ -46,6 +46,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; /** * A silly test codec to verify core support for variable @@ -102,34 +103,42 @@ @Override public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { final IndexOutput out = dir.createOutput(fileName); - out.writeInt(baseBlockSize); - return new VariableIntBlockIndexOutput(out, 2*baseBlockSize) { - - int pendingCount; - final int[] buffer = new int[2+2*baseBlockSize]; - - @Override - protected int add(int value) throws IOException { - assert value >= 0; - buffer[pendingCount++] = value; - // silly variable block length int encoder: if - // first value <= 3, we write N vints at once; - // else, 2*N - final int flushAt = buffer[0] <= 3 ? baseBlockSize : 2*baseBlockSize; - - // intentionally be non-causal here: - if (pendingCount == flushAt+1) { - for(int i=0;i= 0; + buffer[pendingCount++] = value; + // silly variable block length int encoder: if + // first value <= 3, we write N vints at once; + // else, 2*N + final int flushAt = buffer[0] <= 3 ? baseBlockSize : 2*baseBlockSize; + + // intentionally be non-causal here: + if (pendingCount == flushAt+1) { + for(int i=0;i unSyncedFiles; private Set createdFiles; - Set openFilesForWrite = new HashSet(); + private Set openFilesForWrite = new HashSet(); volatile boolean crashed; private ThrottledIndexOutput throttledOutput; private Throttling throttling = Throttling.SOMETIMES; // use this for tracking files for crash. // additionally: provides debugging information in case you leave one open - Map openFileHandles = Collections.synchronizedMap(new IdentityHashMap()); + private Map openFileHandles = Collections.synchronizedMap(new IdentityHashMap()); // NOTE: we cannot initialize the Map here due to the // order in which our constructor actually does this // member initialization vs when it calls super. It seems // like super is called, then our members are initialized: - Map openFiles; + private Map openFiles; // Only tracked if noDeleteOpenFile is true: if an attempt // is made to delete an open file, we enroll it here. - Set openFilesDeleted; + private Set openFilesDeleted; private synchronized void init() { if (openFiles == null) { @@ -127,7 +127,7 @@ SOMETIMES, /** never throttle output */ NEVER - }; + } public void setThrottling(Throttling throttling) { this.throttling = throttling; @@ -362,9 +362,10 @@ ramdir.fileMap.put(name, file); } } + //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name); IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name), name); - openFileHandles.put(io, new RuntimeException("unclosed IndexOutput")); + addFileHandle(io, name, false); openFilesForWrite.add(name); // throttling REALLY slows down tests, so don't do it very often for SOMETIMES. @@ -379,6 +380,18 @@ } } + private void addFileHandle(Closeable c, String name, boolean input) { + Integer v = openFiles.get(name); + if (v != null) { + v = Integer.valueOf(v.intValue()+1); + openFiles.put(name, v); + } else { + openFiles.put(name, Integer.valueOf(1)); + } + + openFileHandles.put(c, new RuntimeException("unclosed Index" + (input ? "Input" : "Output") + ": " + name)); + } + @Override public synchronized IndexInput openInput(String name) throws IOException { maybeYield(); @@ -391,16 +404,8 @@ throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); } - if (openFiles.containsKey(name)) { - Integer v = openFiles.get(name); - v = Integer.valueOf(v.intValue()+1); - openFiles.put(name, v); - } else { - openFiles.put(name, Integer.valueOf(1)); - } - IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name)); - openFileHandles.put(ii, new RuntimeException("unclosed IndexInput")); + addFileHandle(ii, name, true); return ii; } @@ -465,6 +470,31 @@ delegate.close(); } + private synchronized void removeOpenFile(Closeable c, String name) { + Integer v = openFiles.get(name); + // Could be null when crash() was called + if (v != null) { + if (v.intValue() == 1) { + openFiles.remove(name); + openFilesDeleted.remove(name); + } else { + v = Integer.valueOf(v.intValue()-1); + openFiles.put(name, v); + } + } + + openFileHandles.remove(c); + } + + public synchronized void removeIndexOutput(IndexOutput out, String name) { + openFilesForWrite.remove(name); + removeOpenFile(out, name); + } + + public synchronized void removeIndexInput(IndexInput in, String name) { + removeOpenFile(in, name); + } + private CodecProvider codecProvider; // We pass this CodecProvider to checkIndex when dir is closed... Index: lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java --- lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java Sat May 28 12:36:07 2011 -0400 @@ -31,8 +31,7 @@ private IndexInput delegate; private boolean isClone; - /** Construct an empty output buffer. - * @throws IOException */ + /** Construct an empty output buffer. */ public MockIndexInputWrapper(MockDirectoryWrapper dir, String name, IndexInput delegate) { this.name = name; this.dir = dir; @@ -46,20 +45,7 @@ // remove the conditional check so we also track that // all clones get closed: if (!isClone) { - synchronized(dir) { - Integer v = dir.openFiles.get(name); - // Could be null when MockRAMDirectory.crash() was called - if (v != null) { - if (v.intValue() == 1) { - dir.openFiles.remove(name); - dir.openFilesDeleted.remove(name); - } else { - v = Integer.valueOf(v.intValue()-1); - dir.openFiles.put(name, v); - } - } - dir.openFileHandles.remove(this); - } + dir.removeIndexInput(this, name); } } Index: lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java --- lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java Sat May 28 12:36:07 2011 -0400 @@ -57,10 +57,7 @@ dir.maxUsedSize = size; } } - synchronized(dir) { - dir.openFileHandles.remove(this); - dir.openFilesForWrite.remove(name); - } + dir.removeIndexOutput(this, name); } } Index: lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java --- lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java Sat May 28 12:36:07 2011 -0400 @@ -73,9 +73,11 @@ @Override public void close() throws IOException { + try { sleep(closeDelayMillis + getDelay(true)); + } finally { delegate.close(); - + } } @Override Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Sat May 28 12:36:07 2011 -0400 @@ -2123,6 +2123,10 @@ allowInterrupt = true; } } catch (ThreadInterruptedException re) { + if (VERBOSE) { + System.out.println("TEST: got interrupt"); + re.printStackTrace(System.out); + } Throwable e = re.getCause(); assertTrue(e instanceof InterruptedException); if (finish) { Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java --- lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java Sat May 28 12:36:07 2011 -0400 @@ -833,6 +833,10 @@ try { modifier.addDocument(doc); } catch (IOException io) { + if (VERBOSE) { + System.out.println("TEST: got expected exc:"); + io.printStackTrace(System.out); + } break; } } Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java --- lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Sat May 28 08:38:45 2011 -0400 +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Sat May 28 12:36:07 2011 -0400 @@ -919,7 +919,7 @@ assertTrue(failure.failOnCommit && failure.failOnDeleteFile); w.rollback(); assertFalse(dir.fileExists("1.fnx")); - // FIXME: on windows, this often fails! assertEquals(0, dir.listAll().length); + assertEquals(0, dir.listAll().length); dir.close(); } } Index: modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTLookup.java --- modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTLookup.java Sat May 28 08:38:45 2011 -0400 +++ modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTLookup.java Sat May 28 12:36:07 2011 -0400 @@ -510,7 +510,7 @@ this.automaton = new FST(new InputStreamDataInput(is), NoOutputs.getSingleton()); cacheRootArcs(); } finally { - IOUtils.closeSafely(is); + IOUtils.closeSafely(false, is); } return true; } @@ -532,7 +532,7 @@ try { this.automaton.save(new OutputStreamDataOutput(os)); } finally { - IOUtils.closeSafely(os); + IOUtils.closeSafely(false, os); } return true;