Index: src/test/org/apache/lucene/store/MockRAMInputStream.java
===================================================================
--- src/test/org/apache/lucene/store/MockRAMInputStream.java	(revision 603331)
+++ src/test/org/apache/lucene/store/MockRAMInputStream.java	(working copy)
@@ -45,11 +45,14 @@
     if (!isClone) {
       synchronized(dir.openFiles) {
         Integer v = (Integer) dir.openFiles.get(name);
-        if (v.intValue() == 1) {
-          dir.openFiles.remove(name);
-        } else {
-          v = new Integer(v.intValue()-1);
-          dir.openFiles.put(name, v);
+        // Could be null when MockRAMDirectory.crash() was called
+        if (v != null) {
+          if (v.intValue() == 1) {
+            dir.openFiles.remove(name);
+          } else {
+            v = new Integer(v.intValue()-1);
+            dir.openFiles.put(name, v);
+          }
         }
       }
     }
Index: src/test/org/apache/lucene/store/MockRAMOutputStream.java
===================================================================
--- src/test/org/apache/lucene/store/MockRAMOutputStream.java	(revision 603331)
+++ src/test/org/apache/lucene/store/MockRAMOutputStream.java	(working copy)
@@ -30,7 +30,7 @@
 public class MockRAMOutputStream extends RAMOutputStream {
   private MockRAMDirectory dir;
   private boolean first=true;
-  
+
   byte[] singleByte = new byte[1];
 
   /** Construct an empty output buffer. */
@@ -59,6 +59,11 @@
     long freeSpace = dir.maxSize - dir.sizeInBytes();
     long realUsage = 0;
 
+    // If MockRAMDir crashed since we were opened, then
+    // don't write anything:
+    if (dir.crashed)
+      throw new IOException("MockRAMDirectory was crashed");
+
     // Enforce disk full:
     if (dir.maxSize != 0 && freeSpace <= len) {
       // Compute the real disk free.  This will greatly slow
Index: src/test/org/apache/lucene/store/MockRAMDirectory.java
===================================================================
--- src/test/org/apache/lucene/store/MockRAMDirectory.java	(revision 603331)
+++ src/test/org/apache/lucene/store/MockRAMDirectory.java	(working copy)
@@ -24,7 +24,10 @@
 import java.util.Random;
 import java.util.Map;
 import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.ArrayList;
+import java.util.Arrays;
 
 /**
  * This is a subclass of RAMDirectory that adds methods
@@ -40,6 +43,10 @@
   double randomIOExceptionRate;
   Random randomState;
   boolean noDeleteOpenFile = true;
+  boolean preventDoubleWrite = true;
+  private Set unSyncedFiles;
+  private Set createdFiles;
+  volatile boolean crashed;
 
   // NOTE: we cannot initialize the Map here due to the
   // order in which our constructor actually does this
@@ -47,31 +54,79 @@
   // like super is called, then our members are initialized:
   Map openFiles;
 
+  private void init() {
+    if (openFiles == null)
+      openFiles = new HashMap();
+    if (createdFiles == null)
+      createdFiles = new HashSet();
+    if (unSyncedFiles == null)
+      unSyncedFiles = new HashSet();
+  }
+
   public MockRAMDirectory() {
     super();
-    if (openFiles == null) {
-      openFiles = new HashMap();
-    }
+    init();
   }
   public MockRAMDirectory(String dir) throws IOException {
     super(dir);
-    if (openFiles == null) {
-      openFiles = new HashMap();
-    }
+    init();
   }
   public MockRAMDirectory(Directory dir) throws IOException {
     super(dir);
-    if (openFiles == null) {
-      openFiles = new HashMap();
-    }
+    init();
   }
   public MockRAMDirectory(File dir) throws IOException {
     super(dir);
-    if (openFiles == null) {
+    init();
+  }
+
+  /** If set to true, we throw an IOException if the same
+   *  file is opened by createOutput, ever. */
+  public void setPreventDoubleWrite(boolean value) {
+    preventDoubleWrite = value;
+  }
+
+  public synchronized void sync(String name) throws IOException {
+    if (crashed)
+      throw new IOException("cannot sync after crash");
+    if (unSyncedFiles.contains(name))
+      unSyncedFiles.remove(name);
+  }
+
+  /** Simulates a crash of OS or machine by overwriting
+   *  unsycned files. */
+  public void crash() throws IOException {
+    synchronized(this) {
+      crashed = true;
       openFiles = new HashMap();
     }
+    Iterator it = unSyncedFiles.iterator();
+    unSyncedFiles = new HashSet();
+    int count = 0;
+    while(it.hasNext()) {
+      String name = (String) it.next();
+      RAMFile file = (RAMFile) fileMap.get(name);
+      if (count % 3 == 0) {
+        deleteFile(name, true);
+      } else if (count % 3 == 1) {
+        // Zero out file entirely
+        final int numBuffers = file.numBuffers();
+        for(int i=0;i<numBuffers;i++) {
+          byte[] buffer = file.getBuffer(i);
+          Arrays.fill(buffer, (byte) 0);
+        }
+      } else if (count % 3 == 2) {
+        // Truncate the file:
+        file.setLength(file.getLength()/2);
+      }
+      count++;
+    }
   }
 
+  public synchronized void clearCrash() throws IOException {
+    crashed = false;
+  }
+
   public void setMaxSizeInBytes(long maxSize) {
     this.maxSize = maxSize;
   }
@@ -127,24 +182,41 @@
   }
 
   public synchronized void deleteFile(String name) throws IOException {
-    synchronized(openFiles) {
-      if (noDeleteOpenFile && openFiles.containsKey(name)) {
-        throw new IOException("MockRAMDirectory: file \"" + name + "\" is still open: cannot delete");
+    deleteFile(name, false);
+  }
+
+  private synchronized void deleteFile(String name, boolean forced) throws IOException {
+    if (crashed && !forced)
+      throw new IOException("cannot delete after crash");
+
+    if (unSyncedFiles.contains(name))
+      unSyncedFiles.remove(name);
+    if (!forced) {
+      synchronized(openFiles) {
+        if (noDeleteOpenFile && openFiles.containsKey(name)) {
+          throw new IOException("MockRAMDirectory: file \"" + name + "\" is still open: cannot delete");
+        }
       }
     }
     super.deleteFile(name);
   }
 
   public IndexOutput createOutput(String name) throws IOException {
-    if (openFiles == null) {
-      openFiles = new HashMap();
-    }
+    if (crashed)
+      throw new IOException("cannot createOutput after crash");
+    init();
     synchronized(openFiles) {
+      if (preventDoubleWrite && createdFiles.contains(name))
+        throw new IOException("file \"" + name + "\" was already written to");
       if (noDeleteOpenFile && openFiles.containsKey(name))
        throw new IOException("MockRAMDirectory: file \"" + name + "\" is still open: cannot overwrite");
     }
     RAMFile file = new RAMFile(this);
     synchronized (this) {
+      if (crashed)
+        throw new IOException("cannot createOutput after crash");
+      unSyncedFiles.add(name);
+      createdFiles.add(name);
       RAMFile existing = (RAMFile)fileMap.get(name);
       // Enforce write once:
       if (existing!=null && !name.equals("segments.gen"))
Index: src/test/org/apache/lucene/index/TestCrash.java
===================================================================
--- src/test/org/apache/lucene/index/TestCrash.java	(revision 0)
+++ src/test/org/apache/lucene/index/TestCrash.java	(revision 0)
@@ -0,0 +1,139 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.store.MockRAMDirectory;
+import org.apache.lucene.store.NoLockFactory;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+
+public class TestCrash extends LuceneTestCase {
+
+  private IndexWriter initIndex() throws IOException {
+    return initIndex(new MockRAMDirectory());
+  }
+
+  private IndexWriter initIndex(MockRAMDirectory dir) throws IOException {
+    dir.setLockFactory(NoLockFactory.getNoLockFactory());
+
+    IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer());
+    //writer.setMaxBufferedDocs(2);
+    writer.setMaxBufferedDocs(10);
+    ((ConcurrentMergeScheduler) writer.getMergeScheduler()).setSuppressExceptions();
+
+    Document doc = new Document();
+    doc.add(new Field("content", "aaa", Field.Store.YES, Field.Index.TOKENIZED));
+    doc.add(new Field("id", "0", Field.Store.YES, Field.Index.TOKENIZED));
+    for(int i=0;i<157;i++)
+      writer.addDocument(doc);
+
+    return writer;
+  }
+
+  private void crash(final IndexWriter writer) throws IOException {
+    final MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory();
+    ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler) writer.getMergeScheduler();
+    dir.crash();
+    cms.sync();
+    dir.clearCrash();
+  }
+
+  public void testCrashWhileIndexing() throws IOException {
+    IndexWriter writer = initIndex();
+    MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory();
+    crash(writer);
+    IndexReader reader = IndexReader.open(dir);
+    assertTrue(reader.numDocs() < 157);
+  }
+
+  public void testWriterAfterCrash() throws IOException {
+    IndexWriter writer = initIndex();
+    MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory();
+    dir.setPreventDoubleWrite(false);
+    crash(writer);
+    writer = initIndex(dir);
+    writer.close();
+
+    IndexReader reader = IndexReader.open(dir);
+    assertTrue(reader.numDocs() < 314);
+  }
+
+  public void testCrashAfterReopen() throws IOException {
+    IndexWriter writer = initIndex();
+    MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory();
+    writer.close();
+    writer = initIndex(dir);
+    assertEquals(314, writer.docCount());
+    crash(writer);
+
+    /*
+    System.out.println("\n\nTEST: open reader");
+    String[] l = dir.list();
+    Arrays.sort(l);
+    for(int i=0;i<l.length;i++)
+      System.out.println("file " + i + " = " + l[i] + " " +
+    dir.fileLength(l[i]) + " bytes");
+    */
+
+    IndexReader reader = IndexReader.open(dir);
+    assertTrue(reader.numDocs() >= 157);
+  }
+
+  public void testCrashAfterClose() throws IOException {
+    
+    IndexWriter writer = initIndex();
+    MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory();
+
+    writer.close();
+    dir.crash();
+
+    /*
+    String[] l = dir.list();
+    Arrays.sort(l);
+    for(int i=0;i<l.length;i++)
+      System.out.println("file " + i + " = " + l[i] + " " + dir.fileLength(l[i]) + " bytes");
+    */
+
+    IndexReader reader = IndexReader.open(dir);
+    assertEquals(157, reader.numDocs());
+  }
+
+  public void testCrashAfterCloseNoWait() throws IOException {
+    
+    IndexWriter writer = initIndex();
+    MockRAMDirectory dir = (MockRAMDirectory) writer.getDirectory();
+
+    writer.close(false);
+
+    dir.crash();
+
+    /*
+    String[] l = dir.list();
+    Arrays.sort(l);
+    for(int i=0;i<l.length;i++)
+      System.out.println("file " + i + " = " + l[i] + " " + dir.fileLength(l[i]) + " bytes");
+    */
+    IndexReader reader = IndexReader.open(dir);
+    assertEquals(157, reader.numDocs());
+  }
+}
\ No newline at end of file

Property changes on: src/test/org/apache/lucene/index/TestCrash.java
___________________________________________________________________
Name: svn:eol-style
   + native

Index: src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
===================================================================
--- src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java	(revision 603331)
+++ src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java	(working copy)
@@ -76,10 +76,7 @@
         writer.addDocument(doc);
       }
 
-      // Even though this won't delete any docs,
-      // IndexWriter's flush will still make a clone for all
-      // SegmentInfos on hitting the exception:
-      writer.deleteDocuments(new Term("id", "1000"));
+      writer.addDocument(doc);
       failure.setDoFail();
       try {
         writer.flush();
Index: src/test/org/apache/lucene/index/TestAtomicUpdate.java
===================================================================
--- src/test/org/apache/lucene/index/TestAtomicUpdate.java	(revision 603331)
+++ src/test/org/apache/lucene/index/TestAtomicUpdate.java	(working copy)
@@ -127,7 +127,7 @@
       d.add(new Field("contents", English.intToEnglish(i), Field.Store.NO, Field.Index.TOKENIZED));
       writer.addDocument(d);
     }
-    writer.flush();
+    writer.commit();
 
     IndexerThread indexerThread = new IndexerThread(writer, threads);
     threads[0] = indexerThread;
Index: src/test/org/apache/lucene/index/TestIndexWriterDelete.java
===================================================================
--- src/test/org/apache/lucene/index/TestIndexWriterDelete.java	(revision 603331)
+++ src/test/org/apache/lucene/index/TestIndexWriterDelete.java	(working copy)
@@ -31,7 +31,6 @@
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockRAMDirectory;
-import org.apache.lucene.store.RAMDirectory;
 
 public class TestIndexWriterDelete extends LuceneTestCase {
 
@@ -46,7 +45,7 @@
     for(int pass=0;pass<2;pass++) {
       boolean autoCommit = (0==pass);
 
-      Directory dir = new RAMDirectory();
+      Directory dir = new MockRAMDirectory();
       IndexWriter modifier = new IndexWriter(dir, autoCommit,
                                              new WhitespaceAnalyzer(), true);
       modifier.setUseCompoundFile(true);
@@ -66,28 +65,17 @@
         modifier.addDocument(doc);
       }
       modifier.optimize();
+      modifier.commit();
 
-      if (!autoCommit) {
-        modifier.close();
-      }
-
       Term term = new Term("city", "Amsterdam");
       int hitCount = getHitCount(dir, term);
       assertEquals(1, hitCount);
-      if (!autoCommit) {
-        modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
-        modifier.setUseCompoundFile(true);
-      }
       modifier.deleteDocuments(term);
-      if (!autoCommit) {
-        modifier.close();
-      }
+      modifier.commit();
       hitCount = getHitCount(dir, term);
       assertEquals(0, hitCount);
 
-      if (autoCommit) {
-        modifier.close();
-      }
+      modifier.close();
       dir.close();
     }
   }
@@ -97,7 +85,7 @@
     for(int pass=0;pass<2;pass++) {
       boolean autoCommit = (0==pass);
 
-      Directory dir = new RAMDirectory();
+      Directory dir = new MockRAMDirectory();
       IndexWriter modifier = new IndexWriter(dir, autoCommit,
                                              new WhitespaceAnalyzer(), true);
       modifier.setMaxBufferedDocs(2);
@@ -109,38 +97,26 @@
       for (int i = 0; i < 7; i++) {
         addDoc(modifier, ++id, value);
       }
-      modifier.flush();
+      modifier.commit();
 
       assertEquals(0, modifier.getNumBufferedDocuments());
       assertTrue(0 < modifier.getSegmentCount());
 
-      if (!autoCommit) {
-        modifier.close();
-      }
+      modifier.commit();
 
       IndexReader reader = IndexReader.open(dir);
       assertEquals(7, reader.numDocs());
       reader.close();
 
-      if (!autoCommit) {
-        modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
-        modifier.setMaxBufferedDocs(2);
-        modifier.setMaxBufferedDeleteTerms(2);
-      }
-
       modifier.deleteDocuments(new Term("value", String.valueOf(value)));
       modifier.deleteDocuments(new Term("value", String.valueOf(value)));
 
-      if (!autoCommit) {
-        modifier.close();
-      }
+      modifier.commit();
 
       reader = IndexReader.open(dir);
       assertEquals(0, reader.numDocs());
       reader.close();
-      if (autoCommit) {
-        modifier.close();
-      }
+      modifier.close();
       dir.close();
     }
   }
@@ -149,7 +125,7 @@
   public void testRAMDeletes() throws IOException {
     for(int pass=0;pass<2;pass++) {
       boolean autoCommit = (0==pass);
-      Directory dir = new RAMDirectory();
+      Directory dir = new MockRAMDirectory();
       IndexWriter modifier = new IndexWriter(dir, autoCommit,
                                              new WhitespaceAnalyzer(), true);
       modifier.setMaxBufferedDocs(4);
@@ -170,9 +146,7 @@
       assertEquals(0, modifier.getSegmentCount());
       modifier.flush();
 
-      if (!autoCommit) {
-        modifier.close();
-      }
+      modifier.commit();
 
       IndexReader reader = IndexReader.open(dir);
       assertEquals(1, reader.numDocs());
@@ -180,9 +154,7 @@
       int hitCount = getHitCount(dir, new Term("id", String.valueOf(id)));
       assertEquals(1, hitCount);
       reader.close();
-      if (autoCommit) {
-        modifier.close();
-      }
+      modifier.close();
       dir.close();
     }
   }
@@ -192,7 +164,7 @@
     for(int pass=0;pass<2;pass++) {
       boolean autoCommit = (0==pass);
 
-      Directory dir = new RAMDirectory();
+      Directory dir = new MockRAMDirectory();
       IndexWriter modifier = new IndexWriter(dir, autoCommit,
                                              new WhitespaceAnalyzer(), true);
       modifier.setMaxBufferedDocs(100);
@@ -209,23 +181,18 @@
       for (int i = 0; i < 5; i++) {
         addDoc(modifier, ++id, value);
       }
-      modifier.flush();
+      modifier.commit();
 
       for (int i = 0; i < 5; i++) {
         addDoc(modifier, ++id, value);
       }
       modifier.deleteDocuments(new Term("value", String.valueOf(value)));
 
-      modifier.flush();
-      if (!autoCommit) {
-        modifier.close();
-      }
+      modifier.commit();
 
       IndexReader reader = IndexReader.open(dir);
       assertEquals(5, reader.numDocs());
-      if (autoCommit) {
-        modifier.close();
-      }
+      modifier.close();
     }
   }
 
@@ -233,7 +200,7 @@
   public void testBatchDeletes() throws IOException {
     for(int pass=0;pass<2;pass++) {
       boolean autoCommit = (0==pass);
-      Directory dir = new RAMDirectory();
+      Directory dir = new MockRAMDirectory();
       IndexWriter modifier = new IndexWriter(dir, autoCommit,
                                              new WhitespaceAnalyzer(), true);
       modifier.setMaxBufferedDocs(2);
@@ -245,29 +212,17 @@
       for (int i = 0; i < 7; i++) {
         addDoc(modifier, ++id, value);
       }
-      modifier.flush();
-      if (!autoCommit) {
-        modifier.close();
-      }
+      modifier.commit();
 
       IndexReader reader = IndexReader.open(dir);
       assertEquals(7, reader.numDocs());
       reader.close();
       
-      if (!autoCommit) {
-        modifier = new IndexWriter(dir, autoCommit,
-                                   new WhitespaceAnalyzer());
-        modifier.setMaxBufferedDocs(2);
-        modifier.setMaxBufferedDeleteTerms(2);
-      }
-
       id = 0;
       modifier.deleteDocuments(new Term("id", String.valueOf(++id)));
       modifier.deleteDocuments(new Term("id", String.valueOf(++id)));
 
-      if (!autoCommit) {
-        modifier.close();
-      }
+      modifier.commit();
 
       reader = IndexReader.open(dir);
       assertEquals(5, reader.numDocs());
@@ -277,23 +232,13 @@
       for (int i = 0; i < terms.length; i++) {
         terms[i] = new Term("id", String.valueOf(++id));
       }
-      if (!autoCommit) {
-        modifier = new IndexWriter(dir, autoCommit,
-                                   new WhitespaceAnalyzer());
-        modifier.setMaxBufferedDocs(2);
-        modifier.setMaxBufferedDeleteTerms(2);
-      }
       modifier.deleteDocuments(terms);
-      if (!autoCommit) {
-        modifier.close();
-      }
+      modifier.commit();
       reader = IndexReader.open(dir);
       assertEquals(2, reader.numDocs());
       reader.close();
 
-      if (autoCommit) {
-        modifier.close();
-      }
+      modifier.close();
       dir.close();
     }
   }
@@ -339,7 +284,7 @@
       boolean autoCommit = (0==pass);
 
       // First build up a starting index:
-      RAMDirectory startDir = new RAMDirectory();
+      MockRAMDirectory startDir = new MockRAMDirectory();
       IndexWriter writer = new IndexWriter(startDir, autoCommit,
                                            new WhitespaceAnalyzer(), true);
       for (int i = 0; i < 157; i++) {
@@ -445,39 +390,14 @@
             }
           }
 
-          // Whether we succeeded or failed, check that all
-          // un-referenced files were in fact deleted (ie,
-          // we did not create garbage). Just create a
-          // new IndexFileDeleter, have it delete
-          // unreferenced files, then verify that in fact
-          // no files were deleted:
-          String[] startFiles = dir.list();
-          SegmentInfos infos = new SegmentInfos();
-          infos.read(dir);
-          IndexFileDeleter d = new IndexFileDeleter(dir, new KeepOnlyLastCommitDeletionPolicy(), infos, null, null);
-          String[] endFiles = dir.list();
+          // If the close() succeeded, make sure there are
+          // no unreferenced files.  Just create a new
+          // IndexFileDeleter, have it delete unreferenced
+          // files, then verify that in fact no files were
+          // deleted:
+          if (success)
+            TestIndexWriter.assertNoUnreferencedFiles(dir, "after writer.close");
 
-          Arrays.sort(startFiles);
-          Arrays.sort(endFiles);
-
-          // for(int i=0;i<startFiles.length;i++) {
-          // System.out.println(" startFiles: " + i + ": " + startFiles[i]);
-          // }
-
-          if (!Arrays.equals(startFiles, endFiles)) {
-            String successStr;
-            if (success) {
-              successStr = "success";
-            } else {
-              successStr = "IOException";
-              err.printStackTrace();
-            }
-            fail("reader.close() failed to delete unreferenced files after "
-                 + successStr + " (" + diskFree + " bytes): before delete:\n    "
-                 + arrayToString(startFiles) + "\n  after delete:\n    "
-                 + arrayToString(endFiles));
-          }
-
           // Finally, verify index is not corrupt, and, if
           // we succeeded, we see all docs changed, and if
           // we failed, we see either all docs or no docs
@@ -608,12 +528,8 @@
       // flush (and commit if ac)
 
       modifier.optimize();
+      modifier.commit();
 
-      // commit if !ac
-
-      if (!autoCommit) {
-        modifier.close();
-      }
       // one of the two files hits
 
       Term term = new Term("city", "Amsterdam");
@@ -622,11 +538,6 @@
 
       // open the writer again (closed above)
 
-      if (!autoCommit) {
-        modifier = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer());
-        modifier.setUseCompoundFile(true);
-      }
-
       // delete the doc
       // max buf del terms is two, so this is buffered
 
@@ -638,7 +549,7 @@
       Document doc = new Document();
       modifier.addDocument(doc);
 
-      // flush the changes, the buffered deletes, and the new doc
+      // commit the changes, the buffered deletes, and the new doc
 
       // The failure object will fail on the first write after the del
       // file gets created when processing the buffered delete
@@ -649,38 +560,28 @@
       // in the !ac case, a new segments file won't be created but in
       // this case, creation of the cfs file happens next so we need
       // the doc (to test that it's okay that we don't lose deletes if
-      // failing while creating the cfs file
+      // failing while creating the cfs file)
 
       boolean failed = false;
       try {
-        modifier.flush();
+        modifier.commit();
       } catch (IOException ioe) {
         failed = true;
       }
 
       assertTrue(failed);
 
-      // The flush above failed, so we need to retry it (which will
+      // The commit above failed, so we need to retry it (which will
       // succeed, because the failure is a one-shot)
 
-      if (!autoCommit) {
-        modifier.close();
-      } else {
-        modifier.flush();
-      }
+      modifier.commit();
 
       hitCount = getHitCount(dir, term);
 
-      // If the delete was not cleared then hit count will
-      // be 0.  With autoCommit=false, we hit the exception
-      // on creating the compound file, so the delete was
-      // flushed successfully.
-      assertEquals(autoCommit ? 1:0, hitCount);
+      // Make sure the delete was successfully flushed:
+      assertEquals(0, hitCount);
 
-      if (autoCommit) {
-        modifier.close();
-      }
-
+      modifier.close();
       dir.close();
     }
   }
Index: src/test/org/apache/lucene/index/TestIndexReader.java
===================================================================
--- src/test/org/apache/lucene/index/TestIndexReader.java	(revision 603331)
+++ src/test/org/apache/lucene/index/TestIndexReader.java	(working copy)
@@ -779,6 +779,11 @@
       // Iterate w/ ever increasing free disk space:
       while(!done) {
         MockRAMDirectory dir = new MockRAMDirectory(startDir);
+
+        // If IndexReader hits disk full, it will write to
+        // the same files again.
+        dir.setPreventDoubleWrite(false);
+
         IndexReader reader = IndexReader.open(dir);
 
         // For each disk size, first try to commit against
Index: src/test/org/apache/lucene/index/TestIndexWriter.java
===================================================================
--- src/test/org/apache/lucene/index/TestIndexWriter.java	(revision 603331)
+++ src/test/org/apache/lucene/index/TestIndexWriter.java	(working copy)
@@ -637,14 +637,13 @@
       writer.setMaxBufferedDocs(2);
 
       for(int iter=0;iter<10;iter++) {
-
         for(int i=0;i<19;i++)
           writer.addDocument(doc);
 
-        writer.flush();
+        ((ConcurrentMergeScheduler) writer.getMergeScheduler()).sync();
+        writer.commit();
 
         SegmentInfos sis = new SegmentInfos();
-        ((ConcurrentMergeScheduler) writer.getMergeScheduler()).sync();
         sis.read(dir);
 
         final int segCount = sis.size();
@@ -1031,7 +1030,7 @@
      * and add docs to it.
      */
     public void testCommitOnCloseAbort() throws IOException {
-      Directory dir = new RAMDirectory();      
+      Directory dir = new MockRAMDirectory();      
       IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
       writer.setMaxBufferedDocs(10);
       for (int i = 0; i < 14; i++) {
@@ -1259,48 +1258,48 @@
       writer.setMaxBufferedDocs(10);
       writer.setRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
 
-      long lastGen = -1;
+      int lastFlushCount = -1;
       for(int j=1;j<52;j++) {
         Document doc = new Document();
         doc.add(new Field("field", "aaa" + j, Field.Store.YES, Field.Index.TOKENIZED));
         writer.addDocument(doc);
         _TestUtil.syncConcurrentMerges(writer);
-        long gen = SegmentInfos.generationFromSegmentsFileName(SegmentInfos.getCurrentSegmentFileName(dir.list()));
+        int flushCount = writer.getFlushCount();
         if (j == 1)
-          lastGen = gen;
+          lastFlushCount = flushCount;
         else if (j < 10)
           // No new files should be created
-          assertEquals(gen, lastGen);
+          assertEquals(flushCount, lastFlushCount);
         else if (10 == j) {
-          assertTrue(gen > lastGen);
-          lastGen = gen;
+          assertTrue(flushCount > lastFlushCount);
+          lastFlushCount = flushCount;
           writer.setRAMBufferSizeMB(0.000001);
           writer.setMaxBufferedDocs(IndexWriter.DISABLE_AUTO_FLUSH);
         } else if (j < 20) {
-          assertTrue(gen > lastGen);
-          lastGen = gen;
+          assertTrue(flushCount > lastFlushCount);
+          lastFlushCount = flushCount;
         } else if (20 == j) {
           writer.setRAMBufferSizeMB(16);
           writer.setMaxBufferedDocs(IndexWriter.DISABLE_AUTO_FLUSH);
-          lastGen = gen;
+          lastFlushCount = flushCount;
         } else if (j < 30) {
-          assertEquals(gen, lastGen);
+          assertEquals(flushCount, lastFlushCount);
         } else if (30 == j) {
           writer.setRAMBufferSizeMB(0.000001);
           writer.setMaxBufferedDocs(IndexWriter.DISABLE_AUTO_FLUSH);
         } else if (j < 40) {
-          assertTrue(gen> lastGen);
-          lastGen = gen;
+          assertTrue(flushCount> lastFlushCount);
+          lastFlushCount = flushCount;
         } else if (40 == j) {
           writer.setMaxBufferedDocs(10);
           writer.setRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
-          lastGen = gen;
+          lastFlushCount = flushCount;
         } else if (j < 50) {
-          assertEquals(gen, lastGen);
+          assertEquals(flushCount, lastFlushCount);
           writer.setMaxBufferedDocs(10);
           writer.setRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
         } else if (50 == j) {
-          assertTrue(gen > lastGen);
+          assertTrue(flushCount > lastFlushCount);
         }
       }
       writer.close();
@@ -1320,46 +1319,46 @@
         writer.addDocument(doc);
       }
       
-      long lastGen = -1;
+      int lastFlushCount = -1;
       for(int j=1;j<52;j++) {
         writer.deleteDocuments(new Term("field", "aaa" + j));
         _TestUtil.syncConcurrentMerges(writer);
-        long gen = SegmentInfos.generationFromSegmentsFileName(SegmentInfos.getCurrentSegmentFileName(dir.list()));
+        int flushCount = writer.getFlushCount();
         if (j == 1)
-          lastGen = gen;
+          lastFlushCount = flushCount;
         else if (j < 10) {
           // No new files should be created
-          assertEquals(gen, lastGen);
+          assertEquals(flushCount, lastFlushCount);
         } else if (10 == j) {
-          assertTrue(gen > lastGen);
-          lastGen = gen;
+          assertTrue(flushCount > lastFlushCount);
+          lastFlushCount = flushCount;
           writer.setRAMBufferSizeMB(0.000001);
           writer.setMaxBufferedDeleteTerms(IndexWriter.DISABLE_AUTO_FLUSH);
         } else if (j < 20) {
-          assertTrue(gen > lastGen);
-          lastGen = gen;
+          assertTrue(flushCount > lastFlushCount);
+          lastFlushCount = flushCount;
         } else if (20 == j) {
           writer.setRAMBufferSizeMB(16);
           writer.setMaxBufferedDeleteTerms(IndexWriter.DISABLE_AUTO_FLUSH);
-          lastGen = gen;
+          lastFlushCount = flushCount;
         } else if (j < 30) {
-          assertEquals(gen, lastGen);
+          assertEquals(flushCount, lastFlushCount);
         } else if (30 == j) {
           writer.setRAMBufferSizeMB(0.000001);
           writer.setMaxBufferedDeleteTerms(IndexWriter.DISABLE_AUTO_FLUSH);
         } else if (j < 40) {
-          assertTrue(gen> lastGen);
-          lastGen = gen;
+          assertTrue(flushCount> lastFlushCount);
+          lastFlushCount = flushCount;
         } else if (40 == j) {
           writer.setMaxBufferedDeleteTerms(10);
           writer.setRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
-          lastGen = gen;
+          lastFlushCount = flushCount;
         } else if (j < 50) {
-          assertEquals(gen, lastGen);
+          assertEquals(flushCount, lastFlushCount);
           writer.setMaxBufferedDeleteTerms(10);
           writer.setRAMBufferSizeMB(IndexWriter.DISABLE_AUTO_FLUSH);
         } else if (50 == j) {
-          assertTrue(gen > lastGen);
+          assertTrue(flushCount > lastFlushCount);
         }
       }
       writer.close();
@@ -1807,12 +1806,19 @@
     public void eval(MockRAMDirectory dir)  throws IOException {
       if (doFail) {
         StackTraceElement[] trace = new Exception().getStackTrace();
+        boolean sawAppend = false;
+        boolean sawFlush = false;
         for (int i = 0; i < trace.length; i++) {
-          if ("appendPostings".equals(trace[i].getMethodName()) && count++ == 30) {
-            doFail = false;
-            throw new IOException("now failing during flush");
-          }
+          if ("appendPostings".equals(trace[i].getMethodName()))
+            sawAppend = true;
+          if ("doFlush".equals(trace[i].getMethodName()))
+            sawFlush = true;
         }
+
+        if (sawAppend && sawFlush && count++ >= 30) {
+          doFail = false;
+          throw new IOException("now failing during flush");
+        }
       }
     }
   }
Index: src/test/org/apache/lucene/index/TestStressIndexing.java
===================================================================
--- src/test/org/apache/lucene/index/TestStressIndexing.java	(revision 603331)
+++ src/test/org/apache/lucene/index/TestStressIndexing.java	(working copy)
@@ -123,6 +123,7 @@
     modifier.setMaxBufferedDocs(10);
 
     TimedThread[] threads = new TimedThread[4];
+    int numThread = 0;
 
     if (mergeScheduler != null)
       modifier.setMergeScheduler(mergeScheduler);
@@ -130,34 +131,30 @@
     // One modifier that writes 10 docs then removes 5, over
     // and over:
     IndexerThread indexerThread = new IndexerThread(modifier, threads);
-    threads[0] = indexerThread;
+    threads[numThread++] = indexerThread;
     indexerThread.start();
-      
+    
     IndexerThread indexerThread2 = new IndexerThread(modifier, threads);
-    threads[2] = indexerThread2;
+    threads[numThread++] = indexerThread2;
     indexerThread2.start();
       
     // Two searchers that constantly just re-instantiate the
     // searcher:
     SearcherThread searcherThread1 = new SearcherThread(directory, threads);
-    threads[3] = searcherThread1;
+    threads[numThread++] = searcherThread1;
     searcherThread1.start();
 
     SearcherThread searcherThread2 = new SearcherThread(directory, threads);
-    threads[3] = searcherThread2;
+    threads[numThread++] = searcherThread2;
     searcherThread2.start();
 
-    indexerThread.join();
-    indexerThread2.join();
-    searcherThread1.join();
-    searcherThread2.join();
+    for(int i=0;i<numThread;i++)
+      threads[i].join();
 
     modifier.close();
 
-    assertTrue("hit unexpected exception in indexer", !indexerThread.failed);
-    assertTrue("hit unexpected exception in indexer2", !indexerThread2.failed);
-    assertTrue("hit unexpected exception in search1", !searcherThread1.failed);
-    assertTrue("hit unexpected exception in search2", !searcherThread2.failed);
+    for(int i=0;i<numThread;i++)
+      assertTrue(!((TimedThread) threads[i]).failed);
 
     //System.out.println("    Writer: " + indexerThread.count + " iterations");
     //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created");
Index: src/test/org/apache/lucene/index/TestIndexFileDeleter.java
===================================================================
--- src/test/org/apache/lucene/index/TestIndexFileDeleter.java	(revision 603331)
+++ src/test/org/apache/lucene/index/TestIndexFileDeleter.java	(working copy)
@@ -77,8 +77,8 @@
     String[] files = dir.list();
 
     /*
-    for(int i=0;i<files.length;i++) {
-      System.out.println(i + ": " + files[i]);
+    for(int j=0;j<files.length;j++) {
+      System.out.println(j + ": " + files[j]);
     }
     */
 
@@ -145,8 +145,8 @@
     copyFile(dir, "_0.cfs", "deletable");
 
     // Create some old segments file:
-    copyFile(dir, "segments_a", "segments");
-    copyFile(dir, "segments_a", "segments_2");
+    copyFile(dir, "segments_3", "segments");
+    copyFile(dir, "segments_3", "segments_2");
 
     // Create a bogus cfs file shadowing a non-cfs segment:
     copyFile(dir, "_2.cfs", "_3.cfs");
Index: src/test/org/apache/lucene/index/TestDeletionPolicy.java
===================================================================
--- src/test/org/apache/lucene/index/TestDeletionPolicy.java	(revision 603331)
+++ src/test/org/apache/lucene/index/TestDeletionPolicy.java	(working copy)
@@ -270,13 +270,10 @@
       writer.close();
 
       assertEquals(2, policy.numOnInit);
-      if (autoCommit) {
-        assertTrue(policy.numOnCommit > 2);
-      } else {
+      if (!autoCommit)
         // If we are not auto committing then there should
         // be exactly 2 commits (one per close above):
         assertEquals(2, policy.numOnCommit);
-      }
 
       // Simplistic check: just verify all segments_N's still
       // exist, and, I can open a reader on each:
@@ -334,13 +331,10 @@
       writer.close();
 
       assertEquals(2, policy.numOnInit);
-      if (autoCommit) {
-        assertTrue(policy.numOnCommit > 2);
-      } else {
+      if (!autoCommit)
         // If we are not auto committing then there should
         // be exactly 2 commits (one per close above):
         assertEquals(2, policy.numOnCommit);
-      }
 
       // Simplistic check: just verify the index is in fact
       // readable:
@@ -459,11 +453,8 @@
       writer.close();
 
       assertEquals(2*(N+2), policy.numOnInit);
-      if (autoCommit) {
-        assertTrue(policy.numOnCommit > 2*(N+2)-1);
-      } else {
+      if (!autoCommit)
         assertEquals(2*(N+2)-1, policy.numOnCommit);
-      }
 
       IndexSearcher searcher = new IndexSearcher(dir);
       Hits hits = searcher.search(query);
@@ -565,11 +556,8 @@
       }
 
       assertEquals(1+3*(N+1), policy.numOnInit);
-      if (autoCommit) {
-        assertTrue(policy.numOnCommit > 3*(N+1)-1);
-      } else {
+      if (!autoCommit)
         assertEquals(2*(N+1), policy.numOnCommit);
-      }
 
       IndexSearcher searcher = new IndexSearcher(dir);
       Hits hits = searcher.search(query);
Index: src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
===================================================================
--- src/test/org/apache/lucene/index/TestBackwardsCompatibility.java	(revision 603331)
+++ src/test/org/apache/lucene/index/TestBackwardsCompatibility.java	(working copy)
@@ -317,7 +317,6 @@
  
         IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
         writer.setRAMBufferSizeMB(16.0);
-        //IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
         for(int i=0;i<35;i++) {
           addDoc(writer, i);
         }
@@ -358,12 +357,9 @@
         expected = new String[] {"_0.cfs",
                     "_0_1.del",
                     "_0_1.s" + contentFieldIndex,
-                    "segments_4",
+                    "segments_3",
                     "segments.gen"};
 
-        if (!autoCommit)
-          expected[3] = "segments_3";
-
         String[] actual = dir.list();
         Arrays.sort(expected);
         Arrays.sort(actual);
Index: src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
===================================================================
--- src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java	(revision 603331)
+++ src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java	(working copy)
@@ -267,6 +267,10 @@
             exceptions.add(exc);
           }
           
+          // nocommit
+          System.out.println("\nEXC DURING MERGE");
+          exc.printStackTrace(System.out);
+
           if (!suppressExceptions) {
             // suppressExceptions is normally only set during
             // testing.
@@ -276,8 +280,8 @@
         }
       } finally {
         synchronized(ConcurrentMergeScheduler.this) {
+          ConcurrentMergeScheduler.this.notifyAll();
           mergeThreads.remove(this);
-          ConcurrentMergeScheduler.this.notifyAll();
         }
       }
     }
Index: src/java/org/apache/lucene/index/SegmentInfos.java
===================================================================
--- src/java/org/apache/lucene/index/SegmentInfos.java	(revision 603332)
+++ src/java/org/apache/lucene/index/SegmentInfos.java	(working copy)
@@ -620,7 +620,7 @@
             retry = true;
           }
 
-        } else {
+        } else if (0 == method) {
           // Segment file has advanced since our last loop, so
           // reset retry:
           retry = false;
@@ -701,4 +701,11 @@
     infos.addAll(super.subList(first, last));
     return infos;
   }
+
+  // nocommit javadoc
+  void updateGeneration(SegmentInfos other) {
+    assert other.generation > generation;
+    lastGeneration = other.lastGeneration;
+    generation = other.generation;
+  }
 }
Index: src/java/org/apache/lucene/index/IndexWriter.java
===================================================================
--- src/java/org/apache/lucene/index/IndexWriter.java	(revision 603331)
+++ src/java/org/apache/lucene/index/IndexWriter.java	(working copy)
@@ -260,7 +260,7 @@
 
   private Similarity similarity = Similarity.getDefault(); // how to normalize
 
-  private boolean commitPending; // true if segmentInfos has changes not yet committed
+  private volatile boolean commitPending; // true if segmentInfos has changes not yet committed
   private SegmentInfos rollbackSegmentInfos;      // segmentInfos we will fallback to if the commit fails
 
   private SegmentInfos localRollbackSegmentInfos;      // segmentInfos we will fallback to if the commit fails
@@ -268,6 +268,9 @@
   private boolean autoCommit = true;              // false if we should commit only on close
 
   private SegmentInfos segmentInfos = new SegmentInfos();       // the segments
+  private int syncCount;
+  private int syncCountSaved = -1;
+
   private DocumentsWriter docWriter;
   private IndexFileDeleter deleter;
 
@@ -292,6 +295,9 @@
   private List mergeExceptions = new ArrayList();
   private long mergeGen;
 
+  private int flushCount;
+  private SegmentInfo lastMergeInfo;
+
   /**
    * Used internally to throw an {@link
    * AlreadyClosedException} if this IndexWriter has been
@@ -678,6 +684,8 @@
       directory.clearLock(IndexWriter.WRITE_LOCK_NAME);
     }
 
+    this.autoCommit = autoCommit;
+
     Lock writeLock = directory.makeLock(IndexWriter.WRITE_LOCK_NAME);
     if (!writeLock.obtain(writeLockTimeout)) // obtain write lock
       throw new LockObtainFailedException("Index locked for write: " + writeLock);
@@ -695,12 +703,21 @@
         } catch (IOException e) {
           // Likely this means it's a fresh directory
         }
-        segmentInfos.write(directory);
+        commitPending = true;
+        sync(false, false);
       } else {
         segmentInfos.read(directory);
+
+        // We assume that this segments_N was previously
+        // properly sync'd:
+        for(int i=0;i<segmentInfos.size();i++) {
+          final SegmentInfo info = segmentInfos.info(i);
+          List files = info.files();
+          for(int j=0;j<files.size();j++)
+            synced.add(files.get(j));
+        }
       }
 
-      this.autoCommit = autoCommit;
       if (!autoCommit) {
         rollbackSegmentInfos = (SegmentInfos) segmentInfos.clone();
       }
@@ -1196,27 +1213,13 @@
 
       mergeScheduler.close();
 
-      if (commitPending) {
-        boolean success = false;
-        try {
-          segmentInfos.write(directory);         // now commit changes
-          success = true;
-        } finally {
-          if (!success) {
-            if (infoStream != null)
-              message("hit exception committing segments file during close");
-            deletePartialSegmentsFile();
-          }
-        }
-        if (infoStream != null)
-          message("close: wrote segments file \"" + segmentInfos.getCurrentSegmentFileName() + "\"");
-        synchronized(this) {
-          deleter.checkpoint(segmentInfos, true);
-        }
-        commitPending = false;
-        rollbackSegmentInfos = null;
-      }
+      if (infoStream != null)
+        message("now call final sync()");
 
+      sync(true, true);
+
+      rollbackSegmentInfos = null;
+
       if (infoStream != null)
         message("at close: " + segString());
 
@@ -1233,7 +1236,9 @@
         writeLock.release();                          // release write lock
         writeLock = null;
       }
-      closed = true;
+      synchronized(this) {
+        closed = true;
+      }
 
     } finally {
       synchronized(this) {
@@ -1287,34 +1292,24 @@
       
           // Perform the merge
           cfsWriter.close();
+          success = true;
 
-          for(int i=0;i<numSegments;i++) {
-            SegmentInfo si = segmentInfos.info(i);
-            if (si.getDocStoreOffset() != -1 &&
-                si.getDocStoreSegment().equals(docStoreSegment))
-              si.setDocStoreIsCompoundFile(true);
-          }
-          checkpoint();
-          success = true;
         } finally {
           if (!success) {
-
             if (infoStream != null)
               message("hit exception building compound file doc store for segment " + docStoreSegment);
-            
-            // Rollback to no compound file
-            for(int i=0;i<numSegments;i++) {
-              SegmentInfo si = segmentInfos.info(i);
-              if (si.getDocStoreOffset() != -1 &&
-                  si.getDocStoreSegment().equals(docStoreSegment))
-                si.setDocStoreIsCompoundFile(false);
-            }
             deleter.deleteFile(compoundFileName);
-            deletePartialSegmentsFile();
           }
         }
 
-        deleter.checkpoint(segmentInfos, false);
+        for(int i=0;i<numSegments;i++) {
+          SegmentInfo si = segmentInfos.info(i);
+          if (si.getDocStoreOffset() != -1 &&
+              si.getDocStoreSegment().equals(docStoreSegment))
+            si.setDocStoreIsCompoundFile(true);
+        }
+
+        checkpoint();
       }
     }
 
@@ -1564,6 +1559,11 @@
     }
   }
 
+  // for test purpose
+  final synchronized int getFlushCount() {
+    return flushCount;
+  }
+
   final String newSegmentName() {
     // Cannot synchronize on IndexWriter because that causes
     // deadlock
@@ -1698,7 +1698,7 @@
     if (infoStream != null)
       message("optimize: index now " + segString());
 
-    flush();
+    flush(true, false);
 
     synchronized(this) {
       resetMergeExceptions();
@@ -1742,7 +1742,9 @@
               final MergePolicy.OneMerge merge = (MergePolicy.OneMerge) mergeExceptions.get(0);
               if (merge.optimize) {
                 IOException err = new IOException("background merge hit exception: " + merge.segString(directory));
-                err.initCause(merge.getException());
+                final Throwable t = merge.getException();
+                if (t != null)
+                  err.initCause(t);
                 throw err;
               }
             }
@@ -1866,7 +1868,8 @@
       if (infoStream != null)
         message("flush at startTransaction");
 
-      flush();
+      flush(true, false);
+
       // Turn off auto-commit during our local transaction:
       autoCommit = false;
     } else
@@ -1905,6 +1908,7 @@
 
     deleter.refresh();
     finishMerges(false);
+    lastMergeInfo = null;
   }
 
   /*
@@ -1920,27 +1924,26 @@
     // First restore autoCommit in case we hit an exception below:
     autoCommit = localAutoCommit;
 
-    boolean success = false;
-    try {
-      checkpoint();
-      success = true;
-    } finally {
-      if (!success) {
-        if (infoStream != null)
-          message("hit exception committing transaction");
+    // Give deleter a chance to remove files now:
+    checkpoint();
 
-        rollbackTransaction();
+    if (autoCommit) {
+      boolean success = false;
+      try {
+        sync(false, false);
+        success = true;
+      } finally {
+        if (!success) {
+          if (infoStream != null)
+            message("hit exception committing transaction");
+          rollbackTransaction();
+        }
       }
-    }
-
-    if (!autoCommit)
+    } else
       // Remove the incRef we did in startTransaction.
       deleter.decRef(localRollbackSegmentInfos);
 
     localRollbackSegmentInfos = null;
-
-    // Give deleter a chance to remove files now:
-    deleter.checkpoint(segmentInfos, autoCommit);
   }
 
   /**
@@ -1995,16 +1998,20 @@
         // them:
         deleter.checkpoint(segmentInfos, false);
         deleter.refresh();
-        finishMerges(false);
       }
 
-      commitPending = false;
+      // Force the segmentInfos to be written to a new
+      // segments_N file so on re-opening we do not try to
+      // re-write files that were already written (keep
+      // write once even on abort):
+      commitPending = true;
       closeInternal(false);
     } else
       waitForClose();
   }
 
   private synchronized void finishMerges(boolean waitForMerges) {
+
     if (!waitForMerges) {
       // Abort all pending & running merges:
       Iterator it = pendingMerges.iterator();
@@ -2027,6 +2034,7 @@
 
       mergingSegments.clear();
       notifyAll();
+
     } else {
       while(pendingMerges.size() > 0 || runningMerges.size() > 0) {
         try {
@@ -2041,19 +2049,11 @@
   /*
    * Called whenever the SegmentInfos has been updated and
    * the index files referenced exist (correctly) in the
-   * index directory.  If we are in autoCommit mode, we
-   * commit the change immediately.  Else, we mark
-   * commitPending.
+   * index directory.
    */
   private synchronized void checkpoint() throws IOException {
-    if (autoCommit) {
-      segmentInfos.write(directory);
-      commitPending = false;
-      if (infoStream != null)
-        message("checkpoint: wrote segments file \"" + segmentInfos.getCurrentSegmentFileName() + "\"");
-    } else {
-      commitPending = true;
-    }
+    commitPending = true;
+    deleter.checkpoint(segmentInfos, false);
   }
 
   /** Merges all segments from an array of indexes into this index.
@@ -2114,7 +2114,7 @@
     ensureOpen();
     if (infoStream != null)
       message("flush at addIndexes");
-    flush();
+    flush(true, false);
 
     boolean success = false;
 
@@ -2176,7 +2176,7 @@
     ensureOpen();
     if (infoStream != null)
       message("flush at addIndexesNoOptimize");
-    flush();
+    flush(true, false);
 
     boolean success = false;
 
@@ -2348,12 +2348,26 @@
    * not be visible to readers, until {@link #close} is called.
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
+   * @deprecated please call commmit() instead
    */
   public final void flush() throws CorruptIndexException, IOException {  
     flush(true, false);
   }
 
   /**
+   * Commits & syncs all pending updates (added & deleted
+   * documents) to the index.
+   */
+  public final void commit() throws CorruptIndexException, IOException {
+    commit(true);
+  }
+
+  private final void commit(boolean triggerMerges) throws CorruptIndexException, IOException {
+    flush(triggerMerges, true);
+    sync(false, true);
+  }
+
+  /**
    * Flush all in-memory buffered udpates (adds and deletes)
    * to the Directory.
    * @param triggerMerge if true, we may merge segments (if
@@ -2368,11 +2382,16 @@
       maybeMerge();
   }
 
+  // TODO: this method should not have to be entirely
+  // synchronized, ie, merges should be allowed to commit
+  // even while a flush is happening
   private synchronized final boolean doFlush(boolean flushDocStores) throws CorruptIndexException, IOException {
 
     // Make sure no threads are actively adding a document
     docWriter.pauseAllThreads();
 
+    flushCount++;
+
     try {
 
       SegmentInfo newSegment = null;
@@ -2430,121 +2449,103 @@
       // If we are flushing docs, segment must not be null:
       assert segment != null || !flushDocs;
 
-      if (flushDocs || flushDeletes) {
+      if (flushDocs) {
 
-        SegmentInfos rollback = null;
-
-        if (flushDeletes)
-          rollback = (SegmentInfos) segmentInfos.clone();
-
         boolean success = false;
+        final int flushedDocCount;
 
         try {
-          if (flushDocs) {
-
-            if (0 == docStoreOffset && flushDocStores) {
-              // This means we are flushing private doc stores
-              // with this segment, so it will not be shared
-              // with other segments
-              assert docStoreSegment != null;
-              assert docStoreSegment.equals(segment);
-              docStoreOffset = -1;
-              docStoreIsCompoundFile = false;
-              docStoreSegment = null;
-            }
-
-            int flushedDocCount = docWriter.flush(flushDocStores);
-          
-            newSegment = new SegmentInfo(segment,
-                                         flushedDocCount,
-                                         directory, false, true,
-                                         docStoreOffset, docStoreSegment,
-                                         docStoreIsCompoundFile);
-            segmentInfos.addElement(newSegment);
-          }
-
-          if (flushDeletes) {
-            // we should be able to change this so we can
-            // buffer deletes longer and then flush them to
-            // multiple flushed segments, when
-            // autoCommit=false
-            int delCount = applyDeletes(flushDocs);
-            if (infoStream != null)
-              infoStream.println("flushed " + delCount + " deleted documents");
-            doAfterFlush();
-          }
-
-          checkpoint();
+          flushedDocCount = docWriter.flush(flushDocStores);
           success = true;
         } finally {
           if (!success) {
-
             if (infoStream != null)
               message("hit exception flushing segment " + segment);
-                
-            if (flushDeletes) {
-
-              // Carefully check if any partial .del files
-              // should be removed:
-              final int size = rollback.size();
-              for(int i=0;i<size;i++) {
-                final String newDelFileName = segmentInfos.info(i).getDelFileName();
-                final String delFileName = rollback.info(i).getDelFileName();
-                if (newDelFileName != null && !newDelFileName.equals(delFileName))
-                  deleter.deleteFile(newDelFileName);
-              }
-
-              // Fully replace the segmentInfos since flushed
-              // deletes could have changed any of the
-              // SegmentInfo instances:
-              segmentInfos.clear();
-              segmentInfos.addAll(rollback);
-              
-            } else {
-              // Remove segment we added, if any:
-              if (newSegment != null && 
-                  segmentInfos.size() > 0 && 
-                  segmentInfos.info(segmentInfos.size()-1) == newSegment)
-                segmentInfos.remove(segmentInfos.size()-1);
-            }
-            if (flushDocs)
-              docWriter.abort();
-            deletePartialSegmentsFile();
-            deleter.checkpoint(segmentInfos, false);
-
-            if (segment != null)
-              deleter.refresh(segment);
+            docWriter.abort();
+            deleter.refresh(segment);
           }
         }
+        
+        if (0 == docStoreOffset && flushDocStores) {
+          // This means we are flushing private doc stores
+          // with this segment, so it will not be shared
+          // with other segments
+          assert docStoreSegment != null;
+          assert docStoreSegment.equals(segment);
+          docStoreOffset = -1;
+          docStoreIsCompoundFile = false;
+          docStoreSegment = null;
+        }
 
-        deleter.checkpoint(segmentInfos, autoCommit);
+        newSegment = new SegmentInfo(segment,
+                                     flushedDocCount,
+                                     directory, false, true,
+                                     docStoreOffset, docStoreSegment,
+                                     docStoreIsCompoundFile);
+        segmentInfos.addElement(newSegment);
+        checkpoint();
 
-        if (flushDocs && mergePolicy.useCompoundFile(segmentInfos,
-                                                     newSegment)) {
+        if (mergePolicy.useCompoundFile(segmentInfos, newSegment)) {
           success = false;
           try {
             docWriter.createCompoundFile(segment);
-            newSegment.setUseCompoundFile(true);
-            checkpoint();
             success = true;
           } finally {
             if (!success) {
               if (infoStream != null)
                 message("hit exception creating compound file for newly flushed segment " + segment);
-              newSegment.setUseCompoundFile(false);
               deleter.deleteFile(segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
-              deletePartialSegmentsFile();
             }
           }
 
-          deleter.checkpoint(segmentInfos, autoCommit);
+          newSegment.setUseCompoundFile(true);
+          checkpoint();
         }
-      
-        return true;
-      } else {
-        return false;
       }
 
+      if (flushDeletes) {
+        SegmentInfos rollback = (SegmentInfos) segmentInfos.clone();
+
+        boolean success = false;
+        try {
+          // we should be able to change this so we can
+          // buffer deletes longer and then flush them to
+          // multiple flushed segments only when a commit()
+          // finally happens
+          int delCount = applyDeletes(flushDocs);
+          if (infoStream != null)
+            infoStream.println("flushed " + delCount + " deleted documents");
+          success = true;
+        } finally {
+          if (!success) {
+            if (infoStream != null)
+              message("hit exception flushing deletes");
+                
+            // Carefully check if any partial .del files
+            // should be removed:
+            final int size = rollback.size();
+            for(int i=0;i<size;i++) {
+              final String newDelFileName = segmentInfos.info(i).getDelFileName();
+              final String delFileName = rollback.info(i).getDelFileName();
+              if (newDelFileName != null && !newDelFileName.equals(delFileName))
+                deleter.deleteFile(newDelFileName);
+            }
+
+            // Fully replace the segmentInfos since flushed
+            // deletes could have changed any of the
+            // SegmentInfo instances:
+            segmentInfos.clear();
+            segmentInfos.addAll(rollback);
+          }              
+        }
+
+        doAfterFlush();
+        checkpoint();
+      }
+
+      
+      return flushDocs || flushDeletes;
+
     } finally {
       docWriter.clearFlushPending();
       docWriter.resumeAllThreads();
@@ -2560,7 +2561,7 @@
   }
 
   /** Expert:  Return the number of documents whose segments are currently cached in memory.
-   * Useful when calling flush()
+   *  Useful when calling flush()
    */
   public final synchronized int numRamDocs() {
     ensureOpen();
@@ -2590,9 +2591,101 @@
     return first;
   }
 
+  /** Carefully merges deletes for the segments we just
+   *  merged.  This is tricky because, although merging will
+   *  clear all deletes (compacts the documents), new
+   *  deletes may have been flushed to the segments since
+   *  the merge was started.  This method "carries over"
+   *  such new deletes onto the newly merged segment, and
+   *  saves the results deletes file (incrementing the
+   *  delete generation for merge.info).  If no deletes were
+   *  flushed, no new deletes file is saved. */
+  synchronized private void commitMergedDeletes(MergePolicy.OneMerge merge) throws IOException {
+    final SegmentInfos sourceSegmentsClone = merge.segmentsClone;
+    final SegmentInfos sourceSegments = merge.segments;
+
+    if (infoStream != null)
+      message("commitMerge " + merge.segString(directory));
+
+    // Carefully merge deletes that occurred after we
+    // started merging:
+
+    BitVector deletes = null;
+    int docUpto = 0;
+
+    final int numSegmentsToMerge = sourceSegments.size();
+    for(int i=0;i<numSegmentsToMerge;i++) {
+      final SegmentInfo previousInfo = sourceSegmentsClone.info(i);
+      final SegmentInfo currentInfo = sourceSegments.info(i);
+
+      assert currentInfo.docCount == previousInfo.docCount;
+
+      final int docCount = currentInfo.docCount;
+
+      if (previousInfo.hasDeletions()) {
+
+        // There were deletes on this segment when the merge
+        // started.  The merge has collapsed away those
+        // deletes, but, if new deletes were flushed since
+        // the merge started, we must now carefully keep any
+        // newly flushed deletes but mapping them to the new
+        // docIDs.
+
+        assert currentInfo.hasDeletions();
+
+        // Load deletes present @ start of merge, for this segment:
+        BitVector previousDeletes = new BitVector(previousInfo.dir, previousInfo.getDelFileName());
+
+        if (!currentInfo.getDelFileName().equals(previousInfo.getDelFileName())) {
+          // This means this segment has had new deletes
+          // committed since we started the merge, so we
+          // must merge them:
+          if (deletes == null)
+            deletes = new BitVector(merge.info.docCount);
+
+          BitVector currentDeletes = new BitVector(currentInfo.dir, currentInfo.getDelFileName());
+          for(int j=0;j<docCount;j++) {
+            if (previousDeletes.get(j))
+              assert currentDeletes.get(j);
+            else {
+              if (currentDeletes.get(j))
+                deletes.set(docUpto);
+              docUpto++;
+            }
+          }
+        } else
+          docUpto += docCount - previousDeletes.count();
+        
+      } else if (currentInfo.hasDeletions()) {
+        // This segment had no deletes before but now it
+        // does:
+        if (deletes == null)
+          deletes = new BitVector(merge.info.docCount);
+        BitVector currentDeletes = new BitVector(directory, currentInfo.getDelFileName());
+
+        for(int j=0;j<docCount;j++) {
+          if (currentDeletes.get(j))
+            deletes.set(docUpto);
+          docUpto++;
+        }
+            
+      } else
+        // No deletes before or after
+        docUpto += currentInfo.docCount;
+    }
+
+    if (deletes != null) {
+      merge.info.advanceDelGen();
+      deletes.write(directory, merge.info.getDelFileName());
+    }
+  }
+
   /* FIXME if we want to support non-contiguous segment merges */
   synchronized private boolean commitMerge(MergePolicy.OneMerge merge) throws IOException {
 
+    if (infoStream != null)
+      message("commitMerge: " + merge.segString(directory));
+
     assert merge.registerDone;
 
     // If merge was explicitly aborted, or, if abort() or
@@ -2605,104 +2698,14 @@
       if (infoStream != null)
         message("commitMerge: skipping merge " + merge.segString(directory) + ": it was aborted");
 
-      assert merge.increfDone;
-      decrefMergeSegments(merge);
       deleter.refresh(merge.info.name);
       return false;
     }
 
-    boolean success = false;
+    final int start = ensureContiguousMerge(merge);
 
-    int start;
+    commitMergedDeletes(merge);
 
-    try {
-      SegmentInfos sourceSegmentsClone = merge.segmentsClone;
-      SegmentInfos sourceSegments = merge.segments;
-
-      start = ensureContiguousMerge(merge);
-      if (infoStream != null)
-        message("commitMerge " + merge.segString(directory));
-
-      // Carefully merge deletes that occurred after we
-      // started merging:
-
-      BitVector deletes = null;
-      int docUpto = 0;
-
-      final int numSegmentsToMerge = sourceSegments.size();
-      for(int i=0;i<numSegmentsToMerge;i++) {
-        final SegmentInfo previousInfo = sourceSegmentsClone.info(i);
-        final SegmentInfo currentInfo = sourceSegments.info(i);
-
-        assert currentInfo.docCount == previousInfo.docCount;
-
-        final int docCount = currentInfo.docCount;
-
-        if (previousInfo.hasDeletions()) {
-
-          // There were deletes on this segment when the merge
-          // started.  The merge has collapsed away those
-          // deletes, but, if new deletes were flushed since
-          // the merge started, we must now carefully keep any
-          // newly flushed deletes but mapping them to the new
-          // docIDs.
-
-          assert currentInfo.hasDeletions();
-
-          // Load deletes present @ start of merge, for this segment:
-          BitVector previousDeletes = new BitVector(previousInfo.dir, previousInfo.getDelFileName());
-
-          if (!currentInfo.getDelFileName().equals(previousInfo.getDelFileName())) {
-            // This means this segment has had new deletes
-            // committed since we started the merge, so we
-            // must merge them:
-            if (deletes == null)
-              deletes = new BitVector(merge.info.docCount);
-
-            BitVector currentDeletes = new BitVector(currentInfo.dir, currentInfo.getDelFileName());
-            for(int j=0;j<docCount;j++) {
-              if (previousDeletes.get(j))
-                assert currentDeletes.get(j);
-              else {
-                if (currentDeletes.get(j))
-                  deletes.set(docUpto);
-                docUpto++;
-              }
-            }
-          } else
-            docUpto += docCount - previousDeletes.count();
-        
-        } else if (currentInfo.hasDeletions()) {
-          // This segment had no deletes before but now it
-          // does:
-          if (deletes == null)
-            deletes = new BitVector(merge.info.docCount);
-          BitVector currentDeletes = new BitVector(directory, currentInfo.getDelFileName());
-
-          for(int j=0;j<docCount;j++) {
-            if (currentDeletes.get(j))
-              deletes.set(docUpto);
-            docUpto++;
-          }
-            
-        } else
-          // No deletes before or after
-          docUpto += currentInfo.docCount;
-      }
-
-      if (deletes != null) {
-        merge.info.advanceDelGen();
-        deletes.write(directory, merge.info.getDelFileName());
-      }
-      success = true;
-    } finally {
-      if (!success) {
-        if (infoStream != null)
-          message("hit exception creating merged deletes file");
-        deleter.refresh(merge.info.name);
-      }
-    }
-
     // Simple optimization: if the doc store we are using
     // has been closed and is in now compound format (but
     // wasn't when we started), then we will switch to the
@@ -2722,32 +2725,20 @@
       }
     }
 
-    success = false;
-    SegmentInfos rollback = null;
-    try {
-      rollback = (SegmentInfos) segmentInfos.clone();
-      segmentInfos.subList(start, start + merge.segments.size()).clear();
-      segmentInfos.add(start, merge.info);
-      checkpoint();
-      success = true;
-    } finally {
-      if (!success && rollback != null) {
-        if (infoStream != null)
-          message("hit exception when checkpointing after merge");
-        segmentInfos.clear();
-        segmentInfos.addAll(rollback);
-        deletePartialSegmentsFile();
-        deleter.refresh(merge.info.name);
-      }
-    }
+    segmentInfos.subList(start, start + merge.segments.size()).clear();
+    segmentInfos.add(start, merge.info);
+    if (lastMergeInfo == null || segmentInfos.indexOf(lastMergeInfo) < start)
+      lastMergeInfo = merge.info;
 
     if (merge.optimize)
       segmentsToOptimize.add(merge.info);
 
+    commitPending = true;
+
     // Must checkpoint before decrefing so any newly
     // referenced files in the new merge.info are incref'd
     // first:
-    deleter.checkpoint(segmentInfos, autoCommit);
+    deleter.checkpoint(segmentInfos, false);
 
     decrefMergeSegments(merge);
 
@@ -2795,8 +2786,13 @@
     } finally {
       synchronized(this) {
         try {
-          if (!success && infoStream != null)
-            message("hit exception during merge");
+          if (!success) {
+            if (infoStream != null)
+              message("hit exception during merge");
+            if (!segmentInfos.contains(merge.info))
+              deleter.refresh(merge.info.name);
+            addMergeException(merge);
+          }
 
           mergeFinish(merge);
 
@@ -2989,6 +2985,14 @@
                                  docStoreIsCompoundFile);
   }
 
+  // nocommit javadoc
+  protected boolean doSyncBeforeMergeCFS(int mergedDocCount) {
+    if (3*mergedDocCount > docCount())
+      return true;
+    else
+      return false;
+  }
+
   /** Does fininishing for a merge, which is fast but holds
    *  the synchronized lock on IndexWriter instance. */
   final synchronized void mergeFinish(MergePolicy.OneMerge merge) throws IOException {
@@ -3026,11 +3030,10 @@
 
     merger = new SegmentMerger(this, mergedName);
 
+    boolean success = false;
+
     // This is try/finally to make sure merger's readers are
     // closed:
-
-    boolean success = false;
-
     try {
       int totDocCount = 0;
 
@@ -3047,6 +3050,7 @@
       if (merge.isAborted())
         throw new IOException("merge is aborted");
 
+      // This is where all the work happens:
       mergedDocCount = merge.info.docCount = merger.merge(merge.mergeDocStores);
 
       assert mergedDocCount == totDocCount;
@@ -3059,14 +3063,6 @@
       if (merger != null) {
         merger.closeReaders();
       }
-      if (!success) {
-        if (infoStream != null)
-          message("hit exception during merge; now refresh deleter on segment " + mergedName);
-        synchronized(this) {
-          addMergeException(merge);
-          deleter.refresh(mergedName);
-        }
-      }
     }
 
     if (!commitMerge(merge))
@@ -3074,6 +3070,11 @@
       return 0;
 
     if (merge.useCompoundFile) {
+
+      // Force a sync here to allow reclaiming of the disk
+      // space used by the segments we just merged:
+      if (autoCommit && doSyncBeforeMergeCFS(mergedDocCount))
+        sync(false, false);
       
       success = false;
       boolean skip = false;
@@ -3106,7 +3107,7 @@
       } finally {
         if (!success) {
           if (infoStream != null)
-            message("hit exception creating compound file during merge: skip=" + skip);
+            message("hit exception creating compound file during merge");
 
           synchronized(this) {
             if (!skip)
@@ -3116,40 +3117,37 @@
         }
       }
 
+      if (merge.isAborted()) {
+        if (infoStream != null)
+          message("abort merge after building CFS");
+        deleter.deleteFile(compoundFileName);
+        return 0;
+      }
+
       if (!skip) {
 
         synchronized(this) {
-          if (skip || segmentInfos.indexOf(merge.info) == -1 || merge.isAborted()) {
+          if (segmentInfos.indexOf(merge.info) == -1 || merge.isAborted()) {
             // Our segment (committed in non-compound
             // format) got merged away while we were
             // building the compound format.
             deleter.deleteFile(compoundFileName);
           } else {
-            success = false;
-            try {
-              merge.info.setUseCompoundFile(true);
-              checkpoint();
-              success = true;
-            } finally {
-              if (!success) {  
-                if (infoStream != null)
-                  message("hit exception checkpointing compound file during merge");
-
-                // Must rollback:
-                addMergeException(merge);
-                merge.info.setUseCompoundFile(false);
-                deletePartialSegmentsFile();
-                deleter.deleteFile(compoundFileName);
-              }
-            }
-      
-            // Give deleter a chance to remove files now.
-            deleter.checkpoint(segmentInfos, autoCommit);
+            merge.info.setUseCompoundFile(true);
+            checkpoint();
           }
         }
       }
     }
 
+    // We force a sync after commiting the merge.  Once this
+    // sync completes then all index files referenced by the
+    // current segmentInfos are on stable storage so if the
+    // OS/machine crashes, or power cord is yanked, the
+    // index will be intact:
+    if (autoCommit)
+      sync(false, false);
+
     return mergedDocCount;
   }
 
@@ -3158,11 +3156,11 @@
       mergeExceptions.add(merge);
   }
 
-  private void deletePartialSegmentsFile() throws IOException  {
-    if (segmentInfos.getLastGeneration() != segmentInfos.getGeneration()) {
+  private void deletePartialSegmentsFile(SegmentInfos infos) throws IOException  {
+    if (infos.getLastGeneration() != infos.getGeneration()) {
       String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                                      "",
-                                                                     segmentInfos.getGeneration());
+                                                                     infos.getGeneration());
       if (infoStream != null)
         message("now delete partial segments file \"" + segmentFileName + "\"");
 
@@ -3307,4 +3305,218 @@
 
     return buffer.toString();
   }
+
+  // nocommit -- pull all of this sync logic into separate
+  // IndexSyncer class??
+  private HashSet synced = new HashSet();
+  private HashSet syncing = new HashSet();
+
+  private boolean startSync(String fileName) {
+    synchronized(synced) {
+      if (!synced.contains(fileName) && !syncing.contains(fileName)) {
+        syncing.add(fileName);
+        return true;
+      } else
+        return false;
+    }
+  }
+
+  private void finishSync(String fileName) {
+    synchronized(synced) {
+      assert syncing.contains(fileName);
+      syncing.remove(fileName);
+      synced.add(fileName);
+      synced.notifyAll();
+    }
+  }
+
+  private void waitForAllSynced(final SegmentInfos infos) throws IOException {
+    synchronized(synced) {
+      for(int i=0;i<infos.size();i++) {
+        final SegmentInfo info = infos.info(i);
+        List files = info.files();
+        for(int j=0;j<files.size();j++) {
+          final String fileName = (String) files.get(j);
+          while(!synced.contains(fileName)) {
+            assert syncing.contains(fileName);
+            try {
+              synced.wait();
+            } catch (InterruptedException ie) {
+              continue;
+            }
+          }
+        }
+      }
+    }
+  }
+
+  private int syncingCount;
+  private boolean stopSyncing;
+
+  // nocommit -- fix javadoc
+  /** Synchronously walk through all files referenced by the
+   *  current segmentInfos and ask the Directory to sync
+   *  each file if it wasn't already.  If that succeeds,
+   *  then we write a new segments_N file & sync that. */
+  private void sync(boolean duringClose, boolean includeFlushes) throws IOException {
+
+    if (infoStream != null)
+      message("start sync() duringClose=" + duringClose + " includeFlushes=" + includeFlushes);
+
+    if (duringClose) {
+      // During close we must take care to ensure the final
+      // changes get committed.  So we must finish any syncs
+      // in progress so that either they are committed, or,
+      // once we fall through, we will commit them:
+      synchronized(this) {
+        stopSyncing = true;
+        while(syncingCount > 0) {
+          try {
+            wait();
+          } catch (InterruptedException ie) {
+            Thread.currentThread().interrupt();
+          }
+        }
+      }
+    }
+
+    // First, we clone & incref the segmentInfos we intend
+    // to sync, then, without locking we sync() each file
+    // referenced by toSync, in the background
+    SegmentInfos toSync = null;
+    final int mySyncCount;
+    synchronized(this) {
+      if (stopSyncing && !duringClose) {
+        if (infoStream != null)
+          message("  skip sync(): stopSyncing");
+        throw new IOException("sync aborted");
+      }
+      if (!commitPending) {
+        if (infoStream != null)
+          message("  skip sync(): no commit pending");
+        return;
+      }
+      commitPending = false;
+      toSync = (SegmentInfos) segmentInfos.clone();
+      if (lastMergeInfo != null && !includeFlushes) {
+        // Do not sync flushes:
+        assert toSync.contains(lastMergeInfo);
+        int downTo = toSync.size()-1;
+        while(!toSync.info(downTo).equals(lastMergeInfo)) {
+          if (infoStream != null)
+            message("  skip segment " + toSync.info(downTo).name);
+          toSync.remove(downTo);
+          downTo--;
+          commitPending = true;
+        }
+      } else if (toSync.size() > 0)
+        lastMergeInfo = toSync.info(toSync.size()-1);
+
+      mySyncCount = syncCount++;
+      if (deleter != null)
+        deleter.incRef(toSync, false);
+      assert syncingCount >= 0;
+      syncingCount++;
+    }
+
+    boolean success0 = false;
+
+    try {
+
+      for(int i=0;i<toSync.size();i++) {
+        final SegmentInfo info = toSync.info(i);
+        List files = info.files();
+        for(int j=0;j<files.size();j++) {
+          final String fileName = (String) files.get(j);
+          // nocommit -- need to put finishSync into finally
+          // clause
+          if (startSync(fileName)) {
+            // Because we incRef'd this commit point, above,
+            // the file had better exist:
+            assert directory.fileExists(fileName);
+            if (infoStream != null)
+              message("now sync " + fileName);
+            directory.sync(fileName);
+            finishSync(fileName);
+          }
+        }
+      }
+
+      // All files that I require are either synced or being
+      // synced by other threads.  If they are being synced,
+      // we must at this point block until they are done:
+      waitForAllSynced(toSync);
+
+      synchronized(this) {
+        // If someone saved a newer version of segments file
+        // since I first started syncing my version, I can
+        // safely skip saving myself
+        if (mySyncCount > syncCountSaved) {
+          
+          if (segmentInfos.getGeneration() > toSync.getGeneration())
+            toSync.updateGeneration(segmentInfos);
+
+          boolean success = false;
+          try {
+            toSync.write(directory);
+            success = true;
+          } finally {
+            if (!success) {
+              commitPending = true;
+              segmentInfos.updateGeneration(toSync);
+              deletePartialSegmentsFile(toSync);
+            }
+          }
+
+          // nocommit: i think we must add checksum to end
+          // of segments_N file.  if we crash right here,
+          // that file could be corrupt.  likely we would
+          // detect such corruption, but not for sure.  with
+          // checksum we can be much more sure.
+
+          // OK, we wrote file successfully, but sync could
+          // still fail and we have to back out the commit if so:
+          success = false;
+          String segmentsFileName = null;
+          try {
+            segmentsFileName = toSync.getCurrentSegmentFileName();
+            if (infoStream != null)
+              message("now sync segments file " + segmentsFileName);
+            directory.sync(segmentsFileName);
+            success = true;
+          } finally {
+            if (!success && segmentsFileName != null && deleter != null) {
+              commitPending = true;
+              segmentInfos.updateGeneration(toSync);
+              deleter.deleteFile(segmentsFileName);
+            }
+          }
+
+          syncCountSaved = mySyncCount;
+
+          segmentInfos.updateGeneration(toSync);
+
+          if (deleter != null)
+            deleter.checkpoint(toSync, true);
+        }
+      }
+
+      if (infoStream != null)
+        message("done all syncs");
+
+      success0 = true;
+
+    } finally {
+      synchronized(this) {
+        if (deleter != null)
+          deleter.decRef(toSync);
+        syncingCount--;
+        assert syncingCount >= 0;
+        if (0 == syncingCount)
+          notifyAll();
+        if (!success0)
+          commitPending = true;
+      }
+    }
+  }
 }
Index: src/java/org/apache/lucene/index/IndexFileDeleter.java
===================================================================
--- src/java/org/apache/lucene/index/IndexFileDeleter.java	(revision 603331)
+++ src/java/org/apache/lucene/index/IndexFileDeleter.java	(working copy)
@@ -32,13 +32,20 @@
 
 /*
  * This class keeps track of each SegmentInfos instance that
- * is still "live", either because it corresponds to a 
- * segments_N file in the Directory (a "commit", i.e. a 
- * committed SegmentInfos) or because it's the in-memory SegmentInfos 
- * that a writer is actively updating but has not yet committed 
- * (currently this only applies when autoCommit=false in IndexWriter).
- * This class uses simple reference counting to map the live
- * SegmentInfos instances to individual files in the Directory. 
+ * is still "live", either because it corresponds to a
+ * segments_N file in the Directory (a "commit", i.e. a
+ * committed SegmentInfos) or because it's the in-memory
+ * SegmentInfos that a writer is actively updating but has
+ * not yet committedThis class uses simple reference
+ * counting to map the live SegmentInfos instances to
+ * individual files in the Directory.
+ *
+ * When autoCommit=true, IndexWriter currently commits only
+ * on completion of a merge (though this may change with
+ * time: it is not a guarantee).  When autoCommit=false,
+ * IndexWriter only commits when it is closed.  Regardless
+ * of autoCommit, the user may call IndexWriter.commit() to
+ * force a blocking commit.
  * 
  * The same directory file may be referenced by more than
  * one IndexCommitPoints, i.e. more than one SegmentInfos.
@@ -260,7 +267,7 @@
       for(int i=0;i<size;i++) {
         CommitPoint commit = (CommitPoint) commitsToDelete.get(i);
         if (infoStream != null) {
-          message("deleteCommits: now remove commit \"" + commit.getSegmentsFileName() + "\"");
+          message("deleteCommits: now decRef commit \"" + commit.getSegmentsFileName() + "\"");
         }
         int size2 = commit.files.size();
         for(int j=0;j<size2;j++) {
@@ -382,13 +389,6 @@
 
     // Incref the files:
     incRef(segmentInfos, isCommit);
-    final List docWriterFiles;
-    if (docWriter != null) {
-      docWriterFiles = docWriter.files();
-      if (docWriterFiles != null)
-        incRef(docWriterFiles);
-    } else
-      docWriterFiles = null;
 
     if (isCommit) {
       // Append to our commits list:
@@ -399,17 +399,24 @@
 
       // Decref files for commits that were deleted by the policy:
       deleteCommits();
-    }
+    } else {
 
-    // DecRef old files from the last checkpoint, if any:
-    int size = lastFiles.size();
-    if (size > 0) {
-      for(int i=0;i<size;i++)
-        decRef((List) lastFiles.get(i));
-      lastFiles.clear();
-    }
+      final List docWriterFiles;
+      if (docWriter != null) {
+        docWriterFiles = docWriter.files();
+        if (docWriterFiles != null)
+          incRef(docWriterFiles);
+      } else
+        docWriterFiles = null;
 
-    if (!isCommit) {
+      // DecRef old files from the last checkpoint, if any:
+      int size = lastFiles.size();
+      if (size > 0) {
+        for(int i=0;i<size;i++)
+          decRef((List) lastFiles.get(i));
+        lastFiles.clear();
+      }
+
       // Save files so we can decr on next checkpoint/commit:
       size = segmentInfos.size();
       for(int i=0;i<size;i++) {
@@ -418,9 +425,9 @@
           lastFiles.add(segmentInfo.files());
         }
       }
+      if (docWriterFiles != null)
+        lastFiles.add(docWriterFiles);
     }
-    if (docWriterFiles != null)
-      lastFiles.add(docWriterFiles);
   }
 
   void incRef(SegmentInfos segmentInfos, boolean isCommit) throws IOException {
@@ -458,7 +465,7 @@
     }
   }
 
-  private void decRef(String fileName) throws IOException {
+  void decRef(String fileName) throws IOException {
     RefCount rc = getRefCount(fileName);
     if (infoStream != null && VERBOSE_REF_COUNTS) {
       message("  DecRef \"" + fileName + "\": pre-decr count is " + rc.count);
Index: src/java/org/apache/lucene/store/Directory.java
===================================================================
--- src/java/org/apache/lucene/store/Directory.java	(revision 603331)
+++ src/java/org/apache/lucene/store/Directory.java	(working copy)
@@ -83,6 +83,9 @@
       Returns a stream writing this file. */
   public abstract IndexOutput createOutput(String name) throws IOException;
 
+  /** Ensure that any writes to this file name are sync'd to
+   *  stable storage. */
+  public void sync(String name) throws IOException {}
 
   /** Returns a stream reading an existing file. */
   public abstract IndexInput openInput(String name)
Index: src/java/org/apache/lucene/store/FSDirectory.java
===================================================================
--- src/java/org/apache/lucene/store/FSDirectory.java	(revision 603331)
+++ src/java/org/apache/lucene/store/FSDirectory.java	(working copy)
@@ -435,6 +435,47 @@
     return new FSIndexOutput(file);
   }
 
+  // nocommit -- remove this (it's only here so
+  // contrib/benchmark can turn it on/off):
+  public static boolean doSync = true;
+
+  public void sync(String name) throws IOException {
+    if (doSync) {
+      File fullFile = new File(directory, name);
+      boolean success = false;
+      int retryCount = 0;
+      IOException exc = null;
+      while(!success && retryCount < 5) {
+        RandomAccessFile file = null;
+        try {
+          try {
+            file = new RandomAccessFile(fullFile, "rw");
+            System.out.println("SYNC " + fullFile);
+            file.getFD().sync();
+            success = true;
+          } finally {
+            if (file != null)
+              file.close();
+          }
+        } catch (IOException ioe) {
+          if (exc == null)
+            exc = ioe;
+          System.out.println("RETRY AFTER SYNC EXC: " + Thread.currentThread().getName() + ": " + ioe);
+          ioe.printStackTrace(System.out);
+          try {
+            // Pause 5 msec
+            Thread.sleep(5);
+          } catch (InterruptedException ie) {
+            Thread.currentThread().interrupt();
+          }
+        }
+      }
+      if (!success)
+        // Throw original exception
+        throw exc;
+    }
+  }
+
   // Inherit javadoc
   public IndexInput openInput(String name) throws IOException {
     return new FSIndexInput(new File(directory, name));
Index: contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
===================================================================
--- contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java	(revision 603331)
+++ contrib/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java	(working copy)
@@ -126,6 +126,8 @@
         FileUtils.fullyDelete(indexDir);
       }
       indexDir.mkdirs();
+      // nocommit -- only for testing
+      FSDirectory.doSync = config.get("fsdirectory.dosync", false);
       directory = FSDirectory.getDirectory(indexDir);
     } else {
       directory = new RAMDirectory();