Index: CHANGES.txt
===================================================================
--- CHANGES.txt	(revision 574266)
+++ CHANGES.txt	(working copy)
@@ -28,6 +28,30 @@
     termText instead of String.  This gives faster tokenization
     performance (~10-15%).  (Mike McCandless)
 
+ 5. LUCENE-847: Factor MergePolicy and MergeScheduler out of
+    IndexWriter.  This enables users to change how & when segments are
+    selected for merging or optimizing (by creating a MergePolicy) as
+    well as how & when the selected merges are actually performed (by
+    creating a MergeScheduler).  Added new public optimize(boolean
+    doWait), close(boolean doWait) and maybeMerge() methods to
+    IndexWriter.  (Steven Parkes via Mike McCandless).
+
+ 6. LUCENE-870: Add a ConcurrentMergeScheduler which executes merges
+    using background threads, up until a max number of threads.  Once
+    there are too many merges, they are performed serially.  This
+    change also improves overall concurrency of IndexWriter for
+    applications that use multiple threads.  For example, optimize is
+    no longer synchronized (and can run in the background), and
+    multiple threads from the application can be running merges at
+    once even when not using ConcurrentMergeScheduler (Steven Parkes
+    via Mike McCandless).
+
+ 7. LUCENE-845: Added a LogByteSizeMergePolicy which selects merges by
+    roughly equal size (in bytes) of the segments in the Directory.
+    This is a better match when you are flushing by RAM usage instead
+    of document count in IndexWriter or when documents vary
+    substantially in size (Mike McCandless).
+
 Bug fixes
 
  1. LUCENE-933: QueryParser fixed to not produce empty sub 
Index: src/test/org/apache/lucene/store/MockRAMOutputStream.java
===================================================================
--- src/test/org/apache/lucene/store/MockRAMOutputStream.java	(revision 574266)
+++ src/test/org/apache/lucene/store/MockRAMOutputStream.java	(working copy)
@@ -55,7 +55,7 @@
     writeBytes(singleByte, 0, 1);
   }
   
-    public void writeBytes(byte[] b, int offset, int len) throws IOException {
+  public void writeBytes(byte[] b, int offset, int len) throws IOException {
     long freeSpace = dir.maxSize - dir.sizeInBytes();
     long realUsage = 0;
 
Index: src/test/org/apache/lucene/store/MockRAMDirectory.java
===================================================================
--- src/test/org/apache/lucene/store/MockRAMDirectory.java	(revision 574266)
+++ src/test/org/apache/lucene/store/MockRAMDirectory.java	(working copy)
@@ -195,7 +195,7 @@
    * RAMOutputStream.BUFFER_SIZE (now 1024) bytes.
    */
 
-  final long getRecomputedActualSizeInBytes() {
+  final synchronized long getRecomputedActualSizeInBytes() {
     long size = 0;
     Iterator it = fileMap.values().iterator();
     while (it.hasNext())
Index: src/test/org/apache/lucene/index/TestThreadedOptimize.java
===================================================================
--- src/test/org/apache/lucene/index/TestThreadedOptimize.java	(revision 0)
+++ src/test/org/apache/lucene/index/TestThreadedOptimize.java	(revision 0)
@@ -0,0 +1,160 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.MockRAMDirectory;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.English;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+import java.io.File;
+
+public class TestThreadedOptimize extends TestCase {
+  
+  private static final Analyzer ANALYZER = new SimpleAnalyzer();
+
+  private final static int NUM_THREADS = 3;
+  //private final static int NUM_THREADS = 5;
+
+  private final static int NUM_ITER = 2;
+  //private final static int NUM_ITER = 10;
+
+  private final static int NUM_ITER2 = 2;
+  //private final static int NUM_ITER2 = 5;
+
+  private boolean failed;
+
+  private void setFailed() {
+    failed = true;
+  }
+
+  public void runTest(Directory directory, boolean autoCommit, MergeScheduler merger) throws Exception {
+
+    IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true);
+    writer.setMaxBufferedDocs(2);
+    if (merger != null)
+      writer.setMergeScheduler(merger);
+
+    for(int iter=0;iter<NUM_ITER;iter++) {
+      final int iterFinal = iter;
+
+      writer.setMergeFactor(1000);
+
+      for(int i=0;i<200;i++) {
+        Document d = new Document();
+        d.add(new Field("id", Integer.toString(i), Field.Store.YES, Field.Index.UN_TOKENIZED));
+        d.add(new Field("contents", English.intToEnglish(i), Field.Store.NO, Field.Index.TOKENIZED));
+        writer.addDocument(d);
+      }
+
+      writer.setMergeFactor(4);
+      //writer.setInfoStream(System.out);
+
+      final int docCount = writer.docCount();
+
+      Thread[] threads = new Thread[NUM_THREADS];
+      
+      for(int i=0;i<NUM_THREADS;i++) {
+        final int iFinal = i;
+        final IndexWriter writerFinal = writer;
+        threads[i] = new Thread() {
+          public void run() {
+            try {
+              for(int j=0;j<NUM_ITER2;j++) {
+                writerFinal.optimize(false);
+                for(int k=0;k<17*(1+iFinal);k++) {
+                  Document d = new Document();
+                  d.add(new Field("id", iterFinal + "_" + iFinal + "_" + j + "_" + k, Field.Store.YES, Field.Index.UN_TOKENIZED));
+                  d.add(new Field("contents", English.intToEnglish(iFinal+k), Field.Store.NO, Field.Index.TOKENIZED));
+                  writerFinal.addDocument(d);
+                }
+                for(int k=0;k<9*(1+iFinal);k++)
+                  writerFinal.deleteDocuments(new Term("id", iterFinal + "_" + iFinal + "_" + j + "_" + k));
+                writerFinal.optimize();
+              }
+            } catch (Throwable t) {
+              setFailed();
+              System.out.println(Thread.currentThread().getName() + ": hit exception");
+              t.printStackTrace(System.out);
+            }
+          }
+        };
+      }
+
+      for(int i=0;i<NUM_THREADS;i++)
+        threads[i].start();
+
+      for(int i=0;i<NUM_THREADS;i++)
+        threads[i].join();
+
+      assertTrue(!failed);
+
+      final int expectedDocCount = (int) ((1+iter)*(200+8*NUM_ITER2*(NUM_THREADS/2.0)*(1+NUM_THREADS)));
+
+      // System.out.println("TEST: now index=" + writer.segString());
+
+      assertEquals(expectedDocCount, writer.docCount());
+
+      if (!autoCommit) {
+        writer.close();
+        writer = new IndexWriter(directory, autoCommit, ANALYZER, false);
+        writer.setMaxBufferedDocs(2);
+      }
+
+      IndexReader reader = IndexReader.open(directory);
+      assertTrue(reader.isOptimized());
+      assertEquals(expectedDocCount, reader.numDocs());
+      reader.close();
+    }
+    writer.close();
+  }
+
+  /*
+    Run above stress test against RAMDirectory and then
+    FSDirectory.
+  */
+  public void testThreadedOptimize() throws Exception {
+    Directory directory = new MockRAMDirectory();
+    runTest(directory, false, null);
+    runTest(directory, true, null);
+    runTest(directory, false, new ConcurrentMergeScheduler());
+    runTest(directory, true, new ConcurrentMergeScheduler());
+    directory.close();
+
+    String tempDir = System.getProperty("tempDir");
+    if (tempDir == null)
+      throw new IOException("tempDir undefined, cannot run test");
+
+    String dirName = tempDir + "/luceneTestThreadedOptimize";
+    directory = FSDirectory.getDirectory(dirName);
+    runTest(directory, false, null);
+    runTest(directory, true, null);
+    runTest(directory, false, new ConcurrentMergeScheduler());
+    runTest(directory, true, new ConcurrentMergeScheduler());
+    directory.close();
+    _TestUtil.rmDir(dirName);
+  }
+}

Property changes on: src/test/org/apache/lucene/index/TestThreadedOptimize.java
___________________________________________________________________
Name: svn:eol-style
   + native

Index: src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
===================================================================
--- src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java	(revision 0)
+++ src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java	(revision 0)
@@ -0,0 +1,231 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.MockRAMDirectory;
+import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.English;
+
+import junit.framework.TestCase;
+
+import java.io.IOException;
+import java.io.File;
+
+public class TestConcurrentMergeScheduler extends TestCase {
+  
+  private static final Analyzer ANALYZER = new SimpleAnalyzer();
+
+  private static class FailOnlyOnFlush extends MockRAMDirectory.Failure {
+    boolean doFail = false;
+
+    public void setDoFail() {
+      this.doFail = true;
+    }
+    public void clearDoFail() {
+      this.doFail = false;
+    }
+
+    public void eval(MockRAMDirectory dir)  throws IOException {
+      if (doFail) {
+        StackTraceElement[] trace = new Exception().getStackTrace();
+        for (int i = 0; i < trace.length; i++) {
+          if ("doFlush".equals(trace[i].getMethodName())) {
+            //new RuntimeException().printStackTrace(System.out);
+            throw new IOException("now failing during flush");
+          }
+        }
+      }
+    }
+  }
+
+  // Make sure running BG merges still work fine even when
+  // we are hitting exceptions during flushing.
+  public void testFlushExceptions() throws IOException {
+
+    MockRAMDirectory directory = new MockRAMDirectory();
+    FailOnlyOnFlush failure = new FailOnlyOnFlush();
+    directory.failOn(failure);
+
+    IndexWriter writer = new IndexWriter(directory, ANALYZER, true);
+    ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+    writer.setMergeScheduler(cms);
+    writer.setMaxBufferedDocs(2);
+    Document doc = new Document();
+    Field idField = new Field("id", "", Field.Store.YES, Field.Index.UN_TOKENIZED);
+    doc.add(idField);
+    for(int i=0;i<10;i++) {
+      for(int j=0;j<20;j++) {
+        idField.setValue(Integer.toString(i*20+j));
+        writer.addDocument(doc);
+      }
+
+      // Even though this won't delete any docs,
+      // IndexWriter's flush will still make a clone for all
+      // SegmentInfos on hitting the exception:
+      writer.deleteDocuments(new Term("id", "1000"));
+      failure.setDoFail();
+      try {
+        writer.flush();
+        fail("failed to hit IOException");
+      } catch (IOException ioe) {
+        failure.clearDoFail();
+      }
+    }
+
+    assertEquals(0, cms.getExceptions().size());
+
+    writer.close();
+    IndexReader reader = IndexReader.open(directory);
+    assertEquals(200, reader.numDocs());
+    reader.close();
+    directory.close();
+  }
+
+  // Test that deletes committed after a merge started and
+  // before it finishes, are correctly merged back:
+  public void testDeleteMerging() throws IOException {
+
+    RAMDirectory directory = new MockRAMDirectory();
+
+    IndexWriter writer = new IndexWriter(directory, ANALYZER, true);
+    ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+    writer.setMergeScheduler(cms);
+
+    // Force degenerate merging so we can get a mix of
+    // merging of segments with and without deletes at the
+    // start:
+    ((LogDocMergePolicy) writer.getMergePolicy()).setMinMergeDocs(1000);
+
+    Document doc = new Document();
+    Field idField = new Field("id", "", Field.Store.YES, Field.Index.UN_TOKENIZED);
+    doc.add(idField);
+    for(int i=0;i<10;i++) {
+      for(int j=0;j<100;j++) {
+        idField.setValue(Integer.toString(i*100+j));
+        writer.addDocument(doc);
+      }
+
+      int delID = i;
+      while(delID < 100*(1+i)) {
+        writer.deleteDocuments(new Term("id", ""+delID));
+        delID += 10;
+      }
+
+      writer.flush();
+    }
+
+    assertEquals(0, cms.getExceptions().size());
+
+    writer.close();
+    IndexReader reader = IndexReader.open(directory);
+    // Verify that we did not lose any deletes...
+    assertEquals(450, reader.numDocs());
+    reader.close();
+    directory.close();
+  }
+
+  public void testNoExtraFiles() throws IOException {
+
+    RAMDirectory directory = new MockRAMDirectory();
+
+    for(int pass=0;pass<2;pass++) {
+
+      boolean autoCommit = pass==0;
+      IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true);
+
+      for(int iter=0;iter<7;iter++) {
+        ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+        writer.setMergeScheduler(cms);
+        writer.setMaxBufferedDocs(2);
+
+        for(int j=0;j<21;j++) {
+          Document doc = new Document();
+          doc.add(new Field("content", "a b c", Field.Store.NO, Field.Index.TOKENIZED));
+          writer.addDocument(doc);
+        }
+        
+        writer.close();
+        TestIndexWriter.assertNoUnreferencedFiles(directory, "testNoExtraFiles autoCommit=" + autoCommit);
+        assertEquals(0, cms.getExceptions().size());
+
+        // Reopen
+        writer = new IndexWriter(directory, autoCommit, ANALYZER, false);
+      }
+
+      writer.close();
+    }
+
+    directory.close();
+  }
+
+  public void testNoWaitClose() throws IOException {
+    RAMDirectory directory = new MockRAMDirectory();
+
+    Document doc = new Document();
+    Field idField = new Field("id", "", Field.Store.YES, Field.Index.UN_TOKENIZED);
+    doc.add(idField);
+
+    for(int pass=0;pass<2;pass++) {
+      boolean autoCommit = pass==0;
+      IndexWriter writer = new IndexWriter(directory, autoCommit, ANALYZER, true);
+
+      for(int iter=0;iter<10;iter++) {
+        ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler();
+        writer.setMergeScheduler(cms);
+        writer.setMaxBufferedDocs(2);
+
+        for(int j=0;j<201;j++) {
+          idField.setValue(Integer.toString(iter*201+j));
+          writer.addDocument(doc);
+        }
+
+        int delID = iter*201;
+        for(int j=0;j<20;j++) {
+          writer.deleteDocuments(new Term("id", Integer.toString(delID)));
+          delID += 5;
+        }
+
+        writer.close(false);
+        assertEquals(0, cms.getExceptions().size());
+
+        IndexReader reader = IndexReader.open(directory);
+        assertEquals((1+iter)*181, reader.numDocs());
+        reader.close();
+
+        // Reopen
+        writer = new IndexWriter(directory, autoCommit, ANALYZER, false);
+      }
+      writer.close();
+    }
+
+    try {
+      directory.close();
+    } catch (RuntimeException ioe) {
+      // MockRAMDirectory will throw IOExceptions when there
+      // are still open files, which is OK since some merge
+      // threads may still be running at this point.
+    }
+  }
+}

Property changes on: src/test/org/apache/lucene/index/TestConcurrentMergeScheduler.java
___________________________________________________________________
Name: svn:eol-style
   + native

Index: src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java
===================================================================
--- src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java	(revision 574266)
+++ src/test/org/apache/lucene/index/TestIndexWriterMergePolicy.java	(working copy)
@@ -24,6 +24,7 @@
 import org.apache.lucene.document.Field;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util._TestUtil;
 
 import junit.framework.TestCase;
 
@@ -73,13 +74,19 @@
     IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
     writer.setMaxBufferedDocs(10);
     writer.setMergeFactor(10);
+    MergePolicy mp = writer.getMergePolicy();
+    if (mp instanceof LogDocMergePolicy)
+      ((LogDocMergePolicy) mp).setMinMergeDocs(100);
 
     for (int i = 0; i < 100; i++) {
       addDoc(writer);
       writer.close();
 
       writer = new IndexWriter(dir, new WhitespaceAnalyzer(), false);
+      mp = writer.getMergePolicy();
       writer.setMaxBufferedDocs(10);
+      if (mp instanceof LogDocMergePolicy)
+        ((LogDocMergePolicy) mp).setMinMergeDocs(100);
       writer.setMergeFactor(10);
       checkInvariants(writer);
     }
@@ -142,6 +149,7 @@
     for (int i = 0; i < 100; i++) {
       addDoc(writer);
     }
+    _TestUtil.syncConcurrentMerges(writer);
     checkInvariants(writer);
 
     for (int i = 100; i < 1000; i++) {
@@ -191,6 +199,7 @@
   }
 
   private void checkInvariants(IndexWriter writer) throws IOException {
+    _TestUtil.syncConcurrentMerges(writer);
     int maxBufferedDocs = writer.getMaxBufferedDocs();
     int mergeFactor = writer.getMergeFactor();
     int maxMergeDocs = writer.getMaxMergeDocs();
Index: src/test/org/apache/lucene/index/DocHelper.java
===================================================================
--- src/test/org/apache/lucene/index/DocHelper.java	(revision 574266)
+++ src/test/org/apache/lucene/index/DocHelper.java	(working copy)
@@ -236,7 +236,7 @@
     //writer.setUseCompoundFile(false);
     writer.addDocument(doc);
     writer.flush();
-    SegmentInfo info = writer.segmentInfos.info(writer.segmentInfos.size()-1);
+    SegmentInfo info = writer.newestSegment();
     writer.close();
     return info;
   }
Index: src/test/org/apache/lucene/index/TestDoc.java
===================================================================
--- src/test/org/apache/lucene/index/TestDoc.java	(revision 574266)
+++ src/test/org/apache/lucene/index/TestDoc.java	(working copy)
@@ -168,7 +168,7 @@
       Document doc = FileDocument.Document(file);
       writer.addDocument(doc);
       writer.flush();
-      return writer.segmentInfos.info(writer.segmentInfos.size()-1);
+      return writer.newestSegment();
    }
 
 
Index: src/test/org/apache/lucene/index/TestIndexWriter.java
===================================================================
--- src/test/org/apache/lucene/index/TestIndexWriter.java	(revision 574266)
+++ src/test/org/apache/lucene/index/TestIndexWriter.java	(working copy)
@@ -39,6 +39,7 @@
 import org.apache.lucene.store.IndexInput;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util._TestUtil;
 
 import org.apache.lucene.store.MockRAMDirectory;
 import org.apache.lucene.store.LockFactory;
@@ -134,7 +135,6 @@
     */
     public void testAddIndexOnDiskFull() throws IOException
     {
-
       int START_COUNT = 57;
       int NUM_DIR = 50;
       int END_COUNT = START_COUNT + NUM_DIR*25;
@@ -200,6 +200,9 @@
 
       for(int iter=0;iter<6;iter++) {
 
+        if (debug)
+          System.out.println("TEST: iter=" + iter);
+
         // Start with 100 bytes more than we are currently using:
         long diskFree = diskUsage+100;
 
@@ -229,7 +232,16 @@
           writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false);
           IOException err = null;
 
+          MergeScheduler ms = writer.getMergeScheduler();
           for(int x=0;x<2;x++) {
+            if (ms instanceof ConcurrentMergeScheduler)
+              // This test intentionally produces exceptions
+              // in the threads that CMS launches; we don't
+              // want to pollute test output with these.
+              if (0 == x)
+                ((ConcurrentMergeScheduler) ms).setSuppressExceptions();
+              else
+                ((ConcurrentMergeScheduler) ms).clearSuppressExceptions();
 
             // Two loops: first time, limit disk space &
             // throw random IOExceptions; second time, no
@@ -301,7 +313,7 @@
               err = e;
               if (debug) {
                 System.out.println("  hit IOException: " + e);
-                // e.printStackTrace(System.out);
+                e.printStackTrace(System.out);
               }
 
               if (1 == x) {
@@ -310,6 +322,10 @@
               }
             }
 
+            // Make sure all threads from
+            // ConcurrentMergeScheduler are done
+            _TestUtil.syncConcurrentMerges(writer);
+
             if (autoCommit) {
 
               // Whether we succeeded or failed, check that
@@ -411,6 +427,12 @@
           }
 
           writer.close();
+
+          // Wait for all BG threads to finish else
+          // dir.close() will throw IOException because
+          // there are still open files
+          _TestUtil.syncConcurrentMerges(ms);
+
           dir.close();
 
           // Try again with 2000 more bytes of free space:
@@ -427,21 +449,38 @@
      */
     public void testAddDocumentOnDiskFull() throws IOException {
 
+      boolean debug = false;
+
       for(int pass=0;pass<3;pass++) {
+        if (debug)
+          System.out.println("TEST: pass=" + pass);
         boolean autoCommit = pass == 0;
         boolean doAbort = pass == 2;
         long diskFree = 200;
         while(true) {
+          if (debug)
+            System.out.println("TEST: cycle: diskFree=" + diskFree);
           MockRAMDirectory dir = new MockRAMDirectory();
           dir.setMaxSizeInBytes(diskFree);
           IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true);
+
+          MergeScheduler ms = writer.getMergeScheduler();
+          if (ms instanceof ConcurrentMergeScheduler)
+            // This test intentionally produces exceptions
+            // in the threads that CMS launches; we don't
+            // want to pollute test output with these.
+            ((ConcurrentMergeScheduler) ms).setSuppressExceptions();
+
           boolean hitError = false;
           try {
             for(int i=0;i<200;i++) {
               addDoc(writer);
             }
           } catch (IOException e) {
-            // e.printStackTrace();
+            if (debug) {
+              System.out.println("TEST: exception on addDoc");
+              e.printStackTrace(System.out);
+            }
             hitError = true;
           }
 
@@ -452,12 +491,17 @@
               try {
                 writer.close();
               } catch (IOException e) {
-                // e.printStackTrace();
+                if (debug) {
+                  System.out.println("TEST: exception on close");
+                  e.printStackTrace(System.out);
+                }
                 dir.setMaxSizeInBytes(0);
                 writer.close();
               }
             }
 
+            _TestUtil.syncConcurrentMerges(ms);
+
             assertNoUnreferencedFiles(dir, "after disk full during addDocument with autoCommit=" + autoCommit);
 
             // Make sure reader can open the index:
@@ -468,15 +512,15 @@
             // Now try again w/ more space:
             diskFree += 500;
           } else {
+            _TestUtil.syncConcurrentMerges(writer);
             dir.close();
             break;
           }
         }
       }
-    
     }                                               
 
-    public void assertNoUnreferencedFiles(Directory dir, String message) throws IOException {
+    public static void assertNoUnreferencedFiles(Directory dir, String message) throws IOException {
       String[] startFiles = dir.list();
       SegmentInfos infos = new SegmentInfos();
       infos.read(dir);
@@ -544,7 +588,7 @@
       dir.close();
     }
 
-    private String arrayToString(String[] l) {
+    static String arrayToString(String[] l) {
       String s = "";
       for(int i=0;i<l.length;i++) {
         if (i > 0) {
@@ -1107,12 +1151,14 @@
       RAMDirectory dir = new RAMDirectory();      
       IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
       writer.setMaxBufferedDocs(10);
+
       int lastNumFile = dir.list().length;
       long lastGen = -1;
       for(int j=1;j<52;j++) {
         Document doc = new Document();
         doc.add(new Field("field", "aaa" + j, Field.Store.YES, Field.Index.TOKENIZED));
         writer.addDocument(doc);
+        _TestUtil.syncConcurrentMerges(writer);
         long gen = SegmentInfos.generationFromSegmentsFileName(SegmentInfos.getCurrentSegmentFileName(dir.list()));
         if (j == 1)
           lastGen = gen;
@@ -1153,7 +1199,6 @@
     public void testDiverseDocs() throws IOException {
       RAMDirectory dir = new RAMDirectory();      
       IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
-      // writer.setInfoStream(System.out);
       long t0 = System.currentTimeMillis();
       writer.setRAMBufferSizeMB(0.5);
       Random rand = new Random(31415);
@@ -1348,6 +1393,48 @@
       assertEquals(2, reader.numDocs());
     }
 
+    // Test calling optimize(false) whereby optimize is kicked
+    // off but we don't wait for it to finish (but
+    // writer.close()) does wait
+    public void testBackgroundOptimize() throws IOException {
+
+      Directory dir = new MockRAMDirectory();
+      for(int pass=0;pass<2;pass++) {
+        IndexWriter writer  = new IndexWriter(dir, new WhitespaceAnalyzer(), true);      
+        writer.setMergeScheduler(new ConcurrentMergeScheduler());
+        Document doc = new Document();
+        doc.add(new Field("field", "aaa", Field.Store.YES, Field.Index.TOKENIZED, Field.TermVector.WITH_POSITIONS_OFFSETS));
+        writer.setMaxBufferedDocs(2);
+        writer.setMergeFactor(101);
+        for(int i=0;i<200;i++)
+          writer.addDocument(doc);
+        writer.optimize(false);
+
+        if (0 == pass) {
+          writer.close();
+          IndexReader reader = IndexReader.open(dir);
+          assertTrue(reader.isOptimized());
+          reader.close();
+        } else {
+          // Get another segment to flush so we can verify it is
+          // NOT included in the optimization
+          writer.addDocument(doc);
+          writer.addDocument(doc);
+          writer.close();
+
+          IndexReader reader = IndexReader.open(dir);
+          assertTrue(!reader.isOptimized());
+          reader.close();
+
+          SegmentInfos infos = new SegmentInfos();
+          infos.read(dir);
+          assertEquals(2, infos.size());
+        }
+      }      
+
+      dir.close();
+    }
+
     private void rmDir(File dir) {
         File[] files = dir.listFiles();
         if (files != null) {
Index: src/test/org/apache/lucene/index/TestStressIndexing.java
===================================================================
--- src/test/org/apache/lucene/index/TestStressIndexing.java	(revision 574266)
+++ src/test/org/apache/lucene/index/TestStressIndexing.java	(working copy)
@@ -32,105 +32,119 @@
 public class TestStressIndexing extends TestCase {
   private static final Analyzer ANALYZER = new SimpleAnalyzer();
   private static final Random RANDOM = new Random();
-  private static Searcher SEARCHER;
 
-  private static int RUN_TIME_SEC = 15;
-
-  private static class IndexerThread extends Thread {
-    IndexWriter modifier;
-    int nextID;
-    public int count;
+  private static abstract class TimedThread extends Thread {
     boolean failed;
+    int count;
+    private static int RUN_TIME_SEC = 6;
+    private TimedThread[] allThreads;
 
-    public IndexerThread(IndexWriter modifier) {
-      this.modifier = modifier;
+    abstract public void doWork() throws Throwable;
+
+    TimedThread(TimedThread[] threads) {
+      this.allThreads = threads;
     }
 
     public void run() {
-      long stopTime = System.currentTimeMillis() + 1000*RUN_TIME_SEC;
-      try {
-        while(true) {
+      final long stopTime = System.currentTimeMillis() + 1000*RUN_TIME_SEC;
 
-          if (System.currentTimeMillis() > stopTime) {
-            break;
-          }
+      count = 0;
 
-          // Add 10 docs:
-          for(int j=0; j<10; j++) {
-            Document d = new Document();
-            int n = RANDOM.nextInt();
-            d.add(new Field("id", Integer.toString(nextID++), Field.Store.YES, Field.Index.UN_TOKENIZED));
-            d.add(new Field("contents", English.intToEnglish(n), Field.Store.NO, Field.Index.TOKENIZED));
-            modifier.addDocument(d);
-          }
-
-          // Delete 5 docs:
-          int deleteID = nextID;
-          for(int j=0; j<5; j++) {
-            modifier.deleteDocuments(new Term("id", ""+deleteID));
-            deleteID -= 2;
-          }
-
+      try {
+        while(System.currentTimeMillis() < stopTime && !anyErrors()) {
+          doWork();
           count++;
         }
-        
-      } catch (Exception e) {
-        System.out.println(e.toString());
-        e.printStackTrace();
+      } catch (Throwable e) {
+        e.printStackTrace(System.out);
         failed = true;
       }
     }
+
+    private boolean anyErrors() {
+      for(int i=0;i<allThreads.length;i++)
+        if (allThreads[i] != null && allThreads[i].failed)
+          return true;
+      return false;
+    }
   }
 
-  private static class SearcherThread extends Thread {
-    private Directory directory;
+  private static class IndexerThread extends TimedThread {
+    IndexWriter writer;
     public int count;
-    boolean failed;
+    int nextID;
 
-    public SearcherThread(Directory directory) {
-      this.directory = directory;
+    public IndexerThread(IndexWriter writer, TimedThread[] threads) {
+      super(threads);
+      this.writer = writer;
     }
 
-    public void run() {
-      long stopTime = System.currentTimeMillis() + 1000*RUN_TIME_SEC;
-      try {
-        while(true) {
-          for (int i=0; i<100; i++) {
-            (new IndexSearcher(directory)).close();
-          }
-          count += 100;
-          if (System.currentTimeMillis() > stopTime) {
-            break;
-          }
-        }
-      } catch (Exception e) {
-        System.out.println(e.toString());
-        e.printStackTrace();
-        failed = true;
+    public void doWork() throws Exception {
+      // Add 10 docs:
+      for(int j=0; j<10; j++) {
+        Document d = new Document();
+        int n = RANDOM.nextInt();
+        d.add(new Field("id", Integer.toString(nextID++), Field.Store.YES, Field.Index.UN_TOKENIZED));
+        d.add(new Field("contents", English.intToEnglish(n), Field.Store.NO, Field.Index.TOKENIZED));
+        writer.addDocument(d);
       }
+
+      // Delete 5 docs:
+      int deleteID = nextID-1;
+      for(int j=0; j<5; j++) {
+        writer.deleteDocuments(new Term("id", ""+deleteID));
+        deleteID -= 2;
+      }
     }
   }
 
+  private static class SearcherThread extends TimedThread {
+    private Directory directory;
+
+    public SearcherThread(Directory directory, TimedThread[] threads) {
+      super(threads);
+      this.directory = directory;
+    }
+
+    public void doWork() throws Throwable {
+      for (int i=0; i<100; i++)
+        (new IndexSearcher(directory)).close();
+      count += 100;
+    }
+  }
+
   /*
     Run one indexer and 2 searchers against single index as
     stress test.
   */
-  public void runStressTest(Directory directory) throws Exception {
-    IndexWriter modifier = new IndexWriter(directory, ANALYZER, true);
+  public void runStressTest(Directory directory, boolean autoCommit, MergeScheduler mergeScheduler) throws Exception {
+    IndexWriter modifier = new IndexWriter(directory, autoCommit, ANALYZER, true);
 
+    modifier.setMaxBufferedDocs(10);
+
+    TimedThread[] threads = new TimedThread[4];
+
+    if (mergeScheduler != null)
+      modifier.setMergeScheduler(mergeScheduler);
+
     // One modifier that writes 10 docs then removes 5, over
     // and over:
-    IndexerThread indexerThread = new IndexerThread(modifier);
+    IndexerThread indexerThread = new IndexerThread(modifier, threads);
+    threads[0] = indexerThread;
     indexerThread.start();
       
-    IndexerThread indexerThread2 = new IndexerThread(modifier);
+    IndexerThread indexerThread2 = new IndexerThread(modifier, threads);
+    threads[2] = indexerThread2;
     indexerThread2.start();
       
-    // Two searchers that constantly just re-instantiate the searcher:
-    SearcherThread searcherThread1 = new SearcherThread(directory);
+    // Two searchers that constantly just re-instantiate the
+    // searcher:
+    SearcherThread searcherThread1 = new SearcherThread(directory, threads);
+    threads[3] = searcherThread1;
     searcherThread1.start();
 
-    SearcherThread searcherThread2 = new SearcherThread(directory);
+    SearcherThread searcherThread2 = new SearcherThread(directory, threads);
+    threads[3] = searcherThread2;
     searcherThread2.start();
 
     indexerThread.join();
@@ -144,6 +158,7 @@
     assertTrue("hit unexpected exception in indexer2", !indexerThread2.failed);
     assertTrue("hit unexpected exception in search1", !searcherThread1.failed);
     assertTrue("hit unexpected exception in search2", !searcherThread2.failed);
+
     //System.out.println("    Writer: " + indexerThread.count + " iterations");
     //System.out.println("Searcher 1: " + searcherThread1.count + " searchers created");
     //System.out.println("Searcher 2: " + searcherThread2.count + " searchers created");
@@ -155,25 +170,38 @@
   */
   public void testStressIndexAndSearching() throws Exception {
 
-    // First in a RAM directory:
+    // RAMDir
     Directory directory = new MockRAMDirectory();
-    runStressTest(directory);
+    runStressTest(directory, true, null);
     directory.close();
 
-    // Second in an FSDirectory:
+    // FSDir
     String tempDir = System.getProperty("java.io.tmpdir");
     File dirPath = new File(tempDir, "lucene.test.stress");
     directory = FSDirectory.getDirectory(dirPath);
-    runStressTest(directory);
+    runStressTest(directory, true, null);
     directory.close();
-    rmDir(dirPath);
-  }
 
-  private void rmDir(File dir) {
-    File[] files = dir.listFiles();
-    for (int i = 0; i < files.length; i++) {
-      files[i].delete();
-    }
-    dir.delete();
+    // With ConcurrentMergeScheduler, in RAMDir
+    directory = new MockRAMDirectory();
+    runStressTest(directory, true, new ConcurrentMergeScheduler());
+    directory.close();
+
+    // With ConcurrentMergeScheduler, in FSDir
+    directory = FSDirectory.getDirectory(dirPath);
+    runStressTest(directory, true, new ConcurrentMergeScheduler());
+    directory.close();
+
+    // With ConcurrentMergeScheduler and autoCommit=false, in RAMDir
+    directory = new MockRAMDirectory();
+    runStressTest(directory, false, new ConcurrentMergeScheduler());
+    directory.close();
+
+    // With ConcurrentMergeScheduler and autoCommit=false, in FSDir
+    directory = FSDirectory.getDirectory(dirPath);
+    runStressTest(directory, false, new ConcurrentMergeScheduler());
+    directory.close();
+
+    _TestUtil.rmDir(dirPath);
   }
 }
Index: src/test/org/apache/lucene/index/TestDocumentWriter.java
===================================================================
--- src/test/org/apache/lucene/index/TestDocumentWriter.java	(revision 574266)
+++ src/test/org/apache/lucene/index/TestDocumentWriter.java	(working copy)
@@ -62,7 +62,7 @@
     IndexWriter writer = new IndexWriter(dir, analyzer, true);
     writer.addDocument(testDoc);
     writer.flush();
-    SegmentInfo info = writer.segmentInfos.info(writer.segmentInfos.size()-1);
+    SegmentInfo info = writer.newestSegment();
     writer.close();
     //After adding the document, we should be able to read it back in
     SegmentReader reader = SegmentReader.get(info);
@@ -123,7 +123,7 @@
 
     writer.addDocument(doc);
     writer.flush();
-    SegmentInfo info = writer.segmentInfos.info(writer.segmentInfos.size()-1);
+    SegmentInfo info = writer.newestSegment();
     writer.close();
     SegmentReader reader = SegmentReader.get(info);
 
@@ -156,7 +156,7 @@
     
     writer.addDocument(doc);
     writer.flush();
-    SegmentInfo info = writer.segmentInfos.info(writer.segmentInfos.size()-1);
+    SegmentInfo info = writer.newestSegment();
     writer.close();
     SegmentReader reader = SegmentReader.get(info);
 
Index: src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java
===================================================================
--- src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java	(revision 574266)
+++ src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java	(working copy)
@@ -272,7 +272,6 @@
 
     writer.addIndexesNoOptimize(new Directory[] { aux, aux });
     assertEquals(1020, writer.docCount());
-    assertEquals(2, writer.getSegmentCount());
     assertEquals(1000, writer.getDocCount(0));
     writer.close();
 
@@ -373,7 +372,7 @@
 
     writer = newWriter(dir, true);
     writer.setMaxBufferedDocs(1000);
-    // add 1000 documents
+    // add 1000 documents in 1 segment
     addDocs(writer, 1000);
     assertEquals(1000, writer.docCount());
     assertEquals(1, writer.getSegmentCount());
Index: src/test/org/apache/lucene/util/_TestUtil.java
===================================================================
--- src/test/org/apache/lucene/util/_TestUtil.java	(revision 574266)
+++ src/test/org/apache/lucene/util/_TestUtil.java	(working copy)
@@ -19,6 +19,9 @@
 
 import java.io.File;
 import java.io.IOException;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MergeScheduler;
+import org.apache.lucene.index.ConcurrentMergeScheduler;
 
 public class _TestUtil {
 
@@ -37,4 +40,15 @@
   public static void rmDir(String dir) throws IOException {
     rmDir(new File(dir));
   }
+
+  public static void syncConcurrentMerges(IndexWriter writer) {
+    MergeScheduler ms = writer.getMergeScheduler();
+    if (ms instanceof ConcurrentMergeScheduler)
+      ((ConcurrentMergeScheduler) ms).sync();
+  }
+
+  public static void syncConcurrentMerges(MergeScheduler ms) {
+    if (ms instanceof ConcurrentMergeScheduler)
+      ((ConcurrentMergeScheduler) ms).sync();
+  }
 }
Index: src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
===================================================================
--- src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java	(revision 0)
+++ src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java	(revision 0)
@@ -0,0 +1,273 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.LinkedList;
+import java.util.ArrayList;
+
+/** A {@link MergeScheduler} that runs each merge using a
+ *  separate thread, up until a maximum number of threads
+ *  ({@link #setMaxThreadCount}) at which points merges are
+ *  run in the foreground, serially.  This is a simple way
+ *  to use concurrency in the indexing process without
+ *  having to create and manage application level
+ *  threads. */
+
+public class ConcurrentMergeScheduler implements MergeScheduler {
+
+  public static boolean VERBOSE = false;
+
+  private int mergeThreadPriority = -1;
+
+  private List mergeThreads = new ArrayList();
+  private int maxThreadCount = 3;
+
+  private List exceptions = new ArrayList();
+  private Directory dir;
+
+  /** Sets the max # simultaneous threads that may be
+   *  running.  If a merge is necessary yet we already have
+   *  this many threads running, the merge is returned back
+   *  to IndexWriter so that it runs in the "foreground". */
+  public void setMaxThreadCount(int count) {
+    if (count < 1)
+      throw new IllegalArgumentException("count should be at least 1");
+    maxThreadCount = count;
+  }
+
+  /** Get the max # simultaneous threads that may be
+   *  running. @see #setMaxThreadCount. */
+  public int getMaxThreadCount() {
+    return maxThreadCount;
+  }
+
+  /** Return the priority that merge threads run at.  By
+   *  default the priority is 1 plus the priority of (ie,
+   *  slightly higher priority than) the first thread that
+   *  calls merge. */
+  public synchronized int getMergeThreadPriority() {
+    initMergeThreadPriority();
+    return mergeThreadPriority;
+  }
+
+  /** Return the priority that merge threads run at. */
+  public synchronized void setMergeThreadPriority(int pri) {
+    mergeThreadPriority = pri;
+
+    final int numThreads = mergeThreads.size();
+    for(int i=0;i<numThreads;i++) {
+      MergeThread merge = (MergeThread) mergeThreads.get(i);
+      try {
+        merge.setPriority(pri);
+      } catch (NullPointerException npe) {
+        // Strangely, Sun's JDK 1.5 on Linux sometimes
+        // throws NPE out of here...
+      }
+    }
+  }
+
+  /** Returns any exceptions that were caught in the merge
+   *  threads. */
+  public List getExceptions() {
+    return exceptions;
+  }
+
+  private void message(String message) {
+    System.out.println("CMS [" + Thread.currentThread().getName() + "]: " + message);
+  }
+
+  private synchronized void initMergeThreadPriority() {
+    if (mergeThreadPriority == -1)
+      // Default to slightly higher priority than our
+      // calling thread
+      mergeThreadPriority = 1+Thread.currentThread().getPriority();
+  }
+
+  public void close() {}
+
+  private synchronized void finishThreads() {
+    while(mergeThreads.size() > 0) {
+      if (VERBOSE) {
+        message("now wait for threads; currently " + mergeThreads.size() + " still running");
+        for(int i=0;i<mergeThreads.size();i++) {
+          final MergeThread mergeThread = ((MergeThread) mergeThreads.get(i));
+          message("    " + i + ": " + mergeThread.merge.segString(dir));
+        }
+      }
+
+      try {
+        wait();
+      } catch (InterruptedException e) {
+      }
+    }
+  }
+
+  public void sync() {
+    finishThreads();
+  }
+
+  // Used for testing
+  private boolean suppressExceptions;
+
+  /** Used for testing */
+  void setSuppressExceptions() {
+    suppressExceptions = true;
+  }
+  void clearSuppressExceptions() {
+    suppressExceptions = false;
+  }
+
+  public void merge(IndexWriter writer)
+    throws CorruptIndexException, IOException {
+
+    initMergeThreadPriority();
+
+    dir = writer.getDirectory();
+
+    // First, quickly run through the newly proposed merges
+    // and add any orthogonal merges (ie a merge not
+    // involving segments already pending to be merged) to
+    // the queue.  If we are way behind on merging, many of
+    // these newly proposed merges will likely already be
+    // registered.
+
+    if (VERBOSE) {
+      message("now merge");
+      message("  index: " + writer.segString());
+    }
+
+    // Iterate, pulling from the IndexWriter's queue of
+    // pending merges, until its empty:
+    while(true) {
+
+      // TODO: we could be careful about which merges to do in
+      // the BG (eg maybe the "biggest" ones) vs FG, which
+      // merges to do first (the easiest ones?), etc.
+
+      MergePolicy.OneMerge merge = writer.getNextMerge();
+      if (merge == null) {
+        if (VERBOSE)
+          message("  no more merges pending; now return");
+        return;
+      }
+
+      // We do this w/ the primary thread to keep
+      // deterministic assignment of segment names
+      writer.mergeInit(merge);
+
+      if (VERBOSE)
+        message("  consider merge " + merge.segString(dir));
+      
+      if (merge.isExternal) {
+        if (VERBOSE)
+          message("    merge involves segments from an external directory; now run in foreground");
+      } else if (mergeThreads.size() < maxThreadCount) {
+        // OK to spawn a new merge thread to handle this
+        // merge:
+        MergeThread merger = new MergeThread(writer, merge);
+        mergeThreads.add(merger);
+        if (VERBOSE)
+          message("    launch new thread [" + merger.getName() + "]");
+        try {
+          merger.setPriority(mergeThreadPriority);
+        } catch (NullPointerException npe) {
+          // Strangely, Sun's JDK 1.5 on Linux sometimes
+          // throws NPE out of here...
+        }
+        merger.start();
+        continue;
+      } else if (VERBOSE)
+        message("    too many merge threads running; run merge in foreground");
+
+      // Too many merge threads already running, so we do
+      // this in the foreground of the calling thread
+      writer.merge(merge);
+    }
+  }
+
+  private class MergeThread extends Thread {
+
+    IndexWriter writer;
+    MergePolicy.OneMerge merge;
+
+    public MergeThread(IndexWriter writer, MergePolicy.OneMerge merge) throws IOException {
+      this.writer = writer;
+      this.merge = merge;
+    }
+
+    public void run() {
+      try {
+
+        if (VERBOSE)
+          message("  merge thread: start");
+
+        // First time through the while loop we do the merge
+        // that we were started with:
+        MergePolicy.OneMerge merge = this.merge;
+
+        while(true) {
+          writer.merge(merge);
+
+          // Subsequent times through the loop we do any new
+          // merge that writer says is necessary:
+          merge = writer.getNextMerge();
+          if (merge != null) {
+            writer.mergeInit(merge);
+            if (VERBOSE)
+              message("  merge thread: do another merge " + merge.segString(dir));
+          } else
+            break;
+        }
+
+        if (VERBOSE)
+          message("  merge thread: done");
+
+      } catch (Throwable exc) {
+        // When a merge was aborted & IndexWriter closed,
+        // it's possible to get various IOExceptions,
+        // NullPointerExceptions, AlreadyClosedExceptions:
+        merge.setException(exc);
+        writer.addMergeException(merge);
+
+        if (!merge.isAborted()) {
+          // If the merge was not aborted then the exception
+          // is real
+          exceptions.add(exc);
+          
+          if (!suppressExceptions)
+            // suppressExceptions is normally only set during
+            // testing.
+            throw new MergePolicy.MergeException(exc);
+        }
+      } finally {
+        synchronized(ConcurrentMergeScheduler.this) {
+          mergeThreads.remove(this);
+          ConcurrentMergeScheduler.this.notifyAll();
+        }
+      }
+    }
+
+    public String toString() {
+      return "merge thread: " + merge.segString(dir);
+    }
+  }
+}

Property changes on: src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
___________________________________________________________________
Name: svn:eol-style
   + native

Index: src/java/org/apache/lucene/index/LogDocMergePolicy.java
===================================================================
--- src/java/org/apache/lucene/index/LogDocMergePolicy.java	(revision 0)
+++ src/java/org/apache/lucene/index/LogDocMergePolicy.java	(revision 0)
@@ -0,0 +1,75 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/** This is a {@link LogMergePolicy} that measures size of a
+ *  segment as the number of documents (not taking deletions
+ *  into account). */
+
+public class LogDocMergePolicy extends LogMergePolicy {
+
+  /** Default minimum segment size.  @see setMinMergeDocs */
+  public static final int DEFAULT_MIN_MERGE_DOCS = 1000;
+
+  /** Default maximum segment size.  A segment of this size
+   *  or larger will never be merged.  @see setMaxMergeDocs */
+  public static final int DEFAULT_MAX_MERGE_DOCS = Integer.MAX_VALUE;
+
+  public LogDocMergePolicy() {
+    super();
+    minMergeSize = DEFAULT_MIN_MERGE_DOCS;
+    maxMergeSize = DEFAULT_MAX_MERGE_DOCS;
+  }
+  protected long size(SegmentInfo info) {
+    return info.docCount;
+  }
+
+  /** Sets the maximum size for a segment to be merged.
+   *  When a segment is this size or larger it will never be
+   *  merged. */
+  public void setMaxMergeDocs(int maxMergeDocs) {
+    maxMergeSize = maxMergeDocs;
+  }
+
+  /** Get the maximum size for a segment to be merged.
+   *  @see #setMaxMergeDocs */
+  public int getMaxMergeDocs() {
+    return (int) maxMergeSize;
+  }
+
+  /** Sets the minimum size for the lowest level segments.
+   * Any segments below this size are considered to be on
+   * the same level (even if they vary drastically in size)
+   * and will be merged whenever there are mergeFactor of
+   * them.  This effectively truncates the "long tail" of
+   * small segments that would otherwise be created into a
+   * single level.  If you set this too large, it could
+   * greatly increase the merging cost during indexing (if
+   * you flush many small segments). */
+  public void setMinMergeDocs(int minMergeDocs) {
+    minMergeSize = minMergeDocs;
+  }
+
+  /** Get the minimum size for a segment to remain
+   *  un-merged.
+   *  @see #setMinMergeDocs **/
+  public int getMinMergeDocs() {
+    return (int) minMergeSize;
+  }
+}
+

Property changes on: src/java/org/apache/lucene/index/LogDocMergePolicy.java
___________________________________________________________________
Name: svn:eol-style
   + native

Index: src/java/org/apache/lucene/index/PositionBasedTermVectorMapper.java
===================================================================
--- src/java/org/apache/lucene/index/PositionBasedTermVectorMapper.java	(revision 574266)
+++ src/java/org/apache/lucene/index/PositionBasedTermVectorMapper.java	(working copy)
@@ -54,7 +54,7 @@
 
   /**
    * Never ignores positions.  This mapper doesn't make much sense unless there are positions
-   * @return
+   * @return false
    */
   public boolean isIgnoringPositions() {
     return false;
Index: src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java
===================================================================
--- src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java	(revision 0)
+++ src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java	(revision 0)
@@ -0,0 +1,75 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+/** This is a {@link LogMergePolicy} that measures size of a
+ *  segment as the total byte size of the segment's files. */
+public class LogByteSizeMergePolicy extends LogMergePolicy {
+
+  /** Default minimum segment size.  @see setMinMergeMB */
+  public static final double DEFAULT_MIN_MERGE_MB = 1.6;
+
+  /** Default maximum segment size.  A segment of this size
+   *  or larger will never be merged.  @see setMaxMergeMB */
+  public static final double DEFAULT_MAX_MERGE_MB = (double) Long.MAX_VALUE;
+
+  public LogByteSizeMergePolicy() {
+    super();
+    minMergeSize = (long) (DEFAULT_MIN_MERGE_MB*1024*1024);
+    maxMergeSize = (long) (DEFAULT_MAX_MERGE_MB*1024*1024);
+  }
+  protected long size(SegmentInfo info) throws IOException {
+    return info.sizeInBytes();
+  }
+
+  /** Sets the maximum size for a segment to be merged.
+   *  When a segment is this size or larger it will never be
+   *  merged. */
+  public void setMaxMergeMB(double mb) {
+    maxMergeSize = (long) (mb*1024*1024);
+  }
+
+  /** Get the maximum size for a segment to be merged.
+   *  @see #setMaxMergeMB */
+  public double getMaxMergeMB() {
+    return ((double) maxMergeSize)/1024/1024;
+  }
+
+  /** Sets the minimum size for the lowest level segments.
+   * Any segments below this size are considered to be on
+   * the same level (even if they vary drastically in size)
+   * and will be merged whenever there are mergeFactor of
+   * them.  This effectively truncates the "long tail" of
+   * small segments that would otherwise be created into a
+   * single level.  If you set this too large, it could
+   * greatly increase the merging cost during indexing (if
+   * you flush many small segments). */
+  public void setMinMergeMB(double mb) {
+    minMergeSize = (long) (mb*1024*1024);
+  }
+
+  /** Get the minimum size for a segment to remain
+   *  un-merged.
+   *  @see #setMinMergeMB **/
+  public double getMinMergeMB() {
+    return ((double) minMergeSize)/1024/1024;
+  }
+}
+

Property changes on: src/java/org/apache/lucene/index/LogByteSizeMergePolicy.java
___________________________________________________________________
Name: svn:eol-style
   + native

Index: src/java/org/apache/lucene/index/SegmentInfo.java
===================================================================
--- src/java/org/apache/lucene/index/SegmentInfo.java	(revision 574266)
+++ src/java/org/apache/lucene/index/SegmentInfo.java	(working copy)
@@ -65,6 +65,8 @@
   private List files;                             // cached list of files that this segment uses
                                                   // in the Directory
 
+  long sizeInBytes = -1;                          // total byte size of all of our files (computed on demand)
+
   private int docStoreOffset;                     // if this segment shares stored fields & vectors, this
                                                   // offset is where in that file this segment's docs begin
   private String docStoreSegment;                 // name used to derive fields/vectors file we share with
@@ -104,7 +106,7 @@
    * Copy everything from src SegmentInfo into our instance.
    */
   void reset(SegmentInfo src) {
-    files = null;
+    clearFiles();
     name = src.name;
     docCount = src.docCount;
     dir = src.dir;
@@ -199,6 +201,19 @@
     }
   }
 
+  /** Returns total size in bytes of all of files used by
+   *  this segment. */
+  long sizeInBytes() throws IOException {
+    if (sizeInBytes == -1) {
+      List files = files();
+      final int size = files.size();
+      sizeInBytes = 0;
+      for(int i=0;i<size;i++) 
+        sizeInBytes += dir.fileLength((String) files.get(i));
+    }
+    return sizeInBytes;
+  }
+
   boolean hasDeletions()
     throws IOException {
     // Cases:
@@ -231,12 +246,12 @@
     } else {
       delGen++;
     }
-    files = null;
+    clearFiles();
   }
 
   void clearDelGen() {
     delGen = NO;
-    files = null;
+    clearFiles();
   }
 
   public Object clone () {
@@ -345,7 +360,7 @@
     } else {
       normGen[fieldIndex]++;
     }
-    files = null;
+    clearFiles();
   }
 
   /**
@@ -392,7 +407,7 @@
     } else {
       this.isCompoundFile = NO;
     }
-    files = null;
+    clearFiles();
   }
 
   /**
@@ -419,7 +434,7 @@
   
   void setDocStoreIsCompoundFile(boolean v) {
     docStoreIsCompoundFile = v;
-    files = null;
+    clearFiles();
   }
   
   String getDocStoreSegment() {
@@ -428,7 +443,7 @@
   
   void setDocStoreOffset(int offset) {
     docStoreOffset = offset;
-    files = null;
+    clearFiles();
   }
   
   /**
@@ -561,4 +576,52 @@
     }
     return files;
   }
+
+  /* Called whenever any change is made that affects which
+   * files this segment has. */
+  private void clearFiles() {
+    files = null;
+    sizeInBytes = -1;
+  }
+
+  /** Used for debugging */
+  public String segString(Directory dir) {
+    String cfs;
+    try {
+      if (getUseCompoundFile())
+        cfs = "c";
+      else
+        cfs = "C";
+    } catch (IOException ioe) {
+      cfs = "?";
+    }
+
+    String docStore;
+
+    if (docStoreOffset != -1)
+      docStore = "->" + docStoreSegment;
+    else
+      docStore = "";
+
+    return name + ":" +
+      cfs +
+      (this.dir == dir ? "" : "x") +
+      docCount + docStore;
+  }
+
+  /** We consider another SegmentInfo instance equal if it
+   *  has the same dir and same name. */
+  public boolean equals(Object obj) {
+    SegmentInfo other;
+    try {
+      other = (SegmentInfo) obj;
+    } catch (ClassCastException cce) {
+      return false;
+    }
+    return other.dir == dir && other.name.equals(name);
+  }
+
+  public int hashCode() {
+    return dir.hashCode() + name.hashCode();
+  }
 }
Index: src/java/org/apache/lucene/index/IndexModifier.java
===================================================================
--- src/java/org/apache/lucene/index/IndexModifier.java	(revision 574266)
+++ src/java/org/apache/lucene/index/IndexModifier.java	(working copy)
@@ -202,6 +202,10 @@
         indexReader = null;
       }
       indexWriter = new IndexWriter(directory, analyzer, false);
+      // IndexModifier cannot use ConcurrentMergeScheduler
+      // because it synchronizes on the directory which can
+      // cause deadlock
+      indexWriter.setMergeScheduler(new SerialMergeScheduler());
       indexWriter.setInfoStream(infoStream);
       indexWriter.setUseCompoundFile(useCompoundFile);
       if (maxBufferedDocs != 0)
Index: src/java/org/apache/lucene/index/MergePolicy.java
===================================================================
--- src/java/org/apache/lucene/index/MergePolicy.java	(revision 0)
+++ src/java/org/apache/lucene/index/MergePolicy.java	(revision 0)
@@ -0,0 +1,215 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.store.Directory;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Set;
+
+/**
+ * <p>Expert: a MergePolicy determines the sequence of
+ * primitive merge operations to be used for overall merge
+ * and optimize operations.</p>
+ * 
+ * <p>Whenever the segments in an index have been altered by
+ * {@link IndexWriter}, either the addition of a newly
+ * flushed segment, addition of many segments from
+ * addIndexes* calls, or a previous merge that may now need
+ * to cascade, {@link IndexWriter} invokes {@link
+ * #findMerges} to give the MergePolicy a chance to pick
+ * merges that are now required.  This method returns a
+ * {@link MergeSpecification} instance describing the set of
+ * merges that should be done, or null if no merges are
+ * necessary.  When IndexWriter.optimize is called, it calls
+ * {@link #findMergesForOptimize} and the MergePolicy should
+ * then return the necessary merges.</p>
+ *
+ * <p>Note that the policy can return more than one merge at
+ * a time.  In this case, if the writer is using {@link
+ * SerialMergeScheduler}, the merges will be run
+ * sequentially but if it is using {@link
+ * ConcurrentMergeScheduler} they will be run concurrently.</p>
+ * 
+ * <p>The default MergePolicy is {@link
+ * LogByteSizeMergePolicy}.</p>
+ */
+
+public interface MergePolicy {
+
+  /** OneMerge provides the information necessary to perform
+   *  an individual primitive merge operation, resulting in
+   *  a single new segment.  The merge spec includes the
+   *  subset of segments to be merged as well as whether the
+   *  new segment should use the compound file format. */
+
+  public static class OneMerge {
+
+    SegmentInfo info;               // used by IndexWriter
+    boolean mergeDocStores;         // used by IndexWriter
+    boolean optimize;               // used by IndexWriter
+    SegmentInfos segmentsClone;     // used by IndexWriter
+    boolean increfDone;             // used by IndexWriter
+    boolean registerDone;           // used by IndexWriter
+    long mergeGen;                  // used by IndexWriter
+    boolean isExternal;             // used by IndexWriter
+
+    final SegmentInfos segments;
+    final boolean useCompoundFile;
+    boolean aborted;
+    Throwable error;
+
+    public OneMerge(SegmentInfos segments, boolean useCompoundFile) {
+      if (0 == segments.size())
+        throw new RuntimeException("segments must include at least one segment");
+      this.segments = segments;
+      this.useCompoundFile = useCompoundFile;
+    }
+
+    /** Record that an exception occurred while executing
+     *  this merge */
+    public synchronized void setException(Throwable error) {
+      this.error = error;
+    }
+
+    /** Retrieve previous exception set by {@link
+     *  #setException}. */
+    public synchronized Throwable getException() {
+      return error;
+    }
+
+    /** Mark this merge as aborted.  If this is called
+     *  before the merge is committed then the merge will
+     *  not be committed. */
+    public synchronized void abort() {
+      aborted = true;
+    }
+
+    /** Returns true if this merge was aborted. */
+    public synchronized boolean isAborted() {
+      return aborted;
+    }
+
+    public String segString(Directory dir) {
+      StringBuffer b = new StringBuffer();
+      final int numSegments = segments.size();
+      for(int i=0;i<numSegments;i++) {
+        if (i > 0) b.append(" ");
+        b.append(segments.info(i).segString(dir));
+      }
+      if (info != null)
+        b.append(" into " + info.name);
+      if (optimize)
+        b.append(" [optimize]");
+      return b.toString();
+    }
+  }
+
+  /**
+   * A MergeSpecification instance provides the information
+   * necessary to perform multiple merges.  It simply
+   * contains a list of {@link OneMerge} instances.
+   */
+
+  public static class MergeSpecification implements Cloneable {
+
+    /**
+     * The subset of segments to be included in the primitive merge.
+     */
+
+    public List merges = new ArrayList();
+
+    public void add(OneMerge merge) {
+      merges.add(merge);
+    }
+
+    public String segString(Directory dir) {
+      StringBuffer b = new StringBuffer();
+      b.append("MergeSpec:\n");
+      final int count = merges.size();
+      for(int i=0;i<count;i++)
+        b.append("  " + (1+i) + ": " + ((OneMerge) merges.get(i)).segString(dir));
+      return b.toString();
+    }
+  }
+
+  /** Exception thrown if there are any problems while
+   *  executing a merge. */
+  public class MergeException extends RuntimeException {
+    public MergeException(String message) {
+      super(message);
+    }
+    public MergeException(Throwable exc) {
+      super(exc);
+    }
+  }
+
+  /**
+   * Determine what set of merge operations are now
+   * necessary on the index.  The IndexWriter calls this
+   * whenever there is a change to the segments.  This call
+   * is always synchronized on the IndexWriter instance so
+   * only one thread at a time will call this method.
+   *
+   * @param segmentInfos the total set of segments in the index
+   * @param writer IndexWriter instance
+   */
+  MergeSpecification findMerges(SegmentInfos segmentInfos,
+                                IndexWriter writer)
+     throws CorruptIndexException, IOException;
+
+  /**
+   * Determine what set of merge operations are necessary in
+   * order to optimize the index.  The IndexWriter calls
+   * this when its optimize() method is called.  This call
+   * is always synchronized on the IndexWriter instance so
+   * only one thread at a time will call this method.
+   *
+   * @param segmentInfos the total set of segments in the index
+   * @param writer IndexWriter instance
+   * @param maxSegmentCount requested maximum number of
+   *   segments in the index (currently this is always 1)
+   * @param segmentsToOptimize contains the specific
+   *   SegmentInfo instances that must be merged away.  This
+   *   may be a subset of all SegmentInfos.
+   */
+  MergeSpecification findMergesForOptimize(SegmentInfos segmentInfos,
+                                           IndexWriter writer,
+                                           int maxSegmentCount,
+                                           Set segmentsToOptimize)
+     throws CorruptIndexException, IOException;
+
+  /**
+   * Release all resources for the policy.
+   */
+  void close();
+
+  /**
+   * Returns true if a newly flushed (not from merge)
+   * segment should use the compound file format.
+   */
+  boolean useCompoundFile(SegmentInfos segments, SegmentInfo newSegment);
+
+  /**
+   * Returns true if the doc store files should use the
+   * compound file format.
+   */
+  boolean useCompoundDocStore(SegmentInfos segments);
+}

Property changes on: src/java/org/apache/lucene/index/MergePolicy.java
___________________________________________________________________
Name: svn:eol-style
   + native

Index: src/java/org/apache/lucene/index/LogMergePolicy.java
===================================================================
--- src/java/org/apache/lucene/index/LogMergePolicy.java	(revision 0)
+++ src/java/org/apache/lucene/index/LogMergePolicy.java	(revision 0)
@@ -0,0 +1,294 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Set;
+
+import org.apache.lucene.store.Directory;
+
+/** <p>This class implements a {@link MergePolicy} that tries
+ *  to merge segments into levels of exponentially
+ *  increasing size, where each level has < mergeFactor
+ *  segments in it.  Whenever a given levle has mergeFactor
+ *  segments or more in it, they will be merged.</p>
+ *
+ * <p>This class is abstract and requires a subclass to
+ * define the {@link #size} method which specifies how a
+ * segment's size is determined.  {@link LogDocMergePolicy}
+ * is one subclass that measures size by document count in
+ * the segment.  {@link LogByteSizeMergePolicy} is another
+ * subclass that measures size as the total byte size of the
+ * file(s) for the segment.</p>
+ */
+
+public abstract class LogMergePolicy implements MergePolicy {
+
+  /** Defines the allowed range of log(size) for each
+   *  level.  A level is computed by taking the max segment
+   *  log size, minuse LEVEL_LOG_SPAN, and finding all
+   *  segments falling within that range. */
+  public static final double LEVEL_LOG_SPAN = 0.75;
+
+  /** Default merge factor, which is how many segments are
+   *  merged at a time */
+  public static final int DEFAULT_MERGE_FACTOR = 10;
+
+  private int mergeFactor = DEFAULT_MERGE_FACTOR;
+
+  long minMergeSize;
+  long maxMergeSize;
+
+  private boolean useCompoundFile = true;
+  private boolean useCompoundDocStore = true;
+
+  /** <p>Returns the number of segments that are merged at
+   * once and also controls the total number of segments
+   * allowed to accumulate in the index.</p> */
+  public int getMergeFactor() {
+    return mergeFactor;
+  }
+
+  /** Determines how often segment indices are merged by
+   * addDocument().  With smaller values, less RAM is used
+   * while indexing, and searches on unoptimized indices are
+   * faster, but indexing speed is slower.  With larger
+   * values, more RAM is used during indexing, and while
+   * searches on unoptimized indices are slower, indexing is
+   * faster.  Thus larger values (> 10) are best for batch
+   * index creation, and smaller values (< 10) for indices
+   * that are interactively maintained. */
+  public void setMergeFactor(int mergeFactor) {
+    if (mergeFactor < 2)
+      throw new IllegalArgumentException("mergeFactor cannot be less than 2");
+    this.mergeFactor = mergeFactor;
+  }
+
+  // Javadoc inherited
+  public boolean useCompoundFile(SegmentInfos infos, SegmentInfo info) {
+    return useCompoundFile;
+  }
+
+  /** Sets whether compound file format should be used for
+   *  newly flushed and newly merged segments. */
+  public void setUseCompoundFile(boolean useCompoundFile) {
+    this.useCompoundFile = useCompoundFile;
+  }
+
+  /** Returns true if newly flushed and newly merge segments
+   *  are written in compound file format. @see
+   *  #setUseCompoundFile */
+  public boolean getUseCompoundFile() {
+    return useCompoundFile;
+  }
+
+  // Javadoc inherited
+  public boolean useCompoundDocStore(SegmentInfos infos) {
+    return useCompoundDocStore;
+  }
+
+  /** Sets whether compound file format should be used for
+   *  newly flushed and newly merged doc store
+   *  segment files (term vectors and stored fields). */
+  public void setUseCompoundDocStore(boolean useCompoundDocStore) {
+    this.useCompoundDocStore = useCompoundDocStore;
+  }
+
+  /** Returns true if newly flushed and newly merge doc
+   *  store segment files (term vectors and stored fields)
+   *  are written in compound file format. @see
+   *  #setUseCompoundDocStore */
+  public boolean getUseCompoundDocStore() {
+    return useCompoundDocStore;
+  }
+
+  public void close() {}
+
+  abstract protected long size(SegmentInfo info) throws IOException;
+
+  private boolean isOptimized(SegmentInfos infos, IndexWriter writer, int maxNumSegments, Set segmentsToOptimize) throws IOException {
+    final int numSegments = infos.size();
+    int numToOptimize = 0;
+    SegmentInfo optimizeInfo = null;
+    for(int i=0;i<numSegments && numToOptimize <= maxNumSegments;i++) {
+      final SegmentInfo info = infos.info(i);
+      if (segmentsToOptimize.contains(info)) {
+        numToOptimize++;
+        optimizeInfo = info;
+      }
+    }
+
+    return !(numToOptimize > maxNumSegments ||
+             (numToOptimize == 1 &&
+              (optimizeInfo.hasDeletions() ||
+               optimizeInfo.hasSeparateNorms() ||
+               optimizeInfo.dir != writer.getDirectory() ||
+               optimizeInfo.getUseCompoundFile() != useCompoundFile)));
+  }
+
+  /** Returns the merges necessary to optimize the index.
+   *  This merge policy defines "optimized" to mean only one
+   *  segment in the index, where that segment has no
+   *  deletions pending nor separate norms, and it is in
+   *  compound file format if the current useCompoundFile
+   *  setting is true.  This method returns multiple merges
+   *  (mergeFactor at a time) so the {@link MergeScheduler}
+   *  in use may make use of concurrency. */
+  public MergeSpecification findMergesForOptimize(SegmentInfos infos, IndexWriter writer, int maxNumSegments, Set segmentsToOptimize) throws IOException {
+    final Directory dir = writer.getDirectory();
+    MergeSpecification spec;
+    
+    if (!isOptimized(infos, writer, maxNumSegments, segmentsToOptimize)) {
+
+      int numSegments = infos.size();
+      while(numSegments > 0) {
+        final SegmentInfo info = infos.info(--numSegments);
+        if (segmentsToOptimize.contains(info)) {
+          numSegments++;
+          break;
+        }
+      }
+
+      if (numSegments > 0) {
+
+        spec = new MergeSpecification();
+        while (numSegments > 0) {
+        
+          final int first;
+          if (numSegments > mergeFactor)
+            first = numSegments-mergeFactor;
+          else
+            first = 0;
+          spec.add(new OneMerge(infos.range(first, numSegments), useCompoundFile));
+
+          numSegments -= mergeFactor;
+        }
+
+      } else
+        spec = null;
+    } else
+      spec = null;
+
+    return spec;
+  }
+
+  /** Checks if any merges are now necessary and returns a
+   *  {@link MergePolicy.MergeSpecification} if so.  A merge
+   *  is necessary when there are more than {@link
+   *  #setMergeFactor} segments at a given level.  When
+   *  multiple levels have too many segments, this method
+   *  will return multiple merges, allowing the {@link
+   *  MergeScheduler} to use concurrency. */
+  public MergeSpecification findMerges(SegmentInfos infos, IndexWriter writer) throws IOException {
+
+    final int numSegments = infos.size();
+
+    // Compute levels, which is just log (base mergeFactor)
+    // of the size of each segment
+    float[] levels = new float[numSegments];
+    final float norm = (float) Math.log(mergeFactor);
+
+    final Directory directory = writer.getDirectory();
+
+    for(int i=0;i<numSegments;i++) {
+      final SegmentInfo info = infos.info(i);
+      long size = size(info);
+
+      // Refuse to import a segment that's too large
+      if (size >= maxMergeSize && info.dir != directory)
+        throw new IllegalArgumentException("Segment is too large (" + size + " vs max size " + maxMergeSize + ")");
+
+      // Floor tiny segments
+      if (size < 1)
+        size = 1;
+      levels[i] = (float) Math.log(size)/norm;
+    }
+
+    final float levelFloor;
+    if (minMergeSize <= 0)
+      levelFloor = (float) 0.0;
+    else
+      levelFloor = (float) (Math.log(minMergeSize)/norm);
+
+    // Now, we quantize the log values into levels.  The
+    // first level is any segment whose log size is within
+    // LEVEL_LOG_SPAN of the max size, or, who has such as
+    // segment "to the right".  Then, we find the max of all
+    // other segments and use that to define the next level
+    // segment, etc.
+
+    MergeSpecification spec = null;
+
+    int start = 0;
+    while(start < numSegments) {
+
+      // Find max level of all segments not already
+      // quantized.
+      float maxLevel = levels[start];
+      for(int i=1+start;i<numSegments;i++) {
+        final float level = levels[i];
+        if (level > maxLevel)
+          maxLevel = level;
+      }
+
+      // Now search backwards for the rightmost segment that
+      // falls into this level:
+      float levelBottom;
+      if (maxLevel < levelFloor)
+        // All remaining segments fall into the min level
+        levelBottom = -1.0F;
+      else {
+        levelBottom = (float) (maxLevel - LEVEL_LOG_SPAN);
+
+        // Force a boundary at the level floor
+        if (levelBottom < levelFloor && maxLevel >= levelFloor)
+          levelBottom = levelFloor;
+      }
+
+      int upto = numSegments-1;
+      while(upto >= start) {
+        if (levels[upto] >= levelBottom) {
+          break;
+        }
+        upto--;
+      }
+
+      // Finally, record all merges that are viable at this level:
+      int end = start + mergeFactor;
+      while(end <= 1+upto) {
+        boolean anyTooLarge = false;
+        for(int i=start;i<end;i++)
+          anyTooLarge |= size(infos.info(i)) >= maxMergeSize;
+
+        if (!anyTooLarge) {
+          if (spec == null)
+            spec = new MergeSpecification();
+          spec.add(new OneMerge(infos.range(start, end), useCompoundFile));
+        }
+        start = end;
+        end = start + mergeFactor;
+      }
+
+      start = 1+upto;
+    }
+
+    return spec;
+  }
+}

Property changes on: src/java/org/apache/lucene/index/LogMergePolicy.java
___________________________________________________________________
Name: svn:eol-style
   + native

Index: src/java/org/apache/lucene/index/DocumentsWriter.java
===================================================================
--- src/java/org/apache/lucene/index/DocumentsWriter.java	(revision 574266)
+++ src/java/org/apache/lucene/index/DocumentsWriter.java	(working copy)
@@ -113,6 +113,7 @@
 
   private int nextDocID;                          // Next docID to be added
   private int numDocsInRAM;                       // # docs buffered in RAM
+  private int numDocsInStore;                     // # docs written to doc stores
   private int nextWriteDocID;                     // Next docID to be written
 
   // Max # ThreadState instances; if there are more threads
@@ -238,6 +239,7 @@
       String s = docStoreSegment;
       docStoreSegment = null;
       docStoreOffset = 0;
+      numDocsInStore = 0;
       return s;
     } else {
       return null;
@@ -245,7 +247,12 @@
   }
 
   private List files = null;                      // Cached list of files we've created
+  private List abortedFiles = null;               // List of files that were written before last abort()
 
+  List abortedFiles() {
+    return abortedFiles;
+  }
+
   /* Returns list of files in use by this instance,
    * including any flushed segments. */
   List files() {
@@ -278,6 +285,9 @@
    *  docs added since last flush. */
   synchronized void abort() throws IOException {
 
+    if (infoStream != null)
+      infoStream.println("docWriter: now abort");
+
     // Forcefully remove waiting ThreadStates from line
     for(int i=0;i<numWaiting;i++)
       waitingThreadStates[i].isIdle = true;
@@ -290,6 +300,8 @@
 
     try {
 
+      abortedFiles = files();
+
       // Discard pending norms:
       final int numField = fieldInfos.size();
       for (int i=0;i<numField;i++) {
@@ -332,6 +344,7 @@
       }
 
       files = null;
+
     } finally {
       resumeAllThreads();
     }
@@ -398,7 +411,7 @@
 
     newFiles = new ArrayList();
 
-    docStoreOffset += numDocsInRAM;
+    docStoreOffset = numDocsInStore;
 
     if (closeDocStore) {
       assert docStoreSegment != null;
@@ -2119,6 +2132,7 @@
       segment = writer.newSegmentName();
 
     numDocsInRAM++;
+    numDocsInStore++;
 
     // We must at this point commit to flushing to ensure we
     // always get N docs when we flush by doc count, even if
Index: src/java/org/apache/lucene/index/SerialMergeScheduler.java
===================================================================
--- src/java/org/apache/lucene/index/SerialMergeScheduler.java	(revision 0)
+++ src/java/org/apache/lucene/index/SerialMergeScheduler.java	(revision 0)
@@ -0,0 +1,46 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.LinkedList;
+
+/** A {@link MergeScheduler} that simply does each merge
+ *  sequentially, using the current thread. */
+public class SerialMergeScheduler implements MergeScheduler {
+
+  /** Even though this is a serial merge policy, it's still
+   * allowed that multiple threads call merge in the case
+   * where the application is using multiple threads to
+   * add/delete/update documents.  In this case we want to
+   * allow concurrency, and we do this by pulling from a
+   * single queue of pending merges. */
+
+  public void merge(IndexWriter writer)
+    throws CorruptIndexException, IOException {
+
+    while(true) {
+      MergePolicy.OneMerge merge = writer.getNextMerge();
+      if (merge == null)
+        break;
+      writer.merge(merge);
+    }
+  }
+
+  public void close() {}
+}

Property changes on: src/java/org/apache/lucene/index/SerialMergeScheduler.java
___________________________________________________________________
Name: svn:eol-style
   + native

Index: src/java/org/apache/lucene/index/SegmentInfos.java
===================================================================
--- src/java/org/apache/lucene/index/SegmentInfos.java	(revision 574266)
+++ src/java/org/apache/lucene/index/SegmentInfos.java	(working copy)
@@ -330,6 +330,9 @@
   public long getGeneration() {
     return generation;
   }
+  public long getLastGeneration() {
+    return lastGeneration;
+  }
 
   /**
    * Current version number from segments file.
@@ -661,4 +664,16 @@
      */
     protected abstract Object doBody(String segmentFileName) throws CorruptIndexException, IOException;
   }
+
+  /**
+   * Returns a new SegmentInfos containg the SegmentInfo
+   * instances in the specified range first (inclusive) to
+   * last (exclusive), so total number of segments returned
+   * is last-first.
+   */
+  public SegmentInfos range(int first, int last) {
+    SegmentInfos infos = new SegmentInfos();
+    infos.addAll(super.subList(first, last));
+    return infos;
+  }
 }
Index: src/java/org/apache/lucene/index/IndexWriter.java
===================================================================
--- src/java/org/apache/lucene/index/IndexWriter.java	(revision 574266)
+++ src/java/org/apache/lucene/index/IndexWriter.java	(working copy)
@@ -25,13 +25,19 @@
 import org.apache.lucene.store.Lock;
 import org.apache.lucene.store.LockObtainFailedException;
 import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util.BitVector;
 
 import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.util.List;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.LinkedList;
 import java.util.Iterator;
+import java.util.ListIterator;
 import java.util.Map.Entry;
 
 /**
@@ -71,7 +77,10 @@
   a large RAM buffer.  You can also force a flush by calling
   {@link #flush}.  When a flush occurs, both pending deletes
   and added documents are flushed to the index.  A flush may
-  also trigger one or more segment merges.</p>
+  also trigger one or more segment merges which by default
+  run (blocking) with the current thread (see <a
+  href="#mergePolicy">below</a> for changing the {@link
+  MergeScheduler}).</p>
 
   <a name="autoCommit"></a>
   <p>The optional <code>autoCommit</code> argument to the
@@ -135,8 +144,21 @@
   filesystems like NFS that do not support "delete on last
   close" semantics, which Lucene's "point in time" search
   normally relies on. </p>
-  */
 
+  <a name="mergePolicy"></a> <p>Expert:
+  <code>IndexWriter</code> allows you to separately change
+  the {@link MergePolicy} and the {@link MergeScheduler}.
+  The {@link MergePolicy} is invoked whenever there are
+  changes to the segments in the index.  Its role is to
+  select which merges to do, if any, and return a {@link
+  MergePolicy.MergeSpecification} describing the merges.  It
+  also selects merges to do for optimize().  (The default is
+  {@link LogDocMergePolicy}.  Then, the {@link
+  MergeScheduler} is invoked with the requested merges and
+  it decides when and how to run the merges.  The default is
+  {@link SerialMergeScheduler}. </p>
+*/
+
 /*
  * Clarification: Check Points (and commits)
  * Being able to set autoCommit=false allows IndexWriter to flush and 
@@ -177,9 +199,10 @@
   public static final String WRITE_LOCK_NAME = "write.lock";
 
   /**
-   * Default value is 10. Change using {@link #setMergeFactor(int)}.
+   * @deprecated
+   * @see LogMergePolicy#DEFAULT_MERGE_FACTOR
    */
-  public final static int DEFAULT_MERGE_FACTOR = 10;
+  public final static int DEFAULT_MERGE_FACTOR = LogMergePolicy.DEFAULT_MERGE_FACTOR;
 
   /**
    * Default value is 10. Change using {@link #setMaxBufferedDocs(int)}.
@@ -205,9 +228,10 @@
   public final static int DEFAULT_MAX_BUFFERED_DELETE_TERMS = 1000;
 
   /**
-   * Default value is {@link Integer#MAX_VALUE}. Change using {@link #setMaxMergeDocs(int)}.
+   * @deprecated
+   * @see LogDocMergePolicy#DEFAULT_MAX_MERGE_DOCS
    */
-  public final static int DEFAULT_MAX_MERGE_DOCS = Integer.MAX_VALUE;
+  public final static int DEFAULT_MAX_MERGE_DOCS = LogDocMergePolicy.DEFAULT_MAX_MERGE_DOCS;
 
   /**
    * Default value is 10,000. Change using {@link #setMaxFieldLength(int)}.
@@ -239,23 +263,31 @@
   private boolean localAutoCommit;                // saved autoCommit during local transaction
   private boolean autoCommit = true;              // false if we should commit only on close
 
-  SegmentInfos segmentInfos = new SegmentInfos();       // the segments
+  private SegmentInfos segmentInfos = new SegmentInfos();       // the segments
   private DocumentsWriter docWriter;
   private IndexFileDeleter deleter;
 
+  private Set segmentsToOptimize = new HashSet();           // used by optimize to note those needing optimization
+
   private Lock writeLock;
 
   private int termIndexInterval = DEFAULT_TERM_INDEX_INTERVAL;
 
-  /** Use compound file setting. Defaults to true, minimizing the number of
-   * files used.  Setting this to false may improve indexing performance, but
-   * may also cause file handle problems.
-   */
-  private boolean useCompoundFile = true;
-
   private boolean closeDir;
   private boolean closed;
+  private boolean closing;
 
+  // Holds all SegmentInfo instances currently involved in
+  // merges
+  private HashSet mergingSegments = new HashSet();
+
+  private MergePolicy mergePolicy = new LogDocMergePolicy();
+  private MergeScheduler mergeScheduler = new SerialMergeScheduler();
+  private LinkedList pendingMerges = new LinkedList();
+  private Set runningMerges = new HashSet();
+  private List mergeExceptions = new ArrayList();
+  private long mergeGen;
+
   /**
    * Used internally to throw an {@link
    * AlreadyClosedException} if this IndexWriter has been
@@ -268,23 +300,57 @@
     }
   }
 
-  /** Get the current setting of whether to use the compound file format.
-   *  Note that this just returns the value you set with setUseCompoundFile(boolean)
-   *  or the default. You cannot use this to query the status of an existing index.
+  private void message(String message) {
+    infoStream.println("IW [" + Thread.currentThread().getName() + "]: " + message);
+  }
+
+  /**
+   * Casts current mergePolicy to LogMergePolicy, and throws
+   * an exception if the mergePolicy is not a LogMergePolicy.
+   */
+  private LogMergePolicy getLogMergePolicy() {
+    if (mergePolicy instanceof LogMergePolicy)
+      return (LogMergePolicy) mergePolicy;
+    else
+      throw new IllegalArgumentException("this method can only be called when the merge policy is the default LogMergePolicy");
+  }
+
+  private LogDocMergePolicy getLogDocMergePolicy() {
+    if (mergePolicy instanceof LogDocMergePolicy)
+      return (LogDocMergePolicy) mergePolicy;
+    else
+      throw new IllegalArgumentException("this method can only be called when the merge policy is LogDocMergePolicy");
+  }
+
+  /** <p>Get the current setting of whether newly flushed
+   *  segments will use the compound file format.  Note that
+   *  this just returns the value previously set with
+   *  setUseCompoundFile(boolean), or the default value
+   *  (true).  You cannot use this to query the status of
+   *  previously flushed segments.</p>
+   *
+   *  <p>Note that this method is a convenience method: it
+   *  just calls mergePolicy.getUseCompoundFile as long as
+   *  mergePolicy is an instance of {@link LogMergePolicy}.
+   *  Otherwise an IllegalArgumentException is thrown.</p>
+   *
    *  @see #setUseCompoundFile(boolean)
    */
   public boolean getUseCompoundFile() {
-    ensureOpen();
-    return useCompoundFile;
+    return getLogMergePolicy().getUseCompoundFile();
   }
 
-  /** Setting to turn on usage of a compound file. When on, multiple files
-   *  for each segment are merged into a single file once the segment creation
-   *  is finished. This is done regardless of what directory is in use.
+  /** <p>Setting to turn on usage of a compound file. When on,
+   *  multiple files for each segment are merged into a
+   *  single file when a new segment is flushed.</p>
+   *
+   *  <p>Note that this method is a convenience method: it
+   *  just calls mergePolicy.setUseCompoundFile as long as
+   *  mergePolicy is an instance of {@link LogMergePolicy}.
+   *  Otherwise an IllegalArgumentException is thrown.</p>
    */
   public void setUseCompoundFile(boolean value) {
-    ensureOpen();
-    useCompoundFile = value;
+    getLogMergePolicy().setUseCompoundFile(value);
   }
 
   /** Expert: Set the Similarity implementation used by this IndexWriter.
@@ -635,6 +701,8 @@
                                      deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy,
                                      segmentInfos, infoStream, docWriter);
 
+      pushMaxBufferedDocs();
+
     } catch (IOException e) {
       this.writeLock.release();
       this.writeLock = null;
@@ -642,26 +710,83 @@
     }
   }
 
+  /**
+   * Expert: set the merge policy used by this writer.
+   */
+  public void setMergePolicy(MergePolicy mp) {
+    ensureOpen();
+    if (mp == null)
+      throw new NullPointerException("MergePolicy must be non-null");
+
+    if (mergePolicy != mp)
+      mergePolicy.close();
+    mergePolicy = mp;
+    pushMaxBufferedDocs();
+  }
+
+  /**
+   * Expert: returns the current MergePolicy in use by this writer.
+   * @see #setMergePolicy
+   */
+  public MergePolicy getMergePolicy() {
+    ensureOpen();
+    return mergePolicy;
+  }
+
+  /**
+   * Expert: set the merge scheduler used by this writer.
+   */
+  public void setMergeScheduler(MergeScheduler mergeScheduler) throws CorruptIndexException, IOException {
+    ensureOpen();
+    if (mergeScheduler == null)
+      throw new NullPointerException("MergeScheduler must be non-null");
+
+    if (this.mergeScheduler != mergeScheduler) {
+      finishMerges(true);
+      this.mergeScheduler.close();
+    }
+    this.mergeScheduler = mergeScheduler;
+  }
+
+  /**
+   * Expert: returns the current MergePolicy in use by this
+   * writer.
+   * @see #setMergePolicy
+   */
+  public MergeScheduler getMergeScheduler() {
+    ensureOpen();
+    return mergeScheduler;
+  }
+
   /** Determines the largest number of documents ever merged by addDocument().
    * Small values (e.g., less than 10,000) are best for interactive indexing,
    * as this limits the length of pauses while indexing to a few seconds.
    * Larger values are best for batched indexing and speedier searches.
    *
    * <p>The default value is {@link Integer#MAX_VALUE}.
+   *
+   * <p>Note that this method is a convenience method: it
+   * just calls mergePolicy.setMaxMergeDocs as long as
+   * mergePolicy is an instance of {@link LogMergePolicy}.
+   * Otherwise an IllegalArgumentException is thrown.</p>
    */
   public void setMaxMergeDocs(int maxMergeDocs) {
-    ensureOpen();
-    this.maxMergeDocs = maxMergeDocs;
+    getLogDocMergePolicy().setMaxMergeDocs(maxMergeDocs);
   }
 
-  /**
+   /**
    * Returns the largest number of documents allowed in a
    * single segment.
+   *
+   * <p>Note that this method is a convenience method: it
+   * just calls mergePolicy.getMaxMergeDocs as long as
+   * mergePolicy is an instance of {@link LogMergePolicy}.
+   * Otherwise an IllegalArgumentException is thrown.</p>
+   *
    * @see #setMaxMergeDocs
    */
   public int getMaxMergeDocs() {
-    ensureOpen();
-    return maxMergeDocs;
+    return getLogDocMergePolicy().getMaxMergeDocs();
   }
 
   /**
@@ -713,9 +838,30 @@
     if (maxBufferedDocs < 2)
       throw new IllegalArgumentException("maxBufferedDocs must at least be 2");
     docWriter.setMaxBufferedDocs(maxBufferedDocs);
+    pushMaxBufferedDocs();
   }
 
   /**
+   * If we are flushing by doc count (not by RAM usage), and
+   * using LogDocMergePolicy then push maxBufferedDocs down
+   * as its minMergeDocs, to keep backwards compatibility.
+   */
+  private void pushMaxBufferedDocs() {
+    if (docWriter.getRAMBufferSizeMB() == 0.0) {
+      final MergePolicy mp = mergePolicy;
+      if (mp instanceof LogDocMergePolicy) {
+        LogDocMergePolicy lmp = (LogDocMergePolicy) mp;
+        final int maxBufferedDocs = docWriter.getMaxBufferedDocs();
+        if (lmp.getMinMergeDocs() != maxBufferedDocs) {
+          if (infoStream != null)
+            message("now push maxBufferedDocs " + maxBufferedDocs + " to LogDocMergePolicy");
+          lmp.setMinMergeDocs(maxBufferedDocs);
+        }
+      }
+    }
+  }
+
+  /**
    * Returns 0 if this writer is flushing by RAM usage, else
    * returns the number of buffered added documents that will
    * trigger a flush.
@@ -784,24 +930,31 @@
    * for batch index creation, and smaller values (< 10) for indices that are
    * interactively maintained.
    *
+   * <p>Note that this method is a convenience method: it
+   * just calls mergePolicy.setMergeFactor as long as
+   * mergePolicy is an instance of {@link LogMergePolicy}.
+   * Otherwise an IllegalArgumentException is thrown.</p>
+   *
    * <p>This must never be less than 2.  The default value is 10.
    */
   public void setMergeFactor(int mergeFactor) {
-    ensureOpen();
-    if (mergeFactor < 2)
-      throw new IllegalArgumentException("mergeFactor cannot be less than 2");
-    this.mergeFactor = mergeFactor;
+    getLogMergePolicy().setMergeFactor(mergeFactor);
   }
 
   /**
-   * Returns the number of segments that are merged at once
-   * and also controls the total number of segments allowed
-   * to accumulate in the index.
+   * <p>Returns the number of segments that are merged at
+   * once and also controls the total number of segments
+   * allowed to accumulate in the index.</p>
+   *
+   * <p>Note that this method is a convenience method: it
+   * just calls mergePolicy.getMergeFactor as long as
+   * mergePolicy is an instance of {@link LogMergePolicy}.
+   * Otherwise an IllegalArgumentException is thrown.</p>
+   *
    * @see #setMergeFactor
    */
   public int getMergeFactor() {
-    ensureOpen();
-    return mergeFactor;
+    return getLogMergePolicy().getMergeFactor();
   }
 
   /** If non-null, this will be the default infoStream used
@@ -910,15 +1063,75 @@
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public synchronized void close() throws CorruptIndexException, IOException {
-    if (!closed) {
+  public void close() throws CorruptIndexException, IOException {
+    close(true);
+  }
+
+  /**
+   * Closes the index with or without waiting for currently
+   * running merges to finish.  This is only meaningful when
+   * using a MergeScheduler that runs merges in background
+   * threads.
+   * @param waitForMerges if true, this call will block
+   * until all merges complete; else, it will abort all
+   * running merges and return right away
+   */
+  public void close(boolean waitForMerges) throws CorruptIndexException, IOException {
+    boolean doClose;
+    synchronized(this) {
+      // Ensure that only one thread actually gets to do the closing:
+      if (!closing) {
+        doClose = true;
+        closing = true;
+      } else
+        doClose = false;
+    }
+    if (doClose)
+      closeInternal(waitForMerges);
+    else
+      // Another thread beat us to it (is actually doing the
+      // close), so we will block until that other thread
+      // has finished closing
+      waitForClose();
+  }
+
+  synchronized private void waitForClose() {
+    while(!closed && closing) {
+      try {
+        wait();
+      } catch (InterruptedException ie) {
+      }
+    }
+  }
+
+  private void closeInternal(boolean waitForMerges) throws CorruptIndexException, IOException {
+    try {
+
       flush(true, true);
 
+      mergePolicy.close();
+
+      finishMerges(waitForMerges);
+
+      mergeScheduler.close();
+
       if (commitPending) {
-        segmentInfos.write(directory);         // now commit changes
+        boolean success = false;
+        try {
+          segmentInfos.write(directory);         // now commit changes
+          success = true;
+        } finally {
+          if (!success) {
+            if (infoStream != null)
+              message("hit exception committing segments file during close");
+            deletePartialSegmentsFile();
+          }
+        }
         if (infoStream != null)
-          infoStream.println("close: wrote segments file \"" + segmentInfos.getCurrentSegmentFileName() + "\"");
-        deleter.checkpoint(segmentInfos, true);
+          message("close: wrote segments file \"" + segmentInfos.getCurrentSegmentFileName() + "\"");
+        synchronized(this) {
+          deleter.checkpoint(segmentInfos, true);
+        }
         commitPending = false;
         rollbackSegmentInfos = null;
       }
@@ -930,17 +1143,31 @@
       closed = true;
       docWriter = null;
 
-      if(closeDir)
+      synchronized(this) {
+        deleter.close();
+      }
+      
+      if (closeDir)
         directory.close();
+    } finally {
+      synchronized(this) {
+        if (!closed)
+          closing = false;
+        notifyAll();
+      }
     }
   }
 
   /** Tells the docWriter to close its currently open shared
-   *  doc stores (stored fields & vectors files). */
-  private void flushDocStores() throws IOException {
+   *  doc stores (stored fields & vectors files).
+   *  Return value specifices whether new doc store files are compound or not.
+   */
+  private synchronized boolean flushDocStores() throws IOException {
 
     List files = docWriter.files();
 
+    boolean useCompoundDocStore = false;
+
     if (files.size() > 0) {
       String docStoreSegment;
 
@@ -949,20 +1176,25 @@
         docStoreSegment = docWriter.closeDocStore();
         success = true;
       } finally {
-        if (!success)
+        if (!success) {
+          if (infoStream != null)
+            message("hit exception closing doc store segment");
           docWriter.abort();
+        }
       }
 
-      if (useCompoundFile && docStoreSegment != null) {
+      useCompoundDocStore = mergePolicy.useCompoundDocStore(segmentInfos);
+      
+      if (useCompoundDocStore && docStoreSegment != null) {
         // Now build compound doc store file
-        checkpoint();
 
         success = false;
 
         final int numSegments = segmentInfos.size();
+        final String compoundFileName = docStoreSegment + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION;
 
         try {
-          CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, docStoreSegment + "." + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION);
+          CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName);
           final int size = files.size();
           for(int i=0;i<size;i++)
             cfsWriter.addFile((String) files.get(i));
@@ -980,6 +1212,10 @@
           success = true;
         } finally {
           if (!success) {
+
+            if (infoStream != null)
+              message("hit exception building compound file doc store for segment " + docStoreSegment);
+            
             // Rollback to no compound file
             for(int i=0;i<numSegments;i++) {
               SegmentInfo si = segmentInfos.info(i);
@@ -987,13 +1223,16 @@
                   si.getDocStoreSegment().equals(docStoreSegment))
                 si.setDocStoreIsCompoundFile(false);
             }
-            deleter.refresh();
+            deleter.deleteFile(compoundFileName);
+            deletePartialSegmentsFile();
           }
         }
 
         deleter.checkpoint(segmentInfos, false);
       }
     }
+
+    return useCompoundDocStore;
   }
 
   /** Release the write lock, if needed. */
@@ -1067,17 +1306,13 @@
    * free temporary space in the Directory to do the
    * merging.</p>
    *
-   * <p>The amount of free space required when a merge is
-   * triggered is up to 1X the size of all segments being
-   * merged, when no readers/searchers are open against the
-   * index, and up to 2X the size of all segments being
-   * merged when readers/searchers are open against the
-   * index (see {@link #optimize()} for details).  Most
-   * merges are small (merging the smallest segments
-   * together), but whenever a full merge occurs (all
-   * segments in the index, which is the worst case for
-   * temporary space usage) then the maximum free disk space
-   * required is the same as {@link #optimize}.</p>
+   * <p>The amount of free space required when a merge is triggered is
+   * up to 1X the size of all segments being merged, when no
+   * readers/searchers are open against the index, and up to 2X the
+   * size of all segments being merged when readers/searchers are open
+   * against the index (see {@link #optimize()} for details). The
+   * sequence of primitive merge operations performed is governed by
+   * the merge policy.
    *
    * <p>Note that each term in the document can be no longer
    * than 16383 characters, otherwise an
@@ -1105,14 +1340,27 @@
    */
   public void addDocument(Document doc, Analyzer analyzer) throws CorruptIndexException, IOException {
     ensureOpen();
+    boolean doFlush = false;
     boolean success = false;
     try {
-      success = docWriter.addDocument(doc, analyzer);
-    } catch (IOException ioe) {
-      deleter.refresh();
-      throw ioe;
+      doFlush = docWriter.addDocument(doc, analyzer);
+      success = true;
+    } finally {
+      if (!success) {
+
+        if (infoStream != null)
+          message("hit exception adding document");
+
+        synchronized (this) {
+          // If docWriter has some aborted files that were
+          // never incref'd, then we clean them up here
+          final List files = docWriter.abortedFiles();
+          if (files != null)
+            deleter.deleteNewFiles(files);
+        }
+      }
     }
-    if (success)
+    if (doFlush)
       flush(true, false);
   }
 
@@ -1178,11 +1426,24 @@
       throws CorruptIndexException, IOException {
     ensureOpen();
     boolean doFlush = false;
+    boolean success = false;
     try {
       doFlush = docWriter.updateDocument(term, doc, analyzer);
-    } catch (IOException ioe) {
-      deleter.refresh();
-      throw ioe;
+      success = true;
+    } finally {
+      if (!success) {
+
+        if (infoStream != null)
+          message("hit exception updating document");
+
+        synchronized (this) {
+          // If docWriter has some aborted files that were
+          // never incref'd, then we clean them up here
+          final List files = docWriter.abortedFiles();
+          if (files != null)
+            deleter.deleteNewFiles(files);
+        }
+      }
     }
     if (doFlush)
       flush(true, false);
@@ -1208,54 +1469,40 @@
   }
 
   final String newSegmentName() {
-    return "_" + Integer.toString(segmentInfos.counter++, Character.MAX_RADIX);
+    // Cannot synchronize on IndexWriter because that causes
+    // deadlock
+    synchronized(segmentInfos) {
+      return "_" + Integer.toString(segmentInfos.counter++, Character.MAX_RADIX);
+    }
   }
 
-  /** Determines how often segment indices are merged by addDocument().  With
-   * smaller values, less RAM is used while indexing, and searches on
-   * unoptimized indices are faster, but indexing speed is slower.  With larger
-   * values, more RAM is used during indexing, and while searches on unoptimized
-   * indices are slower, indexing is faster.  Thus larger values (> 10) are best
-   * for batch index creation, and smaller values (< 10) for indices that are
-   * interactively maintained.
-   *
-   * <p>This must never be less than 2.  The default value is {@link #DEFAULT_MERGE_FACTOR}.
-
-   */
-  private int mergeFactor = DEFAULT_MERGE_FACTOR;
-
   /** Determines amount of RAM usage by the buffered docs at
    * which point we trigger a flush to the index.
    */
   private double ramBufferSize = DEFAULT_RAM_BUFFER_SIZE_MB*1024F*1024F;
 
-  /** Determines the largest number of documents ever merged by addDocument().
-   * Small values (e.g., less than 10,000) are best for interactive indexing,
-   * as this limits the length of pauses while indexing to a few seconds.
-   * Larger values are best for batched indexing and speedier searches.
-   *
-   * <p>The default value is {@link #DEFAULT_MAX_MERGE_DOCS}.
-
-   */
-  private int maxMergeDocs = DEFAULT_MAX_MERGE_DOCS;
-
   /** If non-null, information about merges will be printed to this.
 
    */
   private PrintStream infoStream = null;
-
   private static PrintStream defaultInfoStream = null;
 
-  /** Merges all segments together into a single segment,
-   * optimizing an index for search.
+  /**
+   * Requests an "optimize" operation on an index, priming the index
+   * for the fastest available search. Traditionally this has meant
+   * merging all segments into a single segment as is done in the
+   * default merge policy, but individaul merge policies may implement
+   * optimize in different ways.
    *
+   * @see LogMergePolicy#findMergesForOptimize
+   *
    * <p>It is recommended that this method be called upon completion of indexing.  In
    * environments with frequent updates, optimize is best done during low volume times, if at all. 
    * 
    * </p>
    * <p>See http://www.gossamer-threads.com/lists/lucene/java-dev/47895 for more discussion. </p>
    *
-   * <p>Note that this requires substantial temporary free
+   * <p>Note that this can require substantial temporary free
    * space in the Directory (see <a target="_top"
    * href="http://issues.apache.org/jira/browse/LUCENE-764">LUCENE-764</a>
    * for details):</p>
@@ -1293,7 +1540,7 @@
    * <p>The actual temporary usage could be much less than
    * these figures (it depends on many factors).</p>
    *
-   * <p>Once the optimize completes, the total size of the
+   * <p>In general, once the optimize completes, the total size of the
    * index will be less than the size of the starting index.
    * It could be quite a bit smaller (if there were many
    * pending deletes) or just slightly smaller.</p>
@@ -1307,24 +1554,158 @@
    * using compound file format.  This will occur when the
    * Exception is hit during conversion of the segment into
    * compound format.</p>
+   *
+   * <p>This call will optimize those segments present in
+   * the index when the call started.  If other threads are
+   * still adding documents and flushing segments, those
+   * newly created segments will not be optimized unless you
+   * call optimize again.</p>
+   *
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
   */
-  public synchronized void optimize() throws CorruptIndexException, IOException {
+  public void optimize() throws CorruptIndexException, IOException {
+    optimize(true);
+  }
+
+  /** Just like {@link #optimize()}, except you can specify
+   *  whether the call should block until the optimize
+   *  completes.  This is only meaningful with a
+   *  {@link MergeScheduler} that is able to run merges in
+   *  background threads. */
+  public void optimize(boolean doWait) throws CorruptIndexException, IOException {
     ensureOpen();
     flush();
-    while (segmentInfos.size() > 1 ||
-           (segmentInfos.size() == 1 &&
-            (SegmentReader.hasDeletions(segmentInfos.info(0)) ||
-             SegmentReader.hasSeparateNorms(segmentInfos.info(0)) ||
-             segmentInfos.info(0).dir != directory ||
-             (useCompoundFile &&
-              !segmentInfos.info(0).getUseCompoundFile())))) {
-      int minSegment = segmentInfos.size() - mergeFactor;
-      mergeSegments(minSegment < 0 ? 0 : minSegment, segmentInfos.size());
+
+    if (infoStream != null)
+      message("optimize: index now " + segString());
+
+    synchronized(this) {
+      resetMergeExceptions();
+      segmentsToOptimize = new HashSet();
+      final int numSegments = segmentInfos.size();
+      for(int i=0;i<numSegments;i++)
+        segmentsToOptimize.add(segmentInfos.info(i));
+      
+      // Now mark all pending & running merges as optimize
+      // merge:
+      Iterator it = pendingMerges.iterator();
+      while(it.hasNext())
+        ((MergePolicy.OneMerge) it.next()).optimize = true;
+
+      it = runningMerges.iterator();
+      while(it.hasNext())
+        ((MergePolicy.OneMerge) it.next()).optimize = true;
     }
+
+    maybeMerge(true);
+
+    if (doWait) {
+      synchronized(this) {
+        while(optimizeMergesPending()) {
+          try {
+            wait();
+          } catch (InterruptedException ie) {
+          }
+
+          if (mergeExceptions.size() > 0) {
+            // Forward any exceptions in background merge
+            // threads to the current thread:
+            final int size = mergeExceptions.size();
+            for(int i=0;i<size;i++) {
+              final MergePolicy.OneMerge merge = (MergePolicy.OneMerge) mergeExceptions.get(0);
+              if (merge.optimize) {
+                IOException err = new IOException("background merge hit exception: " + merge.segString(directory));
+                err.initCause(merge.getException());
+                throw err;
+              }
+            }
+          }
+        }
+      }
+    }
+
+    // NOTE: in the ConcurrentMergeScheduler case, when
+    // doWait is false, we can return immediately while
+    // background threads accomplish the optimization
   }
 
+  /** Returns true if any merges in pendingMerges or
+   *  runningMerges are optimization merges. */
+  private synchronized boolean optimizeMergesPending() {
+    Iterator it = pendingMerges.iterator();
+    while(it.hasNext())
+      if (((MergePolicy.OneMerge) it.next()).optimize)
+        return true;
+
+    it = runningMerges.iterator();
+    while(it.hasNext())
+      if (((MergePolicy.OneMerge) it.next()).optimize)
+        return true;
+
+    return false;
+  }
+
+  /**
+   * Expert: asks the mergePolicy whether any merges are
+   * necessary now and if so, runs the requested merges and
+   * then iterate (test again if merges are needed) until no
+   * more merges are returned by the mergePolicy.
+   *
+   * Explicit calls to maybeMerge() are usually not
+   * necessary. The most common case is when merge policy
+   * parameters have changed.
+   */
+  public final void maybeMerge() throws CorruptIndexException, IOException {
+    maybeMerge(false);
+  }
+
+  private final void maybeMerge(boolean optimize) throws CorruptIndexException, IOException {
+    updatePendingMerges(optimize);
+    mergeScheduler.merge(this);
+  }
+
+  private synchronized void updatePendingMerges(boolean optimize)
+    throws CorruptIndexException, IOException {
+
+    final MergePolicy.MergeSpecification spec;
+    if (optimize) {
+      // Currently hardwired to 1, but once we add method to
+      // IndexWriter to allow "optimizing to <= N segments"
+      // then we will change this.
+      final int maxSegmentCount = 1;
+      spec = mergePolicy.findMergesForOptimize(segmentInfos, this, maxSegmentCount, segmentsToOptimize);
+
+      if (spec != null) {
+        final int numMerges = spec.merges.size();
+        for(int i=0;i<numMerges;i++)
+          ((MergePolicy.OneMerge) spec.merges.get(i)).optimize = true;
+      }
+
+    } else
+      spec = mergePolicy.findMerges(segmentInfos, this);
+
+    if (spec != null) {
+      final int numMerges = spec.merges.size();
+      for(int i=0;i<numMerges;i++)
+        registerMerge((MergePolicy.OneMerge) spec.merges.get(i));
+    }
+  }
+
+  /** Expert: the {@link MergeScheduler} calls this method
+   *  to retrieve the next merge requested by the
+   *  MergePolicy */
+  synchronized MergePolicy.OneMerge getNextMerge() {
+    if (pendingMerges.size() == 0)
+      return null;
+    else {
+      // Advance the merge from pending to running
+      MergePolicy.OneMerge merge = (MergePolicy.OneMerge) pendingMerges.removeFirst();
+      runningMerges.add(merge);
+      return merge;
+    }
+  }
+
   /*
    * Begin a transaction.  During a transaction, any segment
    * merges that happen (or ram segments flushed) will not
@@ -1340,6 +1721,9 @@
    */
   private void startTransaction() throws IOException {
 
+    if (infoStream != null)
+      message("now start transaction");
+
     assert docWriter.getNumBufferedDeleteTerms() == 0 :
            "calling startTransaction with buffered delete terms not supported";
     assert docWriter.getNumDocsInRAM() == 0 :
@@ -1363,6 +1747,9 @@
    */
   private void rollbackTransaction() throws IOException {
 
+    if (infoStream != null)
+      message("now rollback transaction");
+
     // First restore autoCommit in case we hit an exception below:
     autoCommit = localAutoCommit;
 
@@ -1383,6 +1770,7 @@
       deleter.decRef(segmentInfos);
 
     deleter.refresh();
+    finishMerges(false);
   }
 
   /*
@@ -1392,6 +1780,9 @@
    */
   private void commitTransaction() throws IOException {
 
+    if (infoStream != null)
+      message("now commit transaction");
+
     // First restore autoCommit in case we hit an exception below:
     autoCommit = localAutoCommit;
 
@@ -1401,6 +1792,9 @@
       success = true;
     } finally {
       if (!success) {
+        if (infoStream != null)
+          message("hit exception committing transaction");
+
         rollbackTransaction();
       }
     }
@@ -1427,30 +1821,78 @@
    *  the writer was opened with <code>autoCommit=true</code>.
    * @throws IOException if there is a low-level IO error
    */
-  public synchronized void abort() throws IOException {
+  public void abort() throws IOException {
     ensureOpen();
-    if (!autoCommit) {
+    if (autoCommit)
+      throw new IllegalStateException("abort() can only be called when IndexWriter was opened with autoCommit=false");
 
-      // Keep the same segmentInfos instance but replace all
-      // of its SegmentInfo instances.  This is so the next
-      // attempt to commit using this instance of IndexWriter
-      // will always write to a new generation ("write once").
-      segmentInfos.clear();
-      segmentInfos.addAll(rollbackSegmentInfos);
+    boolean doClose;
+    synchronized(this) {
+      // Ensure that only one thread actually gets to do the closing:
+      if (!closing) {
+        doClose = true;
+        closing = true;
+      } else
+        doClose = false;
+    }
 
-      docWriter.abort();
+    if (doClose) {
 
-      // Ask deleter to locate unreferenced files & remove
-      // them:
-      deleter.checkpoint(segmentInfos, false);
-      deleter.refresh();
+      finishMerges(false);
 
+      // Must pre-close these two, in case they set
+      // commitPending=true, so that we can then set it to
+      // false before calling closeInternal
+      mergePolicy.close();
+      mergeScheduler.close();
+
+      synchronized(this) {
+        // Keep the same segmentInfos instance but replace all
+        // of its SegmentInfo instances.  This is so the next
+        // attempt to commit using this instance of IndexWriter
+        // will always write to a new generation ("write
+        // once").
+        segmentInfos.clear();
+        segmentInfos.addAll(rollbackSegmentInfos);
+
+        docWriter.abort();
+
+        // Ask deleter to locate unreferenced files & remove
+        // them:
+        deleter.checkpoint(segmentInfos, false);
+        deleter.refresh();
+        finishMerges(false);
+      }
+
       commitPending = false;
-      docWriter.abort();
-      close();
+      closeInternal(false);
+    } else
+      waitForClose();
+  }
 
+  private synchronized void finishMerges(boolean waitForMerges) {
+    if (!waitForMerges) {
+      // Abort all pending & running merges:
+      Iterator it = pendingMerges.iterator();
+      while(it.hasNext())
+        ((MergePolicy.OneMerge) it.next()).abort();
+
+      pendingMerges.clear();
+      it = runningMerges.iterator();
+      while(it.hasNext())
+        ((MergePolicy.OneMerge) it.next()).abort();
+
+      runningMerges.clear();
+      mergingSegments.clear();
+      notifyAll();
     } else {
-      throw new IllegalStateException("abort() can only be called when IndexWriter was opened with autoCommit=false");
+      while(pendingMerges.size() > 0 || runningMerges.size() > 0) {
+        try {
+          wait();
+        } catch (InterruptedException ie) {
+        }
+      }
+      assert 0 == mergingSegments.size();
     }
   }
  
@@ -1461,11 +1903,11 @@
    * commit the change immediately.  Else, we mark
    * commitPending.
    */
-  private void checkpoint() throws IOException {
+  private synchronized void checkpoint() throws IOException {
     if (autoCommit) {
       segmentInfos.write(directory);
       if (infoStream != null)
-        infoStream.println("checkpoint: wrote segments file \"" + segmentInfos.getCurrentSegmentFileName() + "\"");
+        message("checkpoint: wrote segments file \"" + segmentInfos.getCurrentSegmentFileName() + "\"");
     } else {
       commitPending = true;
     }
@@ -1521,7 +1963,7 @@
     throws CorruptIndexException, IOException {
 
     ensureOpen();
-    optimize();					  // start with zero or 1 seg
+    flush();
 
     int start = segmentInfos.size();
 
@@ -1538,15 +1980,8 @@
         }
       }
 
-      // merge newly added segments in log(n) passes
-      while (segmentInfos.size() > start+mergeFactor) {
-        for (int base = start; base < segmentInfos.size(); base++) {
-          int end = Math.min(segmentInfos.size(), base+mergeFactor);
-          if (end-base > 1) {
-            mergeSegments(base, end);
-          }
-        }
-      }
+      optimize();
+
       success = true;
     } finally {
       if (success) {
@@ -1555,8 +1990,11 @@
         rollbackTransaction();
       }
     }
+  }
 
-    optimize();					  // final cleanup
+  private synchronized void resetMergeExceptions() {
+    mergeExceptions = new ArrayList();
+    mergeGen++;
   }
 
   /**
@@ -1578,40 +2016,10 @@
    */
   public synchronized void addIndexesNoOptimize(Directory[] dirs)
       throws CorruptIndexException, IOException {
-    // Adding indexes can be viewed as adding a sequence of segments S to
-    // a sequence of segments T. Segments in T follow the invariants but
-    // segments in S may not since they could come from multiple indexes.
-    // Here is the merge algorithm for addIndexesNoOptimize():
-    //
-    // 1 Flush ram.
-    // 2 Consider a combined sequence with segments from T followed
-    //   by segments from S (same as current addIndexes(Directory[])).
-    // 3 Assume the highest level for segments in S is h. Call
-    //   maybeMergeSegments(), but instead of starting w/ lowerBound = -1
-    //   and upperBound = maxBufferedDocs, start w/ lowerBound = -1 and
-    //   upperBound = upperBound of level h. After this, the invariants
-    //   are guaranteed except for the last < M segments whose levels <= h.
-    // 4 If the invariants hold for the last < M segments whose levels <= h,
-    //   if some of those < M segments are from S (not merged in step 3),
-    //   properly copy them over*, otherwise done.
-    //   Otherwise, simply merge those segments. If the merge results in
-    //   a segment of level <= h, done. Otherwise, it's of level h+1 and call
-    //   maybeMergeSegments() starting w/ upperBound = upperBound of level h+1.
-    //
-    // * Ideally, we want to simply copy a segment. However, directory does
-    // not support copy yet. In addition, source may use compound file or not
-    // and target may use compound file or not. So we use mergeSegments() to
-    // copy a segment, which may cause doc count to change because deleted
-    // docs are garbage collected.
 
-    // 1 flush ram
-
     ensureOpen();
     flush();
 
-    // 2 copy segment infos and find the highest level from dirs
-    int startUpperBound = docWriter.getMaxBufferedDocs();
-
     /* new merge policy
     if (startUpperBound == 0)
       startUpperBound = 10;
@@ -1634,64 +2042,20 @@
         for (int j = 0; j < sis.size(); j++) {
           SegmentInfo info = sis.info(j);
           segmentInfos.addElement(info); // add each info
-          
-          while (startUpperBound < info.docCount) {
-            startUpperBound *= mergeFactor; // find the highest level from dirs
-            if (startUpperBound > maxMergeDocs) {
-              // upper bound cannot exceed maxMergeDocs
-              throw new IllegalArgumentException("Upper bound cannot exceed maxMergeDocs");
-            }
-          }
         }
       }
 
-      // 3 maybe merge segments starting from the highest level from dirs
-      maybeMergeSegments(startUpperBound);
+      maybeMerge();
 
-      // get the tail segments whose levels <= h
-      int segmentCount = segmentInfos.size();
-      int numTailSegments = 0;
-      while (numTailSegments < segmentCount
-             && startUpperBound >= segmentInfos.info(segmentCount - 1 - numTailSegments).docCount) {
-        numTailSegments++;
-      }
-      if (numTailSegments == 0) {
-        success = true;
-        return;
-      }
+      // If after merging there remain segments in the index
+      // that are in a different directory, just copy these
+      // over into our index.  This is necessary (before
+      // finishing the transaction) to avoid leaving the
+      // index in an unusable (inconsistent) state.
+      copyExternalSegments();
 
-      // 4 make sure invariants hold for the tail segments whose levels <= h
-      if (checkNonDecreasingLevels(segmentCount - numTailSegments)) {
-        // identify the segments from S to be copied (not merged in 3)
-        int numSegmentsToCopy = 0;
-        while (numSegmentsToCopy < segmentCount
-               && directory != segmentInfos.info(segmentCount - 1 - numSegmentsToCopy).dir) {
-          numSegmentsToCopy++;
-        }
-        if (numSegmentsToCopy == 0) {
-          success = true;
-          return;
-        }
+      success = true;
 
-        // copy those segments from S
-        for (int i = segmentCount - numSegmentsToCopy; i < segmentCount; i++) {
-          mergeSegments(i, i + 1);
-        }
-        if (checkNonDecreasingLevels(segmentCount - numSegmentsToCopy)) {
-          success = true;
-          return;
-        }
-      }
-
-      // invariants do not hold, simply merge those segments
-      mergeSegments(segmentCount - numTailSegments, segmentCount);
-
-      // maybe merge segments again if necessary
-      if (segmentInfos.info(segmentInfos.size() - 1).docCount > startUpperBound) {
-        maybeMergeSegments(startUpperBound * mergeFactor);
-      }
-
-      success = true;
     } finally {
       if (success) {
         commitTransaction();
@@ -1701,6 +2065,33 @@
     }
   }
 
+  /* If any of our segments are using a directory != ours
+   * then copy them over.  Currently this is only used by
+   * addIndexesNoOptimize(). */
+  private synchronized void copyExternalSegments() throws CorruptIndexException, IOException {
+    final int numSegments = segmentInfos.size();
+    for(int i=0;i<numSegments;i++) {
+      SegmentInfo info = segmentInfos.info(i);
+      if (info.dir != directory) {
+        MergePolicy.OneMerge merge = new MergePolicy.OneMerge(segmentInfos.range(i, 1+i), info.getUseCompoundFile());
+        if (registerMerge(merge)) {
+          pendingMerges.remove(merge);
+          runningMerges.add(merge);
+          merge(merge);
+        } else
+          // This means there is a bug in the
+          // MergeScheduler.  MergeSchedulers in general are
+          // not allowed to run a merge involving segments
+          // external to this IndexWriter's directory in the
+          // background because this would put the index
+          // into an inconsistent state (where segmentInfos
+          // has been written with such external segments
+          // that an IndexReader would fail to load).
+          throw new MergePolicy.MergeException("segment \"" + info.name + " exists in external directory yet the MergeScheduler executed the merge in a separate thread");
+      }
+    }
+  }
+
   /** Merges the provided indexes into this index.
    * <p>After this completes, the index is optimized. </p>
    * <p>The provided IndexReaders are not closed.</p>
@@ -1754,6 +2145,9 @@
 
       } finally {
         if (!success) {
+          if (infoStream != null)
+            message("hit exception in addIndexes during merge");
+
           rollbackTransaction();
         } else {
           commitTransaction();
@@ -1765,7 +2159,7 @@
       }
     }
     
-    if (useCompoundFile) {
+    if (mergePolicy instanceof LogMergePolicy && getUseCompoundFile()) {
 
       boolean success = false;
 
@@ -1776,6 +2170,9 @@
         info.setUseCompoundFile(true);
       } finally {
         if (!success) {
+          if (infoStream != null)
+            message("hit exception building compound file in addIndexes during merge");
+
           rollbackTransaction();
         } else {
           commitTransaction();
@@ -1784,40 +2181,6 @@
     }
   }
 
-  // Overview of merge policy:
-  //
-  // A flush is triggered either by close() or by the number of ram segments
-  // reaching maxBufferedDocs. After a disk segment is created by the flush,
-  // further merges may be triggered.
-  //
-  // LowerBound and upperBound set the limits on the doc count of a segment
-  // which may be merged. Initially, lowerBound is set to 0 and upperBound
-  // to maxBufferedDocs. Starting from the rightmost* segment whose doc count
-  // > lowerBound and <= upperBound, count the number of consecutive segments
-  // whose doc count <= upperBound.
-  //
-  // Case 1: number of worthy segments < mergeFactor, no merge, done.
-  // Case 2: number of worthy segments == mergeFactor, merge these segments.
-  //         If the doc count of the merged segment <= upperBound, done.
-  //         Otherwise, set lowerBound to upperBound, and multiply upperBound
-  //         by mergeFactor, go through the process again.
-  // Case 3: number of worthy segments > mergeFactor (in the case mergeFactor
-  //         M changes), merge the leftmost* M segments. If the doc count of
-  //         the merged segment <= upperBound, consider the merged segment for
-  //         further merges on this same level. Merge the now leftmost* M
-  //         segments, and so on, until number of worthy segments < mergeFactor.
-  //         If the doc count of all the merged segments <= upperBound, done.
-  //         Otherwise, set lowerBound to upperBound, and multiply upperBound
-  //         by mergeFactor, go through the process again.
-  // Note that case 2 can be considerd as a special case of case 3.
-  //
-  // This merge policy guarantees two invariants if M does not change and
-  // segment doc count is not reaching maxMergeDocs:
-  // B for maxBufferedDocs, f(n) defined as ceil(log_M(ceil(n/B)))
-  //      1: If i (left*) and i+1 (right*) are two consecutive segments of doc
-  //         counts x and y, then f(x) >= f(y).
-  //      2: The number of committed segments on the same level (f(n)) <= M.
-
   // This is called after pending added and deleted
   // documents have been flushed to the Directory but before
   // the change is committed (new segments_N file written).
@@ -1833,7 +2196,7 @@
    * @throws CorruptIndexException if the index is corrupt
    * @throws IOException if there is a low-level IO error
    */
-  public final synchronized void flush() throws CorruptIndexException, IOException {  
+  public final void flush() throws CorruptIndexException, IOException {  
     flush(true, false);
   }
 
@@ -1845,9 +2208,15 @@
    * @param flushDocStores if false we are allowed to keep
    *  doc stores open to share with the next segment
    */
-  protected final synchronized void flush(boolean triggerMerge, boolean flushDocStores) throws CorruptIndexException, IOException {
+  protected final void flush(boolean triggerMerge, boolean flushDocStores) throws CorruptIndexException, IOException {
     ensureOpen();
 
+    if (doFlush(flushDocStores) && triggerMerge)
+      maybeMerge();
+  }
+
+  private synchronized final boolean doFlush(boolean flushDocStores) throws CorruptIndexException, IOException {
+
     // Make sure no threads are actively adding a document
     docWriter.pauseAllThreads();
 
@@ -1877,10 +2246,14 @@
       boolean flushDeletes = docWriter.hasDeletes();
 
       if (infoStream != null)
-        infoStream.println("  flush: flushDocs=" + flushDocs +
-                           " flushDeletes=" + flushDeletes +
-                           " flushDocStores=" + flushDocStores +
-                           " numDocs=" + numDocs);
+        message("  flush: segment=" + docWriter.getSegment() +
+                " docStoreSegment=" + docWriter.getDocStoreSegment() +
+                " docStoreOffset=" + docWriter.getDocStoreOffset() +
+                " flushDocs=" + flushDocs +
+                " flushDeletes=" + flushDeletes +
+                " flushDocStores=" + flushDocStores +
+                " numDocs=" + numDocs +
+                " numBufDelTerms=" + docWriter.getNumBufferedDeleteTerms());
 
       int docStoreOffset = docWriter.getDocStoreOffset();
       boolean docStoreIsCompoundFile = false;
@@ -1891,15 +2264,17 @@
       if (flushDocStores && (!flushDocs || !docWriter.getSegment().equals(docWriter.getDocStoreSegment()))) {
         // We must separately flush the doc store
         if (infoStream != null)
-          infoStream.println("  flush shared docStore segment " + docStoreSegment);
+          message("  flush shared docStore segment " + docStoreSegment);
       
-        flushDocStores();
+        docStoreIsCompoundFile = flushDocStores();
         flushDocStores = false;
-        docStoreIsCompoundFile = useCompoundFile;
       }
 
       String segment = docWriter.getSegment();
 
+      // If we are flushing docs, segment must not be null:
+      assert segment != null || !flushDocs;
+
       if (flushDocs || flushDeletes) {
 
         SegmentInfos rollback = null;
@@ -1948,7 +2323,22 @@
           success = true;
         } finally {
           if (!success) {
+
+            if (infoStream != null)
+              message("hit exception flushing segment " + segment);
+                
             if (flushDeletes) {
+
+              // Carefully check if any partial .del files
+              // should be removed:
+              final int size = rollback.size();
+              for(int i=0;i<size;i++) {
+                final String newDelFileName = segmentInfos.info(i).getDelFileName();
+                final String delFileName = rollback.info(i).getDelFileName();
+                if (newDelFileName != null && !newDelFileName.equals(delFileName))
+                  deleter.deleteFile(newDelFileName);
+              }
+
               // Fully replace the segmentInfos since flushed
               // deletes could have changed any of the
               // SegmentInfo instances:
@@ -1964,14 +2354,18 @@
             }
             if (flushDocs)
               docWriter.abort();
+            deletePartialSegmentsFile();
             deleter.checkpoint(segmentInfos, false);
-            deleter.refresh();
+
+            if (segment != null)
+              deleter.refresh(segment);
           }
         }
 
         deleter.checkpoint(segmentInfos, autoCommit);
 
-        if (flushDocs && useCompoundFile) {
+        if (flushDocs && mergePolicy.useCompoundFile(segmentInfos,
+                                                     newSegment)) {
           success = false;
           try {
             docWriter.createCompoundFile(segment);
@@ -1980,23 +2374,22 @@
             success = true;
           } finally {
             if (!success) {
+              if (infoStream != null)
+                message("hit exception creating compound file for newly flushed segment " + segment);
               newSegment.setUseCompoundFile(false);
-              deleter.refresh();
+              deleter.deleteFile(segment + "." + IndexFileNames.COMPOUND_FILE_EXTENSION);
+              deletePartialSegmentsFile();
             }
           }
 
           deleter.checkpoint(segmentInfos, autoCommit);
         }
+      
+        return true;
+      } else {
+        return false;
+      }
 
-        /* new merge policy
-        if (0 == docWriter.getMaxBufferedDocs())
-          maybeMergeSegments(mergeFactor * numDocs / 2);
-        else
-          maybeMergeSegments(docWriter.getMaxBufferedDocs());
-        */
-        if (triggerMerge)
-          maybeMergeSegments(docWriter.getMaxBufferedDocs());
-      }
     } finally {
       docWriter.clearFlushPending();
       docWriter.resumeAllThreads();
@@ -2018,261 +2411,606 @@
     ensureOpen();
     return docWriter.getNumDocsInRAM();
   }
-  
-  /** Incremental segment merger.  */
-  private final void maybeMergeSegments(int startUpperBound) throws CorruptIndexException, IOException {
-    long lowerBound = -1;
-    long upperBound = startUpperBound;
 
-    /* new merge policy
-    if (upperBound == 0) upperBound = 10;
-    */
+  private int ensureContiguousMerge(MergePolicy.OneMerge merge) {
 
-    while (upperBound < maxMergeDocs) {
-      int minSegment = segmentInfos.size();
-      int maxSegment = -1;
+    int first = segmentInfos.indexOf(merge.segments.info(0));
+    if (first == -1)
+      throw new MergePolicy.MergeException("could not find segment " + merge.segments.info(0).name + " in current segments");
 
-      // find merge-worthy segments
-      while (--minSegment >= 0) {
-        SegmentInfo si = segmentInfos.info(minSegment);
+    final int numSegments = segmentInfos.size();
+    
+    final int numSegmentsToMerge = merge.segments.size();
+    for(int i=0;i<numSegmentsToMerge;i++) {
+      final SegmentInfo info = merge.segments.info(i);
 
-        if (maxSegment == -1 && si.docCount > lowerBound && si.docCount <= upperBound) {
-          // start from the rightmost* segment whose doc count is in bounds
-          maxSegment = minSegment;
-        } else if (si.docCount > upperBound) {
-          // until the segment whose doc count exceeds upperBound
-          break;
-        }
+      if (first + i >= numSegments || !segmentInfos.info(first+i).equals(info)) {
+        if (segmentInfos.indexOf(info) == -1)
+          throw new MergePolicy.MergeException("MergePolicy selected a segment (" + info.name + ") that is not in the index");
+        else
+          throw new MergePolicy.MergeException("MergePolicy selected non-contiguous segments to merge (" + merge + " vs " + segString() + "), which IndexWriter (currently) cannot handle");
       }
+    }
 
-      minSegment++;
-      maxSegment++;
-      int numSegments = maxSegment - minSegment;
+    return first;
+  }
 
-      if (numSegments < mergeFactor) {
-        break;
-      } else {
-        boolean exceedsUpperLimit = false;
+  /* FIXME if we want to support non-contiguous segment merges */
+  synchronized private boolean commitMerge(MergePolicy.OneMerge merge) throws IOException {
 
-        // number of merge-worthy segments may exceed mergeFactor when
-        // mergeFactor and/or maxBufferedDocs change(s)
-        while (numSegments >= mergeFactor) {
-          // merge the leftmost* mergeFactor segments
+    assert merge.registerDone;
 
-          int docCount = mergeSegments(minSegment, minSegment + mergeFactor);
-          numSegments -= mergeFactor;
+    // If merge was explicitly aborted, or, if abort() or
+    // rollbackTransaction() had been called since our merge
+    // started (which results in an unqualified
+    // deleter.refresh() call that will remove any index
+    // file that current segments does not reference), we
+    // abort this merge
+    if (merge.isAborted()) {
 
-          if (docCount > upperBound) {
-            // continue to merge the rest of the worthy segments on this level
-            minSegment++;
-            exceedsUpperLimit = true;
-          } else {
-            // if the merged segment does not exceed upperBound, consider
-            // this segment for further merges on this same level
-            numSegments++;
+      if (infoStream != null) {
+        if (merge.isAborted())
+          message("commitMerge: skipping merge " + merge.segString(directory) + ": it was aborted");
+      }
+
+      assert merge.increfDone;
+      decrefMergeSegments(merge);
+      deleter.refresh(merge.info.name);
+      return false;
+    }
+
+    boolean success = false;
+
+    int start;
+
+    try {
+      SegmentInfos sourceSegmentsClone = merge.segmentsClone;
+      SegmentInfos sourceSegments = merge.segments;
+      final int numSegments = segmentInfos.size();
+
+      start = ensureContiguousMerge(merge);
+      if (infoStream != null)
+        message("commitMerge " + merge.segString(directory));
+
+      // Carefully merge deletes that occurred after we
+      // started merging:
+
+      BitVector deletes = null;
+      int docUpto = 0;
+
+      final int numSegmentsToMerge = sourceSegments.size();
+      for(int i=0;i<numSegmentsToMerge;i++) {
+        final SegmentInfo previousInfo = sourceSegmentsClone.info(i);
+        final SegmentInfo currentInfo = sourceSegments.info(i);
+
+        assert currentInfo.docCount == previousInfo.docCount;
+
+        final int docCount = currentInfo.docCount;
+
+        if (previousInfo.hasDeletions()) {
+
+          // There were deletes on this segment when the merge
+          // started.  The merge has collapsed away those
+          // deletes, but, if new deletes were flushed since
+          // the merge started, we must now carefully keep any
+          // newly flushed deletes but mapping them to the new
+          // docIDs.
+
+          assert currentInfo.hasDeletions();
+
+          // Load deletes present @ start of merge, for this segment:
+          BitVector previousDeletes = new BitVector(previousInfo.dir, previousInfo.getDelFileName());
+
+          if (!currentInfo.getDelFileName().equals(previousInfo.getDelFileName())) {
+            // This means this segment has had new deletes
+            // committed since we started the merge, so we
+            // must merge them:
+            if (deletes == null)
+              deletes = new BitVector(merge.info.docCount);
+
+            BitVector currentDeletes = new BitVector(currentInfo.dir, currentInfo.getDelFileName());
+            for(int j=0;j<docCount;j++) {
+              if (previousDeletes.get(j))
+                assert currentDeletes.get(j);
+              else {
+                if (currentDeletes.get(j))
+                  deletes.set(docUpto);
+                docUpto++;
+              }
+            }
+          } else
+            docUpto += docCount - previousDeletes.count();
+        
+        } else if (currentInfo.hasDeletions()) {
+          // This segment had no deletes before but now it
+          // does:
+          if (deletes == null)
+            deletes = new BitVector(merge.info.docCount);
+          BitVector currentDeletes = new BitVector(directory, currentInfo.getDelFileName());
+
+          for(int j=0;j<docCount;j++) {
+            if (currentDeletes.get(j))
+              deletes.set(docUpto);
+            docUpto++;
           }
-        }
+            
+        } else
+          // No deletes before or after
+          docUpto += currentInfo.docCount;
+      }
 
-        if (!exceedsUpperLimit) {
-          // if none of the merged segments exceed upperBound, done
+      if (deletes != null) {
+        merge.info.advanceDelGen();
+        deletes.write(directory, merge.info.getDelFileName());
+      }
+      success = true;
+    } finally {
+      if (!success) {
+        if (infoStream != null)
+          message("hit exception creating merged deletes file");
+        deleter.refresh(merge.info.name);
+      }
+    }
+
+    // Simple optimization: if the doc store we are using
+    // has been closed and is in now compound format (but
+    // wasn't when we started), then we will switch to the
+    // compound format as well:
+    final String mergeDocStoreSegment = merge.info.getDocStoreSegment(); 
+    if (mergeDocStoreSegment != null && !merge.info.getDocStoreIsCompoundFile()) {
+      final int size = segmentInfos.size();
+      for(int i=0;i<size;i++) {
+        final SegmentInfo info = segmentInfos.info(i);
+        final String docStoreSegment = info.getDocStoreSegment();
+        if (docStoreSegment != null &&
+            docStoreSegment.equals(mergeDocStoreSegment) && 
+            info.getDocStoreIsCompoundFile()) {
+          merge.info.setDocStoreIsCompoundFile(true);
           break;
         }
       }
+    }
 
-      lowerBound = upperBound;
-      upperBound *= mergeFactor;
+    success = false;
+    SegmentInfos rollback = null;
+    try {
+      rollback = (SegmentInfos) segmentInfos.clone();
+      segmentInfos.subList(start, start + merge.segments.size()).clear();
+      segmentInfos.add(start, merge.info);
+      checkpoint();
+      success = true;
+    } finally {
+      if (!success && rollback != null) {
+        if (infoStream != null)
+          message("hit exception when checkpointing after merge");
+        segmentInfos.clear();
+        segmentInfos.addAll(rollback);
+        deletePartialSegmentsFile();
+        deleter.refresh(merge.info.name);
+      }
     }
+
+    if (merge.optimize)
+      segmentsToOptimize.add(merge.info);
+
+    // Must checkpoint before decrefing so any newly
+    // referenced files in the new merge.info are incref'd
+    // first:
+    deleter.checkpoint(segmentInfos, autoCommit);
+
+    decrefMergeSegments(merge);
+
+    return true;
   }
 
+  private void decrefMergeSegments(MergePolicy.OneMerge merge) throws IOException {
+    final SegmentInfos sourceSegmentsClone = merge.segmentsClone;
+    final int numSegmentsToMerge = sourceSegmentsClone.size();
+    assert merge.increfDone;
+    merge.increfDone = false;
+    for(int i=0;i<numSegmentsToMerge;i++) {
+      final SegmentInfo previousInfo = sourceSegmentsClone.info(i);
+      // Decref all files for this SegmentInfo (this
+      // matches the incref in mergeInit):
+      if (previousInfo.dir == directory)
+        deleter.decRef(previousInfo.files());
+    }
+  }
+
   /**
-   * Merges the named range of segments, replacing them in the stack with a
+   * Merges the indicated segments, replacing them in the stack with a
    * single segment.
    */
 
-  private final int mergeSegments(int minSegment, int end)
+  final void merge(MergePolicy.OneMerge merge)
     throws CorruptIndexException, IOException {
 
-    final String mergedName = newSegmentName();
-    
-    SegmentMerger merger = null;
-    SegmentInfo newSegment = null;
+    assert merge.registerDone;
 
-    int mergedDocCount = 0;
+    int mergedDocCount;
+    boolean success = false;
 
-    // This is try/finally to make sure merger's readers are closed:
     try {
 
-      if (infoStream != null) infoStream.print("merging segments");
+      if (merge.info == null)
+        mergeInit(merge);
 
-      // Check whether this merge will allow us to skip
-      // merging the doc stores (stored field & vectors).
-      // This is a very substantial optimization (saves tons
-      // of IO) that can only be applied with
-      // autoCommit=false.
+      if (infoStream != null)
+        message("now merge\n  merge=" + merge.segString(directory) + "\n  index=" + segString());
 
-      Directory lastDir = directory;
-      String lastDocStoreSegment = null;
-      boolean mergeDocStores = false;
-      boolean doFlushDocStore = false;
-      int next = -1;
+      mergedDocCount = mergeMiddle(merge);
 
-      // Test each segment to be merged
-      for (int i = minSegment; i < end; i++) {
-        SegmentInfo si = segmentInfos.info(i);
+      success = true;
+    } finally {
+      synchronized(this) {
+        if (!success && infoStream != null)
+          message("hit exception during merge");
 
-        // If it has deletions we must merge the doc stores
-        if (si.hasDeletions())
-          mergeDocStores = true;
+        mergeFinish(merge);
 
-        // If it has its own (private) doc stores we must
-        // merge the doc stores
-        if (-1 == si.getDocStoreOffset())
-          mergeDocStores = true;
+        // This merge (and, generally, any change to the
+        // segments) may now enable new merges, so we call
+        // merge policy & update pending merges.
+        if (success && !merge.isAborted() && !closed && !closing)
+          updatePendingMerges(merge.optimize);
 
-        // If it has a different doc store segment than
-        // previous segments, we must merge the doc stores
-        String docStoreSegment = si.getDocStoreSegment();
-        if (docStoreSegment == null)
-          mergeDocStores = true;
-        else if (lastDocStoreSegment == null)
-          lastDocStoreSegment = docStoreSegment;
-        else if (!lastDocStoreSegment.equals(docStoreSegment))
-          mergeDocStores = true;
+        runningMerges.remove(merge);
 
-        // Segments' docScoreOffsets must be in-order,
-        // contiguous.  For the default merge policy now
-        // this will always be the case but for an arbitrary
-        // merge policy this may not be the case
-        if (-1 == next)
-          next = si.getDocStoreOffset() + si.docCount;
-        else if (next != si.getDocStoreOffset())
-          mergeDocStores = true;
-        else
-          next = si.getDocStoreOffset() + si.docCount;
+        // Optimize may be waiting on the final optimize
+        // merge to finish; and finishMerges() may be
+        // waiting for all merges to finish:
+        notifyAll();
+      }
+    }
+  }
+
+  /** Checks whether this merge involves any segments
+   *  already participating in a merge.  If not, this merge
+   *  is "registered", meaning we record that its segments
+   *  are now participating in a merge, and true is
+   *  returned.  Else (the merge conflicts) false is
+   *  returned. */
+  final synchronized boolean registerMerge(MergePolicy.OneMerge merge) {
+
+    if (merge.registerDone)
+      return true;
+
+    final int count = merge.segments.size();
+    boolean isExternal = false;
+    for(int i=0;i<count;i++) {
+      final SegmentInfo info = merge.segments.info(i);
+      if (mergingSegments.contains(info))
+        return false;
+      if (segmentInfos.indexOf(info) == -1)
+        return false;
+      if (info.dir != directory)
+        isExternal = true;
+    }
+
+    pendingMerges.add(merge);
+
+    if (infoStream != null)
+      message("now enroll merge into pendingMerges: " + merge.segString(directory) + " total " + pendingMerges.size());
+
+    merge.mergeGen = mergeGen;
+    merge.isExternal = isExternal;
+
+    // OK it does not conflict; now record that this merge
+    // is running (while synchronized) to avoid race
+    // condition where two conflicting merges from different
+    // threads, start
+    for(int i=0;i<count;i++)
+      mergingSegments.add(merge.segments.info(i));
+
+    // Merge is now registered
+    merge.registerDone = true;
+    return true;
+  }
+
+  /** Does initial setup for a merge, which is fast but holds
+   *  the synchronized lock on IndexWriter instance. */
+  final synchronized void mergeInit(MergePolicy.OneMerge merge) throws IOException {
+
+    // Bind a new segment name here so even with
+    // ConcurrentMergePolicy we keep deterministic segment
+    // names.
+
+    assert merge.registerDone;
+
+    final SegmentInfos sourceSegments = merge.segments;
+    final int end = sourceSegments.size();
+    final int numSegments = segmentInfos.size();
+
+    final int start = ensureContiguousMerge(merge);
+
+    // Check whether this merge will allow us to skip
+    // merging the doc stores (stored field & vectors).
+    // This is a very substantial optimization (saves tons
+    // of IO) that can only be applied with
+    // autoCommit=false.
+
+    Directory lastDir = directory;
+    String lastDocStoreSegment = null;
+    int next = -1;
+
+    boolean mergeDocStores = false;
+    boolean doFlushDocStore = false;
+    final String currentDocStoreSegment = docWriter.getDocStoreSegment();
+
+    // Test each segment to be merged: check if we need to
+    // flush/merge doc stores
+    for (int i = 0; i < end; i++) {
+      SegmentInfo si = sourceSegments.info(i);
+
+      // If it has deletions we must merge the doc stores
+      if (si.hasDeletions())
+        mergeDocStores = true;
+
+      // If it has its own (private) doc stores we must
+      // merge the doc stores
+      if (-1 == si.getDocStoreOffset())
+        mergeDocStores = true;
+
+      // If it has a different doc store segment than
+      // previous segments, we must merge the doc stores
+      String docStoreSegment = si.getDocStoreSegment();
+      if (docStoreSegment == null)
+        mergeDocStores = true;
+      else if (lastDocStoreSegment == null)
+        lastDocStoreSegment = docStoreSegment;
+      else if (!lastDocStoreSegment.equals(docStoreSegment))
+        mergeDocStores = true;
+
+      // Segments' docScoreOffsets must be in-order,
+      // contiguous.  For the default merge policy now
+      // this will always be the case but for an arbitrary
+      // merge policy this may not be the case
+      if (-1 == next)
+        next = si.getDocStoreOffset() + si.docCount;
+      else if (next != si.getDocStoreOffset())
+        mergeDocStores = true;
+      else
+        next = si.getDocStoreOffset() + si.docCount;
       
-        // If the segment comes from a different directory
-        // we must merge
-        if (lastDir != si.dir)
-          mergeDocStores = true;
+      // If the segment comes from a different directory
+      // we must merge
+      if (lastDir != si.dir)
+        mergeDocStores = true;
 
-        // If the segment is referencing the current "live"
-        // doc store outputs then we must merge
-        if (si.getDocStoreOffset() != -1 && si.getDocStoreSegment().equals(docWriter.getDocStoreSegment()))
-          doFlushDocStore = true;
-      }
+      // If the segment is referencing the current "live"
+      // doc store outputs then we must merge
+      if (si.getDocStoreOffset() != -1 && currentDocStoreSegment != null && si.getDocStoreSegment().equals(currentDocStoreSegment))
+        doFlushDocStore = true;
+    }
 
-      final int docStoreOffset;
-      final String docStoreSegment;
-      final boolean docStoreIsCompoundFile;
-      if (mergeDocStores) {
-        docStoreOffset = -1;
-        docStoreSegment = null;
-        docStoreIsCompoundFile = false;
-      } else {
-        SegmentInfo si = segmentInfos.info(minSegment);        
-        docStoreOffset = si.getDocStoreOffset();
-        docStoreSegment = si.getDocStoreSegment();
-        docStoreIsCompoundFile = si.getDocStoreIsCompoundFile();
-      }
+    final int docStoreOffset;
+    final String docStoreSegment;
+    final boolean docStoreIsCompoundFile;
 
-      if (mergeDocStores && doFlushDocStore)
-        // SegmentMerger intends to merge the doc stores
-        // (stored fields, vectors), and at least one of the
-        // segments to be merged refers to the currently
-        // live doc stores.
-        flushDocStores();
+    if (mergeDocStores) {
+      docStoreOffset = -1;
+      docStoreSegment = null;
+      docStoreIsCompoundFile = false;
+    } else {
+      SegmentInfo si = sourceSegments.info(0);        
+      docStoreOffset = si.getDocStoreOffset();
+      docStoreSegment = si.getDocStoreSegment();
+      docStoreIsCompoundFile = si.getDocStoreIsCompoundFile();
+    }
 
-      merger = new SegmentMerger(this, mergedName);
+    if (mergeDocStores && doFlushDocStore) {
+      // SegmentMerger intends to merge the doc stores
+      // (stored fields, vectors), and at least one of the
+      // segments to be merged refers to the currently
+      // live doc stores.
 
-      for (int i = minSegment; i < end; i++) {
-        SegmentInfo si = segmentInfos.info(i);
-        if (infoStream != null)
-          infoStream.print(" " + si.name + " (" + si.docCount + " docs)");
-        IndexReader reader = SegmentReader.get(si, MERGE_READ_BUFFER_SIZE, mergeDocStores); // no need to set deleter (yet)
-        merger.add(reader);
-      }
+      // TODO: if we know we are about to merge away these
+      // newly flushed doc store files then we should not
+      // make compound file out of them...
+      flush(false, true);
+    }
 
-      SegmentInfos rollback = null;
-      boolean success = false;
+    // We must take a full copy at this point so that we can
+    // properly merge deletes in commitMerge()
+    merge.segmentsClone = (SegmentInfos) merge.segments.clone();
 
-      // This is try/finally to rollback our internal state
-      // if we hit exception when doing the merge:
-      try {
+    for (int i = 0; i < end; i++) {
+      SegmentInfo si = merge.segmentsClone.info(i);
 
-        mergedDocCount = merger.merge(mergeDocStores);
+      // IncRef all files for this segment info to make sure
+      // they are not removed while we are trying to merge.
+      if (si.dir == directory)
+        deleter.incRef(si.files());
+    }
 
-        if (infoStream != null) {
-          infoStream.println(" into "+mergedName+" ("+mergedDocCount+" docs)");
-        }
+    merge.increfDone = true;
 
-        newSegment = new SegmentInfo(mergedName, mergedDocCount,
-                                     directory, false, true,
-                                     docStoreOffset,
-                                     docStoreSegment,
-                                     docStoreIsCompoundFile);
-        
-        rollback = (SegmentInfos) segmentInfos.clone();
+    merge.mergeDocStores = mergeDocStores;
+    merge.info = new SegmentInfo(newSegmentName(), 0,
+                                 directory, false, true,
+                                 docStoreOffset,
+                                 docStoreSegment,
+                                 docStoreIsCompoundFile);
+  }
 
-        for (int i = end-1; i > minSegment; i--)     // remove old infos & add new
-          segmentInfos.remove(i);
+  /** Does fininishing for a merge, which is fast but holds
+   *  the synchronized lock on IndexWriter instance. */
+  final synchronized void mergeFinish(MergePolicy.OneMerge merge) throws IOException {
 
-        segmentInfos.set(minSegment, newSegment);
+    if (merge.increfDone)
+      decrefMergeSegments(merge);
 
-        checkpoint();
+    assert merge.registerDone;
 
-        success = true;
+    final SegmentInfos sourceSegments = merge.segments;
+    final SegmentInfos sourceSegmentsClone = merge.segmentsClone;
+    final int end = sourceSegments.size();
+    for(int i=0;i<end;i++)
+      mergingSegments.remove(sourceSegments.info(i));
+    merge.registerDone = false;
+  }
 
-      } finally {
-        if (!success) {
-          if (rollback != null) {
-            // Rollback the individual SegmentInfo
-            // instances, but keep original SegmentInfos
-            // instance (so we don't try to write again the
-            // same segments_N file -- write once):
-            segmentInfos.clear();
-            segmentInfos.addAll(rollback);
-          }
+  /** Does the actual (time-consuming) work of the merge,
+   *  but without holding synchronized lock on IndexWriter
+   *  instance */
+  final private int mergeMiddle(MergePolicy.OneMerge merge) 
+    throws CorruptIndexException, IOException {
 
-          // Delete any partially created and now unreferenced files:
-          deleter.refresh();
-        }
+    final String mergedName = merge.info.name;
+    
+    SegmentMerger merger = null;
+
+    int mergedDocCount = 0;
+
+    SegmentInfos sourceSegments = merge.segments;
+    SegmentInfos sourceSegmentsClone = merge.segmentsClone;
+    final int numSegments = sourceSegments.size();
+
+    if (infoStream != null)
+      message("merging " + merge.segString(directory));
+
+    merger = new SegmentMerger(this, mergedName);
+
+    // This is try/finally to make sure merger's readers are
+    // closed:
+
+    boolean success = false;
+
+    try {
+      int totDocCount = 0;
+      for (int i = 0; i < numSegments; i++) {
+        SegmentInfo si = sourceSegmentsClone.info(i);
+        IndexReader reader = SegmentReader.get(si, MERGE_READ_BUFFER_SIZE, merge.mergeDocStores); // no need to set deleter (yet)
+        merger.add(reader);
+        if (infoStream != null)
+          totDocCount += reader.numDocs();
       }
+      if (infoStream != null) {
+        message("merge: total "+totDocCount+" docs");
+      }
+
+      mergedDocCount = merge.info.docCount = merger.merge(merge.mergeDocStores);
+
+      if (infoStream != null)
+        assert mergedDocCount == totDocCount;
+
+      success = true;
+
     } finally {
-      // close readers before we attempt to delete now-obsolete segments
+      // close readers before we attempt to delete
+      // now-obsolete segments
       if (merger != null) {
         merger.closeReaders();
       }
+      if (!success) {
+        if (infoStream != null)
+          message("hit exception during merge; now refresh deleter on segment " + mergedName);
+        synchronized(this) {
+          addMergeException(merge);
+          deleter.refresh(mergedName);
+        }
+      }
     }
 
-    // Give deleter a chance to remove files now.
-    deleter.checkpoint(segmentInfos, autoCommit);
+    if (!commitMerge(merge))
+      // commitMerge will return false if this merge was aborted
+      return 0;
 
-    if (useCompoundFile) {
+    if (merge.useCompoundFile) {
+      
+      success = false;
+      boolean skip = false;
+      final String compoundFileName = mergedName + "." + IndexFileNames.COMPOUND_FILE_EXTENSION;
 
-      boolean success = false;
-
       try {
+        try {
+          merger.createCompoundFile(compoundFileName);
+          success = true;
+        } catch (IOException ioe) {
+          synchronized(this) {
+            if (segmentInfos.indexOf(merge.info) == -1) {
+              // If another merge kicked in and merged our
+              // new segment away while we were trying to
+              // build the compound file, we can hit a
+              // FileNotFoundException and possibly
+              // IOException over NFS.  We can tell this has
+              // happened because our SegmentInfo is no
+              // longer in the segments; if this has
+              // happened it is safe to ignore the exception
+              // & skip finishing/committing our compound
+              // file creating.
+              skip = true;
+            } else
+              throw ioe;
+          }
+        }
+      } finally {
+        if (!success) {
+          if (infoStream != null)
+            message("hit exception creating compound file during merge: skip=" + skip);
 
-        merger.createCompoundFile(mergedName + ".cfs");
-        newSegment.setUseCompoundFile(true);
-        checkpoint();
-        success = true;
+          synchronized(this) {
+            if (!skip)
+              addMergeException(merge);
+            deleter.deleteFile(compoundFileName);
+          }
+        }
+      }
 
-      } finally {
-        if (!success) {  
-          // Must rollback:
-          newSegment.setUseCompoundFile(false);
-          deleter.refresh();
+      if (!skip) {
+
+        synchronized(this) {
+          if (skip || segmentInfos.indexOf(merge.info) == -1 || merge.isAborted()) {
+            // Our segment (committed in non-compound
+            // format) got merged away while we were
+            // building the compound format.
+            deleter.deleteFile(compoundFileName);
+          } else {
+            success = false;
+            try {
+              merge.info.setUseCompoundFile(true);
+              checkpoint();
+              success = true;
+            } finally {
+              if (!success) {  
+                if (infoStream != null)
+                  message("hit exception checkpointing compound file during merge");
+
+                // Must rollback:
+                addMergeException(merge);
+                merge.info.setUseCompoundFile(false);
+                deletePartialSegmentsFile();
+                deleter.deleteFile(compoundFileName);
+              }
+            }
+      
+            // Give deleter a chance to remove files now.
+            deleter.checkpoint(segmentInfos, autoCommit);
+          }
         }
       }
-      
-      // Give deleter a chance to remove files now.
-      deleter.checkpoint(segmentInfos, autoCommit);
     }
 
     return mergedDocCount;
   }
 
+  void addMergeException(MergePolicy.OneMerge merge) {
+    if (!mergeExceptions.contains(merge) && mergeGen == merge.mergeGen)
+      mergeExceptions.add(merge);
+  }
+
+  private void deletePartialSegmentsFile() throws IOException  {
+    if (segmentInfos.getLastGeneration() != segmentInfos.getGeneration()) {
+      String segmentFileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
+                                                                     "",
+                                                                     segmentInfos.getGeneration());
+      if (infoStream != null)
+        message("now delete partial segments file \"" + segmentFileName + "\"");
+
+      deleter.deleteFile(segmentFileName);
+    }
+  }
+
   // Called during flush to apply any buffered deletes.  If
   // flushedNewSegment is true then a new segment was just
   // created and flushed from the ram segments, so we will
@@ -2284,8 +3022,8 @@
     int delCount = 0;
     if (bufferedDeleteTerms.size() > 0) {
       if (infoStream != null)
-        infoStream.println("flush " + docWriter.getNumBufferedDeleteTerms() + " buffered deleted terms on "
-                           + segmentInfos.size() + " segments.");
+        message("flush " + docWriter.getNumBufferedDeleteTerms() + " buffered deleted terms on "
+                + segmentInfos.size() + " segments.");
 
       if (flushedNewSegment) {
         IndexReader reader = null;
@@ -2341,29 +3079,6 @@
     return delCount;
   }
 
-  private final boolean checkNonDecreasingLevels(int start) {
-    int lowerBound = -1;
-    int upperBound = docWriter.getMaxBufferedDocs();
-
-    /* new merge policy
-    if (upperBound == 0)
-      upperBound = 10;
-    */
-
-    for (int i = segmentInfos.size() - 1; i >= start; i--) {
-      int docCount = segmentInfos.info(i).docCount;
-      if (docCount <= lowerBound) {
-        return false;
-      }
-
-      while (docCount > upperBound) {
-        lowerBound = upperBound;
-        upperBound *= mergeFactor;
-      }
-    }
-    return true;
-  }
-
   // For test purposes.
   final synchronized int getBufferedDeleteTermsSize() {
     return docWriter.getBufferedDeleteTerms().size();
@@ -2417,13 +3132,18 @@
     return delCount;
   }
 
+  // utility routines for tests
+  SegmentInfo newestSegment() {
+    return segmentInfos.info(segmentInfos.size()-1);
+  }
+
   public synchronized String segString() {
     StringBuffer buffer = new StringBuffer();
     for(int i = 0; i < segmentInfos.size(); i++) {
       if (i > 0) {
         buffer.append(' ');
       }
-      buffer.append(segmentInfos.info(i).name + ":" + segmentInfos.info(i).docCount);
+      buffer.append(segmentInfos.info(i).segString(directory));
     }
 
     return buffer.toString();
Index: src/java/org/apache/lucene/index/IndexFileDeleter.java
===================================================================
--- src/java/org/apache/lucene/index/IndexFileDeleter.java	(revision 574266)
+++ src/java/org/apache/lucene/index/IndexFileDeleter.java	(working copy)
@@ -105,7 +105,7 @@
   }
   
   private void message(String message) {
-    infoStream.println(this + " " + Thread.currentThread().getName() + ": " + message);
+    infoStream.println("Deleter [" + Thread.currentThread().getName() + "]: " + message);
   }
 
   /**
@@ -275,25 +275,59 @@
    * Writer calls this when it has hit an error and had to
    * roll back, to tell us that there may now be
    * unreferenced files in the filesystem.  So we re-list
-   * the filesystem and delete such files:
+   * the filesystem and delete such files.  If segmentName
+   * is non-null, we will only delete files corresponding to
+   * that segment.
    */
-  public void refresh() throws IOException {
+  public void refresh(String segmentName) throws IOException {
     String[] files = directory.list();
     if (files == null)
       throw new IOException("cannot read directory " + directory + ": list() returned null");
     IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
+    String segmentPrefix1;
+    String segmentPrefix2;
+    if (segmentName != null) {
+      segmentPrefix1 = segmentName + ".";
+      segmentPrefix2 = segmentName + "_";
+    } else {
+      segmentPrefix1 = null;
+      segmentPrefix2 = null;
+    }
+    
     for(int i=0;i<files.length;i++) {
       String fileName = files[i];
-      if (filter.accept(null, fileName) && !refCounts.containsKey(fileName) && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
+      if (filter.accept(null, fileName) &&
+          (segmentName == null || fileName.startsWith(segmentPrefix1) || fileName.startsWith(segmentPrefix2)) &&
+          !refCounts.containsKey(fileName) &&
+          !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
         // Unreferenced file, so remove it
         if (infoStream != null) {
-          message("refresh: removing newly created unreferenced file \"" + fileName + "\"");
+          message("refresh [prefix=" + segmentName + "]: removing newly created unreferenced file \"" + fileName + "\"");
         }
         deleteFile(fileName);
       }
     }
   }
 
+  public void refresh() throws IOException {
+    refresh(null);
+  }
+
+  public void close() throws IOException {
+    deletePendingFiles();
+  }
+
+  private void deletePendingFiles() throws IOException {
+    if (deletable != null) {
+      List oldDeletable = deletable;
+      deletable = null;
+      int size = oldDeletable.size();
+      for(int i=0;i<size;i++) {
+        deleteFile((String) oldDeletable.get(i));
+      }
+    }
+  }
+
   /**
    * For definition of "check point" see IndexWriter comments:
    * "Clarification: Check Points (and commits)".
@@ -322,19 +356,17 @@
 
     // Try again now to delete any previously un-deletable
     // files (because they were in use, on Windows):
-    if (deletable != null) {
-      List oldDeletable = deletable;
-      deletable = null;
-      int size = oldDeletable.size();
-      for(int i=0;i<size;i++) {
-        deleteFile((String) oldDeletable.get(i));
-      }
-    }
+    deletePendingFiles();
 
     // Incref the files:
     incRef(segmentInfos, isCommit);
-    if (docWriter != null)
-      incRef(docWriter.files());
+    final List docWriterFiles;
+    if (docWriter != null) {
+      docWriterFiles = docWriter.files();
+      if (docWriterFiles != null)
+        incRef(docWriterFiles);
+    } else
+      docWriterFiles = null;
 
     if (isCommit) {
       // Append to our commits list:
@@ -364,9 +396,9 @@
           lastFiles.add(segmentInfo.files());
         }
       }
-      if (docWriter != null)
-        lastFiles.add(docWriter.files());
     }
+    if (docWriterFiles != null)
+      lastFiles.add(docWriterFiles);
   }
 
   void incRef(SegmentInfos segmentInfos, boolean isCommit) throws IOException {
@@ -385,7 +417,7 @@
     }
   }
 
-  private void incRef(List files) throws IOException {
+  void incRef(List files) throws IOException {
     int size = files.size();
     for(int i=0;i<size;i++) {
       String fileName = (String) files.get(i);
@@ -397,7 +429,7 @@
     }
   }
 
-  private void decRef(List files) throws IOException {
+  void decRef(List files) throws IOException {
     int size = files.size();
     for(int i=0;i<size;i++) {
       decRef((String) files.get(i));
@@ -438,7 +470,22 @@
     return rc;
   }
 
-  private void deleteFile(String fileName)
+  void deleteFiles(List files) throws IOException {
+    final int size = files.size();
+    for(int i=0;i<size;i++)
+      deleteFile((String) files.get(i));
+  }
+
+  /** Delets the specified files, but only if they are new
+   *  (have not yet been incref'd). */
+  void deleteNewFiles(List files) throws IOException {
+    final int size = files.size();
+    for(int i=0;i<size;i++)
+      if (!refCounts.containsKey(files.get(i)))
+        deleteFile((String) files.get(i));
+  }
+
+  void deleteFile(String fileName)
        throws IOException {
     try {
       if (infoStream != null) {
@@ -447,7 +494,6 @@
       directory.deleteFile(fileName);
     } catch (IOException e) {			  // if delete fails
       if (directory.fileExists(fileName)) {
-
         // Some operating systems (e.g. Windows) don't
         // permit a file to be deleted while it is opened
         // for read (e.g. by another process or thread). So
@@ -490,11 +536,12 @@
 
     int count;
 
-    final private int IncRef() {
+    final public int IncRef() {
       return ++count;
     }
 
-    final private int DecRef() {
+    final public int DecRef() {
+      assert count > 0;
       return --count;
     }
   }
Index: src/java/org/apache/lucene/index/MergeScheduler.java
===================================================================
--- src/java/org/apache/lucene/index/MergeScheduler.java	(revision 0)
+++ src/java/org/apache/lucene/index/MergeScheduler.java	(revision 0)
@@ -0,0 +1,36 @@
+package org.apache.lucene.index;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+/** Expert: {@link IndexWriter} uses an instance
+ *  implementing this interface to execute the merges
+ *  selected by a {@link MergePolicy}.  The default
+ *  MergeScheduler is {@link SerialMergeScheduler}. */
+
+public interface MergeScheduler {
+
+  /** Run the merges provided by {@link IndexWriter#getNextMerge()}. */
+  void merge(IndexWriter writer)
+    throws CorruptIndexException, IOException;
+
+  /** Close this MergeScheduler. */
+  void close()
+    throws CorruptIndexException, IOException;
+}

Property changes on: src/java/org/apache/lucene/index/MergeScheduler.java
___________________________________________________________________
Name: svn:eol-style
   + native