Index: lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
--- lucene/src/java/org/apache/lucene/index/DocumentsWriter.java	Sat Jan 29 09:26:17 2011 -0500
+++ lucene/src/java/org/apache/lucene/index/DocumentsWriter.java	Sat Jan 29 11:15:04 2011 -0500
@@ -35,9 +35,11 @@
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMFile;
 import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitVector;
+import org.apache.lucene.util.RamUsageEstimator;
 import org.apache.lucene.util.RecyclingByteBlockAllocator;
 import org.apache.lucene.util.ThreadInterruptedException;
-import org.apache.lucene.util.RamUsageEstimator;
+
 import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_MASK;
 import static org.apache.lucene.util.ByteBlockPool.BYTE_BLOCK_SIZE;
 
@@ -577,7 +579,29 @@
 
       final SegmentWriteState flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos,
                                                                  numDocs, writer.getConfig().getTermIndexInterval(),
-                                                                 SegmentCodecs.build(fieldInfos, writer.codecs));
+                                                                 SegmentCodecs.build(fieldInfos, writer.codecs),
+                                                                 pendingDeletes);
+      // Apply delete-by-docID now (delete-byDocID only
+      // happens when an exception is hit processing that
+      // doc, eg if analyzer has some problem w/ the text):
+      if (pendingDeletes.docIDs.size() > 0) {
+        // nocommit -- must remove where we apply docIDs
+        // now!  it should never happen; put assert
+        flushState.deletedDocs = new BitVector(numDocs);
+        for(int delDocID : pendingDeletes.docIDs) {
+          flushState.deletedDocs.set(delDocID);
+        }
+        pendingDeletes.bytesUsed.addAndGet(-pendingDeletes.docIDs.size() * SegmentDeletes.BYTES_PER_DEL_DOCID);
+        pendingDeletes.docIDs.clear();
+      }
+
+      // nocommit -- if nothing is left in the
+      // pendingDeletes (ie there are no buffered
+      // del-by-Query) then don't push it!
+
+      // nocommit -- on push, cutover to more efficient
+      // structure; eg just sort the terms into an array?
+      // (after landing delGen) -- it's frozen at that point
 
       newSegment = new SegmentInfo(segment, numDocs, directory, false, fieldInfos.hasProx(), flushState.segmentCodecs, false);
 
@@ -589,10 +613,14 @@
       double startMBUsed = bytesUsed()/1024./1024.;
 
       consumer.flush(threads, flushState);
+
       newSegment.setHasVectors(flushState.hasVectors);
 
       if (infoStream != null) {
         message("new segment has " + (flushState.hasVectors ? "vectors" : "no vectors"));
+        if (flushState.deletedDocs != null) {
+          message("new segment has " + flushState.deletedDocs.count() + " deleted docs");
+        }
         message("flushedFiles=" + newSegment.files());
         message("flushed codecs=" + newSegment.getSegmentCodecs());
       }
@@ -613,6 +641,30 @@
         newSegment.setUseCompoundFile(true);
       }
 
+      // Must write deleted docs after the CFS so we don't
+      // slurp the del file into CFS:
+      if (flushState.deletedDocs != null) {
+        final int delCount = flushState.deletedDocs.count();
+        assert delCount > 0;
+        newSegment.setDelCount(delCount);
+        newSegment.advanceDelGen();
+        final String delFileName = newSegment.getDelFileName();
+        boolean success2 = false;
+        try {
+          flushState.deletedDocs.write(directory, delFileName);
+          success2 = true;
+        } finally {
+          if (!success2) {
+            try {
+              directory.deleteFile(delFileName);
+            } catch (Throwable t) {
+              // suppress this so we keep throwing the
+              // original exception
+            }
+          }
+        }
+      }
+
       if (infoStream != null) {
         message("flush: segment=" + newSegment);
         final double newSegmentSizeNoStore = newSegment.sizeInBytes(false)/1024./1024.;
Index: lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java
--- lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java	Sat Jan 29 09:26:17 2011 -0500
+++ lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java	Sat Jan 29 11:15:04 2011 -0500
@@ -26,8 +26,9 @@
 
 import org.apache.lucene.index.codecs.FieldsConsumer;
 import org.apache.lucene.index.codecs.PostingsConsumer;
+import org.apache.lucene.index.codecs.TermStats;
 import org.apache.lucene.index.codecs.TermsConsumer;
-import org.apache.lucene.index.codecs.TermStats;
+import org.apache.lucene.util.BitVector;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.CollectionUtil;
 
@@ -108,7 +109,7 @@
 
       // If this field has postings then add them to the
       // segment
-      appendPostings(fields, consumer);
+      appendPostings(fieldName, state, fields, consumer);
 
       for(int i=0;i<fields.length;i++) {
         TermsHashPerField perField = fields[i].termsHashPerField;
@@ -133,7 +134,8 @@
   /* Walk through all unique text tokens (Posting
    * instances) found in this field and serialize them
    * into a single RAM segment. */
-  void appendPostings(FreqProxTermsWriterPerField[] fields,
+  void appendPostings(String fieldName, SegmentWriteState state,
+                      FreqProxTermsWriterPerField[] fields,
                       FieldsConsumer consumer)
     throws CorruptIndexException, IOException {
 
@@ -156,11 +158,20 @@
       assert result;
     }
 
+    final Term protoTerm = new Term(fieldName);
+
     FreqProxFieldMergeState[] termStates = new FreqProxFieldMergeState[numFields];
 
     final boolean currentFieldOmitTermFreqAndPositions = fields[0].fieldInfo.omitTermFreqAndPositions;
     //System.out.println("flush terms field=" + fields[0].fieldInfo.name);
 
+    final Map<Term,Integer> segDeletes;
+    if (state.segDeletes != null && state.segDeletes.terms.size() > 0) {
+      segDeletes = state.segDeletes.terms;
+    } else {
+      segDeletes = null;
+    }
+
     // TODO: really TermsHashPerField should take over most
     // of this loop, including merge sort of terms from
     // multiple threads and interacting with the
@@ -195,6 +206,26 @@
 
       final PostingsConsumer postingsConsumer = termsConsumer.startTerm(text);
 
+      // nocommit -- if docIDUpto is 0, don't buffer in the map!
+
+      final int delDocLimit;
+      if (segDeletes != null) {
+        // nocommit -- cutover to simple Map for these
+        // buffered del terms:
+        // nocommit -- when buffering a term into this
+        // map, we should skip it if the term is not
+        // indexed
+        // nocommit -- cutover to BytesRefHash
+        final Integer docIDUpto = segDeletes.get(protoTerm.createTerm(text));
+        if (docIDUpto != null) {
+          delDocLimit = docIDUpto;
+        } else {
+          delDocLimit = 0;
+        }
+      } else {
+        delDocLimit = 0;
+      }
+
       // Now termStates has numToMerge FieldMergeStates
       // which all share the same term.  Now we must
       // interleave the docID streams.
@@ -214,7 +245,28 @@
 
         assert minState.docID < flushedDocCount: "doc=" + minState.docID + " maxDoc=" + flushedDocCount;
 
+        // NOTE: we could check here if the docID was
+        // deleted, and skip it.  However, this is somewhat
+        // dangerous because it can yield non-deterministic
+        // behavior since we may see the docID before we see
+        // the term that caused it to be deleted.  This
+        // would mean some (but not all) of its postings may
+        // make it into the index, which'd alter the docFreq
+        // for those terms.  We could fix this by doing two
+        // passes, ie first sweep marks all del docs, and
+        // 2nd sweep does the real flush, but I suspect
+        // that'd add too much time to flush.
+
         postingsConsumer.startDoc(minState.docID, termDocFreq);
+        if (minState.docID < delDocLimit) {
+          // Mark it deleted.  TODO: we could also skip
+          // writing its postings; this would be
+          // deterministic (just for this Term's docs).
+          if (state.deletedDocs == null) {
+            state.deletedDocs = new BitVector(state.numDocs);
+          }
+          state.deletedDocs.set(minState.docID);
+        }
 
         final ByteSliceReader prox = minState.prox;
 
Index: lucene/src/java/org/apache/lucene/index/SegmentMerger.java
--- lucene/src/java/org/apache/lucene/index/SegmentMerger.java	Sat Jan 29 09:26:17 2011 -0500
+++ lucene/src/java/org/apache/lucene/index/SegmentMerger.java	Sat Jan 29 11:15:04 2011 -0500
@@ -266,7 +266,7 @@
       // details.
       throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption");
 
-    segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo);
+    segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo, null);
     
     return docCount;
   }
Index: lucene/src/java/org/apache/lucene/index/SegmentWriteState.java
--- lucene/src/java/org/apache/lucene/index/SegmentWriteState.java	Sat Jan 29 09:26:17 2011 -0500
+++ lucene/src/java/org/apache/lucene/index/SegmentWriteState.java	Sat Jan 29 11:15:04 2011 -0500
@@ -18,8 +18,10 @@
  */
 
 import java.io.PrintStream;
+import java.util.Map;
 
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BitVector;
 
 /**
  * @lucene.experimental
@@ -32,6 +34,16 @@
   public final int numDocs;
   public boolean hasVectors;
 
+  // Deletes to apply while we are flushing the segment.  A
+  // Term is enrolled in here if it was deleted at one
+  // point, and it's mapped to the docIDUpto, meaning any
+  // docID < docIDUpto containing this term should be
+  // deleted.
+  public final SegmentDeletes segDeletes;
+
+  // Lazily created:
+  public BitVector deletedDocs;
+
   final SegmentCodecs segmentCodecs;
   public final String codecId;
 
@@ -57,8 +69,9 @@
 
 
   public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos,
-                           int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs) {
+                           int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, SegmentDeletes segDeletes) {
     this.infoStream = infoStream;
+    this.segDeletes = segDeletes;
     this.directory = directory;
     this.segmentName = segmentName;
     this.fieldInfos = fieldInfos;
@@ -80,5 +93,6 @@
     termIndexInterval = state.termIndexInterval;
     segmentCodecs = state.segmentCodecs;
     this.codecId = codecId;
+    segDeletes = state.segDeletes;
   }
 }
Index: lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
--- lucene/src/test/org/apache/lucene/index/TestAddIndexes.java	Sat Jan 29 09:26:17 2011 -0500
+++ lucene/src/test/org/apache/lucene/index/TestAddIndexes.java	Sat Jan 29 11:15:04 2011 -0500
@@ -157,6 +157,7 @@
 
     setUpDirs(dir, aux);
     IndexWriter writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND));
+    writer.setInfoStream(VERBOSE ? System.out : null);
     writer.addIndexes(aux);
 
     // Adds 10 docs, then replaces them with another 10
Index: lucene/src/test/org/apache/lucene/index/TestCodecs.java
--- lucene/src/test/org/apache/lucene/index/TestCodecs.java	Sat Jan 29 09:26:17 2011 -0500
+++ lucene/src/test/org/apache/lucene/index/TestCodecs.java	Sat Jan 29 11:15:04 2011 -0500
@@ -589,7 +589,7 @@
 
     final int termIndexInterval = _TestUtil.nextInt(random, 13, 27);
     final SegmentCodecs codecInfo = SegmentCodecs.build(fieldInfos, CodecProvider.getDefault());
-    final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, 10000, termIndexInterval, codecInfo);
+    final SegmentWriteState state = new SegmentWriteState(null, dir, SEGMENT, fieldInfos, 10000, termIndexInterval, codecInfo, null);
 
     final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state);
     Arrays.sort(fields);