Index: lucene/src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (revision 1149727) +++ lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -309,6 +309,9 @@ } private boolean postUpdate(DocumentsWriterPerThread flushingDWPT, boolean maybeMerge) throws IOException { + if (flushControl.doApplyAllDeletes()) { + applyAllDeletes(deleteQueue); + } if (flushingDWPT != null) { maybeMerge |= doFlush(flushingDWPT); } else { Index: lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (revision 1149727) +++ lucene/src/java/org/apache/lucene/index/DocumentsWriterFlushControl.java (working copy) @@ -339,7 +339,11 @@ * Returns the number of delete terms in the global pool */ public int getNumGlobalTermDeletes() { - return documentsWriter.deleteQueue.numGlobalTermDeletes(); + return documentsWriter.deleteQueue.numGlobalTermDeletes() + documentsWriter.indexWriter.bufferedDeletesStream.numTerms(); + } + + public long getDeleteBytesUsed() { + return documentsWriter.deleteQueue.bytesUsed() + documentsWriter.indexWriter.bufferedDeletesStream.bytesUsed(); } synchronized int numFlushingDWPT() { Index: lucene/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java (revision 1149727) +++ lucene/src/java/org/apache/lucene/index/FlushByRamOrCountsPolicy.java (working copy) @@ -65,7 +65,7 @@ // prevent too-frequent flushing of a long tail of // tiny segments: if ((flushOnRAM() && - writer.deleteQueue.bytesUsed() > (1024*1024*indexWriterConfig.getRAMBufferSizeMB()/2))) { + control.getDeleteBytesUsed() > (1024*1024*indexWriterConfig.getRAMBufferSizeMB()/2))) { control.setApplyAllDeletes(); if (writer.infoStream != null) { writer.message("force apply deletes bytesUsed=" + writer.deleteQueue.bytesUsed() + " vs ramBuffer=" + (1024*1024*indexWriterConfig.getRAMBufferSizeMB())); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (revision 1149727) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (working copy) @@ -944,4 +944,88 @@ w.close(); dir.close(); } + + // LUCENE-3340: make sure deletes that we don't apply + // during flush (ie are just pushed into the stream) are + // in fact later flushed due to their RAM usage: + public void testFlushPushedDeletesByRAM() throws Exception { + Directory dir = newDirectory(); + // Cannot use RandomIndexWriter because we don't want to + // ever call commit() for this test: + IndexWriter w = new IndexWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setRAMBufferSizeMB(1.0f).setMaxBufferedDocs(1000).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).setReaderPooling(false)); + w.setInfoStream(VERBOSE ? System.out : null); + int count = 0; + while(true) { + Document doc = new Document(); + doc.add(new Field("id", count+"", Field.Store.NO, Field.Index.NOT_ANALYZED)); + final Term delTerm; + if (count == 1010) { + // This is the only delete that applies + delTerm = new Term("id", ""+0); + } else { + // These get buffered, taking up RAM, but delete + // nothing when applied: + delTerm = new Term("id", "x" + count); + } + w.updateDocument(delTerm, doc); + // Eventually segment 0 should get a del docs: + if (dir.fileExists("_0_1.del")) { + if (VERBOSE) { + System.out.println("TEST: deletes created @ count=" + count); + } + break; + } + count++; + + // Today we applyDelets @ count=7199; even if we make + // sizable improvements to RAM efficiency of buffered + // del term we're unlikely to go over 100K: + if (count > 100000) { + fail("delete's were not applied"); + } + } + w.close(); + dir.close(); + } + + // LUCENE-3340: make sure deletes that we don't apply + // during flush (ie are just pushed into the stream) are + // in fact later flushed due to their RAM usage: + public void testFlushPushedDeletesByCount() throws Exception { + Directory dir = newDirectory(); + // Cannot use RandomIndexWriter because we don't want to + // ever call commit() for this test: + final int flushAtDelCount = atLeast(1020); + IndexWriter w = new IndexWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setMaxBufferedDeleteTerms(flushAtDelCount).setMaxBufferedDocs(1000).setRAMBufferSizeMB(IndexWriterConfig.DISABLE_AUTO_FLUSH).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES).setReaderPooling(false)); + w.setInfoStream(VERBOSE ? System.out : null); + int count = 0; + while(true) { + Document doc = new Document(); + doc.add(new Field("id", count+"", Field.Store.NO, Field.Index.NOT_ANALYZED)); + final Term delTerm; + if (count == 1010) { + // This is the only delete that applies + delTerm = new Term("id", ""+0); + } else { + // These get buffered, taking up RAM, but delete + // nothing when applied: + delTerm = new Term("id", "x" + count); + } + w.updateDocument(delTerm, doc); + // Eventually segment 0 should get a del docs: + if (dir.fileExists("_0_1.del")) { + break; + } + count++; + if (count > flushAtDelCount) { + fail("delete's were not applied at count=" + flushAtDelCount); + } + } + w.close(); + dir.close(); + } }