Index: lucene/src/test/org/apache/lucene/search/QueryUtils.java =================================================================== --- lucene/src/test/org/apache/lucene/search/QueryUtils.java (revision 1062425) +++ lucene/src/test/org/apache/lucene/search/QueryUtils.java (working copy) @@ -172,6 +172,7 @@ } w.commit(); w.deleteDocuments( new MatchAllDocsQuery() ); + w.keepFullyDeletedSegments(); w.commit(); if (0 < numDeletedDocs) Index: lucene/src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReader.java (revision 1062425) +++ lucene/src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -359,7 +359,7 @@ // CREATE A NEW READER and re-test reader = IndexReader.open(dir, false); - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm)); + assertEquals("deleted docFreq", 0, reader.docFreq(searchTerm)); assertTermDocsCount("deleted termDocs", reader, searchTerm, 0); reader.close(); reader2.close(); @@ -692,7 +692,6 @@ // CREATE A NEW READER and re-test reader = IndexReader.open(dir, false); - assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm)); assertEquals("deleted docFreq", 100, reader.docFreq(searchTerm2)); assertTermDocsCount("deleted termDocs", reader, searchTerm, 0); assertTermDocsCount("deleted termDocs", reader, searchTerm2, 100); @@ -833,7 +832,6 @@ writer.close(); IndexReader reader = IndexReader.open(dir, false); reader.deleteDocument(0); - reader.deleteDocument(1); reader.close(); reader = IndexReader.open(dir, false); reader.undeleteAll(); @@ -850,7 +848,6 @@ writer.close(); IndexReader reader = IndexReader.open(dir, false); reader.deleteDocument(0); - reader.deleteDocument(1); reader.close(); reader = IndexReader.open(dir, false); reader.undeleteAll(); @@ -1284,9 +1281,6 @@ // Open another reader to confirm that everything is deleted reader2 = IndexReader.open(dir, false); - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm1)); - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm2)); - assertEquals("reopened 2", 100, reader2.docFreq(searchTerm3)); assertTermDocsCount("reopened 2", reader2, searchTerm1, 0); assertTermDocsCount("reopened 2", reader2, searchTerm2, 0); assertTermDocsCount("reopened 2", reader2, searchTerm3, 100); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1062425) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -101,19 +101,12 @@ } reader.close(); - // test doc count before segments are merged/index is optimized - writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer())); - assertEquals(100, writer.maxDoc()); - writer.close(); - reader = IndexReader.open(dir, true); - assertEquals(100, reader.maxDoc()); assertEquals(60, reader.numDocs()); reader.close(); // optimize the index and check that the new doc count is correct writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer())); - assertEquals(100, writer.maxDoc()); assertEquals(60, writer.numDocs()); writer.optimize(); assertEquals(60, writer.maxDoc()); @@ -1431,7 +1424,6 @@ w.close(); IndexReader ir = IndexReader.open(dir, true); - assertEquals(1, ir.maxDoc()); assertEquals(0, ir.numDocs()); ir.close(); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 1062425) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (working copy) @@ -567,24 +567,25 @@ System.out.println("TEST: open reader"); } IndexReader reader = IndexReader.open(dir, true); - int expected = 3+(1-i)*2; - assertEquals(expected, reader.docFreq(new Term("contents", "here"))); - assertEquals(expected, reader.maxDoc()); - int numDel = 0; - final Bits delDocs = MultiFields.getDeletedDocs(reader); - assertNotNull(delDocs); - for(int j=0;j commits = IndexReader.listCommits(dir); for (final IndexCommit commit : commits) { Index: lucene/src/test/org/apache/lucene/index/TestAddIndexes.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (revision 1062425) +++ lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (working copy) @@ -428,7 +428,7 @@ ); writer.addIndexes(aux, new MockDirectoryWrapper(random, new RAMDirectory(aux))); - assertEquals(1060, writer.maxDoc()); + assertEquals(1020, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); dir.close(); @@ -480,7 +480,7 @@ ); writer.addIndexes(aux, aux2); - assertEquals(1060, writer.maxDoc()); + assertEquals(1040, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); dir.close(); Index: lucene/src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DirectoryReader.java (revision 1062425) +++ lucene/src/java/org/apache/lucene/index/DirectoryReader.java (working copy) @@ -710,6 +710,9 @@ for (int i = 0; i < subReaders.length; i++) subReaders[i].commit(); + // Remove segments that contain only 100% deleted docs: + segmentInfos.pruneDeletedSegments(); + // Sync all files we just wrote directory.sync(segmentInfos.files(directory, false)); segmentInfos.commit(directory); Index: lucene/src/java/org/apache/lucene/index/BufferedDeletes.java =================================================================== --- lucene/src/java/org/apache/lucene/index/BufferedDeletes.java (revision 1062425) +++ lucene/src/java/org/apache/lucene/index/BufferedDeletes.java (working copy) @@ -270,9 +270,9 @@ } private synchronized long applyDeletes(IndexWriter.ReaderPool readerPool, - SegmentInfo info, - SegmentDeletes coalescedDeletes, - SegmentDeletes segmentDeletes) throws IOException { + SegmentInfo info, + SegmentDeletes coalescedDeletes, + SegmentDeletes segmentDeletes) throws IOException { assert readerPool.infoIsLive(info); assert coalescedDeletes == null || coalescedDeletes.docIDs.size() == 0; Index: lucene/src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexReader.java (revision 1062425) +++ lucene/src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -1163,8 +1163,15 @@ return n; } - /** Undeletes all documents currently marked as deleted in this index. + /** Undeletes all documents currently marked as deleted in + * this index. * + *

NOTE: this is only a best-effort process. For + * example, if all documents in a given segment were + * deleted, Lucene now drops that segment from the index, + * which means its documents will not be recovered by this + * method. + * * @throws StaleReaderException if the index has changed * since this reader was opened * @throws LockObtainFailedException if another writer Index: lucene/src/java/org/apache/lucene/index/SegmentInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfos.java (revision 1062425) +++ lucene/src/java/org/apache/lucene/index/SegmentInfos.java (working copy) @@ -308,6 +308,19 @@ } } + /** Prunes any segment whose docs are all deleted. */ + public void pruneDeletedSegments() { + int segIdx = 0; + while(segIdx < size()) { + final SegmentInfo info = info(segIdx); + if (info.getDelCount() == info.docCount) { + remove(segIdx); + } else { + segIdx++; + } + } + } + /** * Returns a copy of this instance, also copying each * SegmentInfo. Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 1062876) +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -3276,6 +3276,13 @@ } } + private boolean keepFullyDeletedSegments; + + /** Only for testing */ + public void keepFullyDeletedSegments() { + keepFullyDeletedSegments = true; + } + // called only from assert private boolean filesExist(SegmentInfos toSync) throws IOException { Collection files = toSync.files(directory, false); @@ -3334,6 +3341,10 @@ readerPool.commit(); toSync = (SegmentInfos) segmentInfos.clone(); + if (!keepFullyDeletedSegments) { + toSync.pruneDeletedSegments(); + } + assert filesExist(toSync); if (commitUserData != null)