Index: lucene/core/src/test/org/apache/lucene/index/TestTryDelete.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestTryDelete.java (revision 0) +++ lucene/core/src/test/org/apache/lucene/index/TestTryDelete.java (working copy) @@ -0,0 +1,201 @@ +package org.apache.lucene.index; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ReferenceManager; +import org.apache.lucene.search.SearcherFactory; +import org.apache.lucene.search.SearcherManager; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.LuceneTestCase; + + +public class TestTryDelete extends LuceneTestCase +{ + private static IndexWriter getWriter (Directory directory) + throws IOException + { + LogMergePolicy policy = new LogByteSizeMergePolicy(); + IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random())); + conf.setMergePolicy(policy); + conf.setOpenMode(OpenMode.CREATE_OR_APPEND); + + IndexWriter writer = new IndexWriter(directory, conf); + + return writer; + } + + private static Directory createIndex () + throws IOException + { + Directory directory = new RAMDirectory(); + + IndexWriter writer = getWriter(directory); + + for (int i = 0; i < 10; i++) { + Document doc = new Document(); + doc.add(new StringField("foo", String.valueOf(i), Store.YES)); + writer.addDocument(doc); + } + + writer.commit(); + writer.close(); + + return directory; + } + + public void testTryDeleteDocument () + throws IOException + { + Directory directory = createIndex(); + + IndexWriter writer = getWriter(directory); + + ReferenceManager mgr = new SearcherManager(writer, + true, + new SearcherFactory()); + + TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer); + + IndexSearcher searcher = mgr.acquire(); + + TopDocs topDocs = searcher.search(new TermQuery(new Term("foo", "0")), + 100); + assertEquals(1, topDocs.totalHits); + + long result; + if (random().nextBoolean()) { + IndexReader r = DirectoryReader.open(writer, true); + result = mgrWriter.tryDeleteDocument(r, 0); + r.close(); + } else { + result = mgrWriter.tryDeleteDocument(searcher.getIndexReader(), 0); + } + + // The tryDeleteDocument should have succeeded: + assertTrue(result != -1); + + assertTrue(writer.hasDeletions()); + + if (random().nextBoolean()) { + writer.commit(); + } + + assertTrue(writer.hasDeletions()); + + mgr.maybeRefresh(); + + searcher = mgr.acquire(); + + topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100); + + assertEquals(0, topDocs.totalHits); + } + + public void testTryDeleteDocumentCloseAndReopen () + throws IOException + { + Directory directory = createIndex(); + + IndexWriter writer = getWriter(directory); + + ReferenceManager mgr = new SearcherManager(writer, + true, + new SearcherFactory()); + + IndexSearcher searcher = mgr.acquire(); + + TopDocs topDocs = searcher.search(new TermQuery(new Term("foo", "0")), + 100); + assertEquals(1, topDocs.totalHits); + + TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer); + long result = mgrWriter.tryDeleteDocument(DirectoryReader.open(writer, + true), 0); + + assertEquals(1, result); + + writer.commit(); + + assertTrue(writer.hasDeletions()); + + mgr.maybeRefresh(); + + searcher = mgr.acquire(); + + topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100); + + assertEquals(0, topDocs.totalHits); + + writer.close(); + + searcher = new IndexSearcher(DirectoryReader.open(directory)); + + topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100); + + assertEquals(0, topDocs.totalHits); + + } + + public void testDeleteDocuments () + throws IOException + { + Directory directory = createIndex(); + + IndexWriter writer = getWriter(directory); + + ReferenceManager mgr = new SearcherManager(writer, + true, + new SearcherFactory()); + + IndexSearcher searcher = mgr.acquire(); + + TopDocs topDocs = searcher.search(new TermQuery(new Term("foo", "0")), + 100); + assertEquals(1, topDocs.totalHits); + + TrackingIndexWriter mgrWriter = new TrackingIndexWriter(writer); + long result = mgrWriter.deleteDocuments(new TermQuery(new Term("foo", + "0"))); + + assertEquals(1, result); + + // writer.commit(); + + assertTrue(writer.hasDeletions()); + + mgr.maybeRefresh(); + + searcher = mgr.acquire(); + + topDocs = searcher.search(new TermQuery(new Term("foo", "0")), 100); + + assertEquals(0, topDocs.totalHits); + } +} Property changes on: lucene/core/src/test/org/apache/lucene/index/TestTryDelete.java ___________________________________________________________________ Added: svn:eol-style ## -0,0 +1 ## +native \ No newline at end of property Index: lucene/core/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (revision 1480364) +++ lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -368,7 +368,7 @@ } success = true; // Prevent segmentInfos from changing while opening the - // reader; in theory we could do similar retry logic, + // reader; in theory we could instead do similar retry logic, // just like we do when loading segments_N synchronized(this) { maybeApplyDeletes(applyAllDeletes); @@ -435,6 +435,16 @@ } } + public synchronized boolean anyPendingDeletes() { + for(ReadersAndLiveDocs rld : readerMap.values()) { + if (rld.getPendingDeleteCount() != 0) { + return true; + } + } + + return false; + } + public synchronized void release(ReadersAndLiveDocs rld) throws IOException { // Matches incRef in get: @@ -553,8 +563,21 @@ rld.incRef(); } + assert noDups(); + return rld; } + + // Make sure that every segment appears only once in the + // pool: + private synchronized boolean noDups() { + Set seen = new HashSet(); + for(SegmentInfoPerCommit info : readerMap.keySet()) { + assert !seen.contains(info.info.name); + seen.add(info.info.name); + } + return true; + } } /** @@ -676,8 +699,7 @@ // Record that we have a change (zero out all // segments) pending: - changeCount++; - segmentInfos.changed(); + changed(); } else { segmentInfos.read(directory); @@ -693,8 +715,7 @@ SegmentInfos oldInfos = new SegmentInfos(); oldInfos.read(directory, commit.getSegmentsFileName()); segmentInfos.replace(oldInfos); - changeCount++; - segmentInfos.changed(); + changed(); if (infoStream.isEnabled("IW")) { infoStream.message("IW", "init: loaded commit \"" + commit.getSegmentsFileName() + "\""); } @@ -721,8 +742,7 @@ // We have to mark ourself as changed so that if we // are closed w/o any further changes we write a new // segments_N file. - changeCount++; - segmentInfos.changed(); + changed(); } if (infoStream.isEnabled("IW")) { @@ -1085,6 +1105,9 @@ if (docWriter.anyDeletions()) { return true; } + if (readerPool.anyPendingDeletes()) { + return true; + } for (final SegmentInfoPerCommit info : segmentInfos) { if (info.hasDeletions()) { return true; @@ -1354,7 +1377,7 @@ // Must bump changeCount so if no other changes // happened, we still commit this change: - changeCount++; + changed(); } //System.out.println(" yes " + info.info.name + " " + docID); return true; @@ -2171,9 +2194,14 @@ * index directory. */ synchronized void checkpoint() throws IOException { + changed(); + deleter.checkpoint(segmentInfos, false); + } + + /** Called internally if any index state has changed. */ + synchronized void changed() { changeCount++; segmentInfos.changed(); - deleter.checkpoint(segmentInfos, false); } synchronized void publishFrozenDeletes(FrozenBufferedDeletes packet) { @@ -4113,8 +4141,7 @@ //System.out.println("IW.nrtIsCurrent " + (infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any())); ensureOpen(); if (infoStream.isEnabled("IW")) { - infoStream.message("IW", "nrtIsCurrent: infoVersion matches: " + (infos.version == segmentInfos.version) + " DW changes: " + docWriter.anyChanges() + " BD changes: "+bufferedDeletesStream.any()); - + infoStream.message("IW", "nrtIsCurrent: infoVersion matches: " + (infos.version == segmentInfos.version) + "; DW changes: " + docWriter.anyChanges() + "; BD changes: "+ bufferedDeletesStream.any()); } return infos.version == segmentInfos.version && !docWriter.anyChanges() && !bufferedDeletesStream.any(); } Index: lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (revision 1480364) +++ lucene/core/src/java/org/apache/lucene/index/StandardDirectoryReader.java (working copy) @@ -89,6 +89,10 @@ IOException prior = null; boolean success = false; try { + // NOTE: important that we use infos not + // segmentInfos here, so that we are passing the + // actual instance of SegmentInfoPerCommit in + // IndexWriter's segmentInfos: final SegmentInfoPerCommit info = infos.info(i); assert info.info.dir == dir; final ReadersAndLiveDocs rld = writer.readerPool.get(info, true); @@ -118,9 +122,10 @@ writer, segmentInfos, writer.getConfig().getReaderTermsIndexDivisor(), applyAllDeletes); } - /** This constructor is only used for {@link #doOpenIfChanged(SegmentInfos, IndexWriter)} */ - private static DirectoryReader open(Directory directory, IndexWriter writer, SegmentInfos infos, List oldReaders, + /** This constructor is only used for {@link #doOpenIfChanged(SegmentInfos)} */ + private static DirectoryReader open(Directory directory, SegmentInfos infos, List oldReaders, int termInfosIndexDivisor) throws IOException { + // we put the old SegmentReaders in a map, that allows us // to lookup a reader using its segment name final Map segmentReaders = new HashMap(); @@ -207,7 +212,7 @@ } } } - return new StandardDirectoryReader(directory, newReaders, writer, infos, termInfosIndexDivisor, false); + return new StandardDirectoryReader(directory, newReaders, null, infos, termInfosIndexDivisor, false); } @Override @@ -232,7 +237,7 @@ @Override protected DirectoryReader doOpenIfChanged() throws IOException { - return doOpenIfChanged(null); + return doOpenIfChanged((IndexCommit) null); } @Override @@ -302,13 +307,13 @@ protected Object doBody(String segmentFileName) throws IOException { final SegmentInfos infos = new SegmentInfos(); infos.read(directory, segmentFileName); - return doOpenIfChanged(infos, null); + return doOpenIfChanged(infos); } }.run(commit); } - DirectoryReader doOpenIfChanged(SegmentInfos infos, IndexWriter writer) throws IOException { - return StandardDirectoryReader.open(directory, writer, infos, getSequentialSubReaders(), termInfosIndexDivisor); + DirectoryReader doOpenIfChanged(SegmentInfos infos) throws IOException { + return StandardDirectoryReader.open(directory, infos, getSequentialSubReaders(), termInfosIndexDivisor); } @Override