Index: src/test/org/apache/lucene/index/TestStressIndexing2.java =================================================================== --- src/test/org/apache/lucene/index/TestStressIndexing2.java (revision 1037570) +++ src/test/org/apache/lucene/index/TestStressIndexing2.java (working copy) @@ -90,7 +90,19 @@ dir1.close(); dir2.close(); } - + + public void testMultiConfigMany() throws Throwable { + for (int x=0; x < 500; x++) { + try { + testMultiConfig(); + } catch (Throwable th) { + System.err.println("x:"+x); + //th.printStackTrace(); + throw th; + } + } + } + public void testMultiConfig() throws Throwable { // test lots of smaller different params together Index: src/java/org/apache/lucene/index/BufferedDeletes.java =================================================================== --- src/java/org/apache/lucene/index/BufferedDeletes.java (revision 1037570) +++ src/java/org/apache/lucene/index/BufferedDeletes.java (working copy) @@ -90,7 +90,19 @@ docIDs.addAll(in.docIDs); in.clear(); } + + void updateSetLimit(BufferedDeletes in, Integer limit) { + numTerms += in.numTerms; + bytesUsed += in.bytesUsed; + for (Term term : in.terms.keySet()) { + terms.put(term, new Num(limit)); + } + for (Query query : in.queries.keySet()) { + queries.put(query, limit); + } + } + void clear() { terms.clear(); queries.clear(); Index: src/java/org/apache/lucene/index/SegmentInfo.java =================================================================== --- src/java/org/apache/lucene/index/SegmentInfo.java (revision 1037570) +++ src/java/org/apache/lucene/index/SegmentInfo.java (working copy) @@ -39,7 +39,7 @@ * @lucene.experimental */ public final class SegmentInfo { - + BufferedDeletes deletes = new BufferedDeletes(true); static final int NO = -1; // e.g. no norms; no deletes; static final int YES = 1; // e.g. have norms; have deletes; static final int WITHOUT_GEN = 0; // a file name that has no GEN in it. Index: src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- src/java/org/apache/lucene/index/DocumentsWriter.java (revision 1037570) +++ src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -115,7 +115,8 @@ */ final class DocumentsWriter { - + private static final boolean PERSEGMENTDELETES = false; + private static final Integer NO_LIMIT = new Integer(Integer.MAX_VALUE); final AtomicLong bytesUsed = new AtomicLong(0); IndexWriter writer; Directory directory; @@ -303,7 +304,9 @@ private int maxBufferedDocs = IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS; private int flushedDocCount; // How many docs already flushed to index - + + SegmentInfos segmentInfos; + synchronized void updateFlushedDocCount(int n) { flushedDocCount += n; } @@ -715,6 +718,18 @@ synchronized void pushDeletes() { deletesFlushed.update(deletesInRAM); } + + synchronized void pushSegmentDeletes(SegmentInfo info) { + info.deletes.update(deletesFlushed); + } + + synchronized void pushSegmentInfos(SegmentInfos infos) { + SegmentInfos copy = new SegmentInfos(); + for (SegmentInfo info : infos) { + copy.add(info); + } + this.segmentInfos = copy; + } synchronized void close() { closed = true; @@ -1016,7 +1031,7 @@ return deletesFlushed.any(); } - synchronized boolean applyDeletes(SegmentInfos infos) throws IOException { + synchronized boolean applyDeletes(SegmentInfos infos, SegmentInfo flushedInfo) throws IOException { if (!hasDeletes()) return false; @@ -1030,7 +1045,9 @@ + infos.size() + " segments."); final int infosEnd = infos.size(); - + + SegmentInfo startInfo = infos.get(0); + int docStart = 0; boolean any = false; for (int i = 0; i < infosEnd; i++) { @@ -1041,7 +1058,14 @@ SegmentReader reader = writer.readerPool.get(infos.info(i), false); try { - any |= applyDeletes(reader, docStart); + if (PERSEGMENTDELETES) { + System.out.println("applyDeletes per-segment"); + BufferedDeletes segmentDeletes = getDeletesSegmentsForward(infos, startInfo, flushedInfo); + any |= applyDeletes(reader, docStart, segmentDeletes); + } else { + System.out.println("applyDeletes regular"); + any |= applyDeletes(reader, docStart, deletesFlushed); + } docStart += reader.maxDoc(); } finally { writer.readerPool.release(reader); @@ -1055,7 +1079,26 @@ return any; } - + + synchronized BufferedDeletes getDeletesSegmentsForward(SegmentInfos infos, SegmentInfo startInfo, SegmentInfo flushedInfo) { + BufferedDeletes aggregateDeletes = new BufferedDeletes(true); + // keep the docid-upto for the first segment + int idx = infos.indexOf(startInfo); + for (int x=idx; x < infos.size(); x++) { + SegmentInfo info = infos.info(x); + // the limit on the go forward segments + // is none + aggregateDeletes.updateSetLimit(info.deletes, NO_LIMIT); + //dels.updateSetLimit(info.deletes, NO_LIMIT); + } + //if (flushedInfo != null && !flushedInfo.equals(startInfo)) { + //if (flushedInfo == null) { + aggregateDeletes.updateSetLimit(deletesInRAM, NO_LIMIT); + //} + //} + return aggregateDeletes; + } + // used only by assert private Term lastDeleteTerm; @@ -1070,7 +1113,7 @@ // Apply buffered delete terms, queries and docIDs to the // provided reader - private final synchronized boolean applyDeletes(IndexReader reader, int docIDStart) + private final synchronized boolean applyDeletes(IndexReader reader, int docIDStart, BufferedDeletes deletes) throws CorruptIndexException, IOException { final int docEnd = docIDStart + reader.maxDoc(); @@ -1079,7 +1122,7 @@ assert checkDeleteTerm(null); // Delete by term - if (deletesFlushed.terms.size() > 0) { + if (deletes.terms.size() > 0) { Fields fields = reader.fields(); if (fields == null) { // This reader has no postings @@ -1091,7 +1134,7 @@ String currentField = null; DocsEnum docs = null; - for (Entry entry: deletesFlushed.terms.entrySet()) { + for (Entry entry: deletes.terms.entrySet()) { Term term = entry.getKey(); // Since we visit terms sorted, we gain performance // by re-using the same TermsEnum and seeking only @@ -1132,7 +1175,7 @@ } // Delete by docID - for (Integer docIdInt : deletesFlushed.docIDs) { + for (Integer docIdInt : deletes.docIDs) { int docID = docIdInt.intValue(); if (docID >= docIDStart && docID < docEnd) { reader.deleteDocument(docID-docIDStart); @@ -1141,10 +1184,10 @@ } // Delete by query - if (deletesFlushed.queries.size() > 0) { + if (deletes.queries.size() > 0) { IndexSearcher searcher = new IndexSearcher(reader); try { - for (Entry entry : deletesFlushed.queries.entrySet()) { + for (Entry entry : deletes.queries.entrySet()) { Query query = entry.getKey(); int limit = entry.getValue().intValue(); Weight weight = query.weight(searcher); Index: src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- src/java/org/apache/lucene/index/IndexWriter.java (revision 1037570) +++ src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -3416,6 +3416,7 @@ docWriter.pushDeletes(); if (flushDocs) { + docWriter.pushSegmentDeletes(newSegment); segmentInfos.add(newSegment); checkpoint(); } @@ -3439,7 +3440,7 @@ } if (flushDeletes) { - applyDeletes(); + applyDeletes(newSegment); } if (flushDocs) @@ -3814,7 +3815,7 @@ if (merge.isAborted()) return; - applyDeletes(); + applyDeletes(null); final SegmentInfos sourceSegments = merge.segments; final int end = sourceSegments.size(); @@ -4277,7 +4278,7 @@ } // Apply buffered deletes to all segments. - private final synchronized boolean applyDeletes() throws CorruptIndexException, IOException { + private final synchronized boolean applyDeletes(SegmentInfo flushedInfo) throws CorruptIndexException, IOException { assert testPoint("startApplyDeletes"); if (infoStream != null) { message("applyDeletes"); @@ -4286,7 +4287,7 @@ boolean success = false; boolean changed; try { - changed = docWriter.applyDeletes(segmentInfos); + changed = docWriter.applyDeletes(segmentInfos, flushedInfo); success = true; } finally { if (!success && infoStream != null) {