These changes are buffered in memory and periodically - flushed to the {@link Directory} (during the above method calls). A flush is triggered when there are - enough buffered deletes (see {@link - #setMaxBufferedDeleteTerms}) or enough added documents - (see {@link #setMaxBufferedDocs}) since the last flush, + flushed to the {@link Directory} (during the above method + calls). A flush is triggered when there are enough + buffered deletes (see {@link #setMaxBufferedDeleteTerms}) + or enough added documents (see {@link #setMaxBufferedDocs} + and {@link #setRAMBufferSizeMB}) since the last flush, whichever is sooner. When a flush occurs, both pending deletes and added documents are flushed to the index. A flush may also trigger one or more segment merges.
@@ -171,11 +172,17 @@ public final static int DEFAULT_MERGE_FACTOR = 10; /** - * Default value is 10. Change using {@link #setMaxBufferedDocs(int)}. + * Default value is 0 (meaning flush is based on RAM usage + * by default). Change using {@link #setMaxBufferedDocs}. */ - public final static int DEFAULT_MAX_BUFFERED_DOCS = 10; + public final static int DEFAULT_MAX_BUFFERED_DOCS = 0; /** + * Default value is 16 MB. Change using {@link #setRAMBufferSizeMB}. + */ + public final static float DEFAULT_RAM_BUFFER_SIZE_MB = 16F; + + /** * Default value is 1000. Change using {@link #setMaxBufferedDeleteTerms(int)}. */ public final static int DEFAULT_MAX_BUFFERED_DELETE_TERMS = 1000; @@ -208,8 +215,7 @@ private boolean autoCommit = true; // false if we should commit only on close SegmentInfos segmentInfos = new SegmentInfos(); // the segments - SegmentInfos ramSegmentInfos = new SegmentInfos(); // the segments in ramDirectory - private final RAMDirectory ramDirectory = new RAMDirectory(); // for temp segs + private DocumentsWriter docWriter; private IndexFileDeleter deleter; private Lock writeLock; @@ -563,6 +569,10 @@ } } + IndexFileDeleter getDeleter() { + return deleter; + } + private void init(Directory d, Analyzer a, final boolean create, boolean closeDir, IndexDeletionPolicy deletionPolicy, boolean autoCommit) throws CorruptIndexException, LockObtainFailedException, IOException { this.closeDir = closeDir; @@ -602,11 +612,14 @@ rollbackSegmentInfos = (SegmentInfos) segmentInfos.clone(); } + docWriter = new DocumentsWriter(newSegmentName(), directory, this, !autoCommit); + docWriter.setInfoStream(infoStream); + // Default deleter (for backwards compatibility) is // KeepOnlyLastCommitDeleter: deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, - segmentInfos, infoStream); + segmentInfos, infoStream, docWriter); } catch (IOException e) { this.writeLock.release(); @@ -661,19 +674,26 @@ } /** Determines the minimal number of documents required before the buffered - * in-memory documents are merged and a new Segment is created. + * in-memory documents are flushed as a new Segment. * Since Documents are merged in a {@link org.apache.lucene.store.RAMDirectory}, * large value gives faster indexing. At the same time, mergeFactor limits * the number of files open in a FSDirectory. * - *The default value is 10. + *
If this is 0, then the RAM buffer is flushed instead + * by overally RAM usage (see {@link + * #setRAMBufferSizeMB}). If this is non-zero, then + * flushing is triggered by maxBufferedDocs and not by + * overall RAM usage.
* - * @throws IllegalArgumentException if maxBufferedDocs is smaller than 2 + *The default value is 0.
+ * + * @throws IllegalArgumentException if maxBufferedDocs is + * non-zero and smaller than 2 */ public void setMaxBufferedDocs(int maxBufferedDocs) { ensureOpen(); - if (maxBufferedDocs < 2) - throw new IllegalArgumentException("maxBufferedDocs must at least be 2"); + if (maxBufferedDocs != 0 && maxBufferedDocs < 2) + throw new IllegalArgumentException("maxBufferedDocs must at least be 2 or 0 to disable"); this.minMergeDocs = maxBufferedDocs; } @@ -685,7 +705,28 @@ return minMergeDocs; } + /** Determines the amount of RAM that may be used for + * buffering before the in-memory documents are flushed as + * a new Segment. This only applies when maxBufferedDocs + * is set to 0. Generally for faster indexing performance + * it's best to flush by RAM usage instead of document + * count. + */ + public void setRAMBufferSizeMB(float mb) { + if (mb < 1.0) + throw new IllegalArgumentException("ramBufferSize must at least be 1.0 MB"); + ramBufferSize = mb*1024F*1024F; + docWriter.setRAMBufferSizeMB(mb); + } + /** + * @see #setRAMBufferSizeMB + */ + public float getRAMBufferSizeMB() { + return ramBufferSize/1024F/1024F; + } + + /** *Determines the minimal number of delete terms required before the buffered
* in-memory delete terms are applied and flushed. If there are documents
* buffered in memory at the time, they are merged and a new segment is
@@ -756,7 +797,9 @@
public void setInfoStream(PrintStream infoStream) {
ensureOpen();
this.infoStream = infoStream;
- deleter.setInfoStream(infoStream);
+ docWriter.setInfoStream(infoStream);
+ // nocommit
+ //deleter.setInfoStream(infoStream);
}
/**
@@ -835,7 +878,7 @@
*/
public synchronized void close() throws CorruptIndexException, IOException {
if (!closed) {
- flushRamSegments();
+ flush();
if (commitPending) {
segmentInfos.write(directory); // now commit changes
@@ -844,12 +887,12 @@
rollbackSegmentInfos = null;
}
- ramDirectory.close();
if (writeLock != null) {
writeLock.release(); // release write lock
writeLock = null;
}
closed = true;
+ docWriter = null;
if(closeDir)
directory.close();
@@ -884,7 +927,7 @@
/** Returns the number of documents currently in this index. */
public synchronized int docCount() {
ensureOpen();
- int count = ramSegmentInfos.size();
+ int count = docWriter.docID;
for (int i = 0; i < segmentInfos.size(); i++) {
SegmentInfo si = segmentInfos.info(i);
count += si.docCount;
@@ -962,24 +1005,15 @@
*/
public void addDocument(Document doc, Analyzer analyzer) throws CorruptIndexException, IOException {
ensureOpen();
- SegmentInfo newSegmentInfo = buildSingleDocSegment(doc, analyzer);
- synchronized (this) {
- ramSegmentInfos.addElement(newSegmentInfo);
- maybeFlushRamSegments();
+ synchronized(this) {
+ docWriter.addDocument(doc, analyzer);
+ // For the non-autoCommit case, DocumentsWriter
+ // takes care of flushing its pending state to disk
+ if (autoCommit)
+ maybeFlush();
}
}
- SegmentInfo buildSingleDocSegment(Document doc, Analyzer analyzer)
- throws CorruptIndexException, IOException {
- DocumentWriter dw = new DocumentWriter(ramDirectory, analyzer, this);
- dw.setInfoStream(infoStream);
- String segmentName = newRamSegmentName();
- dw.addDocument(segmentName, doc);
- SegmentInfo si = new SegmentInfo(segmentName, 1, ramDirectory, false, false);
- si.setNumFields(dw.getNumFields());
- return si;
- }
-
/**
* Deletes the document(s) containing term.
* @param term the term to identify the documents to be deleted
@@ -989,7 +1023,7 @@
public synchronized void deleteDocuments(Term term) throws CorruptIndexException, IOException {
ensureOpen();
bufferDeleteTerm(term);
- maybeFlushRamSegments();
+ maybeFlush();
}
/**
@@ -1005,7 +1039,7 @@
for (int i = 0; i < terms.length; i++) {
bufferDeleteTerm(terms[i]);
}
- maybeFlushRamSegments();
+ maybeFlush();
}
/**
@@ -1041,26 +1075,25 @@
public void updateDocument(Term term, Document doc, Analyzer analyzer)
throws CorruptIndexException, IOException {
ensureOpen();
- SegmentInfo newSegmentInfo = buildSingleDocSegment(doc, analyzer);
- synchronized (this) {
- bufferDeleteTerm(term);
- ramSegmentInfos.addElement(newSegmentInfo);
- maybeFlushRamSegments();
+ bufferDeleteTerm(term);
+ synchronized(this) {
+ docWriter.addDocument(doc, analyzer);
+ // nocommit: what if we need to trigger on max delete terms?
+ // For the non-autoCommit case, DocumentsWriter
+ // takes care of flushing its pending state to disk
+ if (autoCommit)
+ maybeFlush();
}
}
- final synchronized String newRamSegmentName() {
- return "_ram_" + Integer.toString(ramSegmentInfos.counter++, Character.MAX_RADIX);
- }
-
// for test purpose
final synchronized int getSegmentCount(){
return segmentInfos.size();
}
// for test purpose
- final synchronized int getRamSegmentCount(){
- return ramSegmentInfos.size();
+ final synchronized int getNumBufferedDocuments(){
+ return docWriter.docID;
}
// for test purpose
@@ -1089,6 +1122,7 @@
*/
private int mergeFactor = DEFAULT_MERGE_FACTOR;
+ // nocommit fix javadocs
/** Determines the minimal number of documents required before the buffered
* in-memory documents are merging and a new Segment is created.
* Since Documents are merged in a {@link org.apache.lucene.store.RAMDirectory},
@@ -1096,10 +1130,11 @@
* the number of files open in a FSDirectory.
*
*
The default value is {@link #DEFAULT_MAX_BUFFERED_DOCS}.
-
*/
private int minMergeDocs = DEFAULT_MAX_BUFFERED_DOCS;
+ // nocommit javadoc
+ private float ramBufferSize = DEFAULT_RAM_BUFFER_SIZE_MB*1024F*1024F;
/** Determines the largest number of documents ever merged by addDocument().
* Small values (e.g., less than 10,000) are best for interactive indexing,
@@ -1183,7 +1218,7 @@
*/
public synchronized void optimize() throws CorruptIndexException, IOException {
ensureOpen();
- flushRamSegments();
+ flush();
while (segmentInfos.size() > 1 ||
(segmentInfos.size() == 1 &&
(SegmentReader.hasDeletions(segmentInfos.info(0)) ||
@@ -1192,7 +1227,7 @@
(useCompoundFile &&
(!SegmentReader.usesCompoundFile(segmentInfos.info(0))))))) {
int minSegment = segmentInfos.size() - mergeFactor;
- mergeSegments(segmentInfos, minSegment < 0 ? 0 : minSegment, segmentInfos.size());
+ mergeSegments(minSegment < 0 ? 0 : minSegment, segmentInfos.size());
}
}
@@ -1209,7 +1244,7 @@
localRollbackSegmentInfos = (SegmentInfos) segmentInfos.clone();
localAutoCommit = autoCommit;
if (localAutoCommit) {
- flushRamSegments();
+ flush();
// Turn off auto-commit during our local transaction:
autoCommit = false;
} else
@@ -1299,16 +1334,18 @@
segmentInfos.clear();
segmentInfos.addAll(rollbackSegmentInfos);
+ docWriter.abort();
+
// Ask deleter to locate unreferenced files & remove
// them:
deleter.checkpoint(segmentInfos, false);
deleter.refresh();
- ramSegmentInfos = new SegmentInfos();
bufferedDeleteTerms.clear();
numBufferedDeleteTerms = 0;
commitPending = false;
+ docWriter.abort();
close();
} else {
@@ -1403,7 +1440,7 @@
for (int base = start; base < segmentInfos.size(); base++) {
int end = Math.min(segmentInfos.size(), base+mergeFactor);
if (end-base > 1) {
- mergeSegments(segmentInfos, base, end);
+ mergeSegments(base, end);
}
}
}
@@ -1443,7 +1480,7 @@
// segments in S may not since they could come from multiple indexes.
// Here is the merge algorithm for addIndexesNoOptimize():
//
- // 1 Flush ram segments.
+ // 1 Flush ram.
// 2 Consider a combined sequence with segments from T followed
// by segments from S (same as current addIndexes(Directory[])).
// 3 Assume the highest level for segments in S is h. Call
@@ -1464,14 +1501,18 @@
// copy a segment, which may cause doc count to change because deleted
// docs are garbage collected.
- // 1 flush ram segments
+ // 1 flush ram
ensureOpen();
- flushRamSegments();
+ flush();
// 2 copy segment infos and find the highest level from dirs
int startUpperBound = minMergeDocs;
+ // nocommit: what to do?
+ if (startUpperBound == 0)
+ startUpperBound = 10;
+
boolean success = false;
startTransaction();
@@ -1530,7 +1571,7 @@
// copy those segments from S
for (int i = segmentCount - numSegmentsToCopy; i < segmentCount; i++) {
- mergeSegments(segmentInfos, i, i + 1);
+ mergeSegments(i, i + 1);
}
if (checkNonDecreasingLevels(segmentCount - numSegmentsToCopy)) {
success = true;
@@ -1539,7 +1580,7 @@
}
// invariants do not hold, simply merge those segments
- mergeSegments(segmentInfos, segmentCount - numTailSegments, segmentCount);
+ mergeSegments(segmentCount - numTailSegments, segmentCount);
// maybe merge segments again if necessary
if (segmentInfos.info(segmentInfos.size() - 1).docCount > startUpperBound) {
@@ -1679,19 +1720,23 @@
throws IOException {
}
- protected final void maybeFlushRamSegments() throws CorruptIndexException, IOException {
+ protected final synchronized void maybeFlush() throws CorruptIndexException, IOException {
// A flush is triggered if enough new documents are buffered or
- // if enough delete terms are buffered
- if (ramSegmentInfos.size() >= minMergeDocs || numBufferedDeleteTerms >= maxBufferedDeleteTerms) {
- flushRamSegments();
- }
- }
+ // if enough delete terms are buffered or enough RAM is
+ // being consumed
+ // nocommit
+ if (numBufferedDeleteTerms >= maxBufferedDeleteTerms ||
+ (autoCommit && ((minMergeDocs != 0 && docWriter.docID >= minMergeDocs) ||
+ ((true || minMergeDocs == 0) && autoCommit && docWriter.timeToFlush())))) {
- /** Expert: Flushes all RAM-resident segments (buffered documents), then may merge segments. */
- private final synchronized void flushRamSegments() throws CorruptIndexException, IOException {
- if (ramSegmentInfos.size() > 0 || bufferedDeleteTerms.size() > 0) {
- mergeSegments(ramSegmentInfos, 0, ramSegmentInfos.size());
- maybeMergeSegments(minMergeDocs);
+ /*
+ if (minMergeDocs < docWriter.docID)
+ throw new RuntimeException("too small minMergeDocs=" + minMergeDocs + " #docs=" + docWriter.docID);
+ if (minMergeDocs > 6*docWriter.docID)
+ throw new RuntimeException("too large minMergeDocs=" + minMergeDocs + " #docs=" + docWriter.docID);
+ */
+
+ flush();
}
}
@@ -1705,7 +1750,86 @@
*/
public final synchronized void flush() throws CorruptIndexException, IOException {
ensureOpen();
- flushRamSegments();
+
+ SegmentInfo newSegment = null;
+ boolean anything = false;
+
+ boolean flushDocs = docWriter.docID > 0;
+ boolean flushDeletes = bufferedDeleteTerms.size() > 0;
+ final int numDocs = docWriter.docID;
+
+ if (flushDocs || flushDeletes) {
+
+ SegmentInfos rollback = null;
+
+ if (flushDeletes)
+ rollback = (SegmentInfos) segmentInfos.clone();
+
+ boolean success = false;
+
+ try {
+ if (flushDocs) {
+ int mergedDocCount = docWriter.docID;
+ String segment = docWriter.segment;
+ docWriter.flush(newSegmentName());
+ newSegment = new SegmentInfo(segment,
+ mergedDocCount,
+ directory, false, true);
+ segmentInfos.addElement(newSegment);
+ }
+
+ if (flushDeletes) {
+ maybeApplyDeletes(flushDocs);
+ doAfterFlush();
+ }
+
+ checkpoint();
+ success = true;
+ } finally {
+ if (!success) {
+ if (flushDeletes) {
+ // Fully replace the segmentInfos since flushed
+ // deletes could have changed any of the
+ // SegmentInfo instances:
+ segmentInfos.clear();
+ segmentInfos.addAll(rollback);
+ } else {
+ // Remove segment we added, if any:
+ if (newSegment != null &&
+ segmentInfos.size() > 0 &&
+ segmentInfos.info(segmentInfos.size()-1) == newSegment)
+ segmentInfos.remove(segmentInfos.size()-1);
+ docWriter.abort();
+ }
+ deleter.checkpoint(segmentInfos, false);
+ deleter.refresh();
+ }
+ }
+
+ deleter.checkpoint(segmentInfos, autoCommit);
+
+ if (flushDocs && useCompoundFile) {
+ success = false;
+ try {
+ docWriter.createCompoundFile(newSegment.name);
+ newSegment.setUseCompoundFile(true);
+ checkpoint();
+ success = true;
+ } finally {
+ if (!success) {
+ newSegment.setUseCompoundFile(false);
+ deleter.refresh();
+ }
+ }
+
+ deleter.checkpoint(segmentInfos, autoCommit);
+ }
+
+ // nocommit
+ // maybeMergeSegments(mergeFactor * numDocs / 2);
+
+ maybeMergeSegments(minMergeDocs);
+ }
}
/** Expert: Return the total size of all index files currently cached in memory.
@@ -1713,15 +1837,15 @@
*/
public final long ramSizeInBytes() {
ensureOpen();
- return ramDirectory.sizeInBytes();
+ return docWriter.getRAMUsed();
}
/** Expert: Return the number of documents whose segments are currently cached in memory.
- * Useful when calling flushRamSegments()
+ * Useful when calling flush()
*/
public final synchronized int numRamDocs() {
ensureOpen();
- return ramSegmentInfos.size();
+ return docWriter.docID;
}
/** Incremental segment merger. */
@@ -1729,6 +1853,9 @@
long lowerBound = -1;
long upperBound = startUpperBound;
+ // nocommit
+ if (upperBound == 0) upperBound = 10;
+
while (upperBound < maxMergeDocs) {
int minSegment = segmentInfos.size();
int maxSegment = -1;
@@ -1760,7 +1887,7 @@
while (numSegments >= mergeFactor) {
// merge the leftmost* mergeFactor segments
- int docCount = mergeSegments(segmentInfos, minSegment, minSegment + mergeFactor);
+ int docCount = mergeSegments(minSegment, minSegment + mergeFactor);
numSegments -= mergeFactor;
if (docCount > upperBound) {
@@ -1789,39 +1916,33 @@
* Merges the named range of segments, replacing them in the stack with a
* single segment.
*/
- private final int mergeSegments(SegmentInfos sourceSegments, int minSegment, int end)
+ long netMergeTime;
+
+ private final int mergeSegments(int minSegment, int end)
throws CorruptIndexException, IOException {
- // We may be called solely because there are deletes
- // pending, in which case doMerge is false:
- boolean doMerge = end > 0;
final String mergedName = newSegmentName();
+
+ long t0 = System.currentTimeMillis();
+
SegmentMerger merger = null;
-
- final List ramSegmentsToDelete = new ArrayList();
-
SegmentInfo newSegment = null;
int mergedDocCount = 0;
- boolean anyDeletes = (bufferedDeleteTerms.size() != 0);
// This is try/finally to make sure merger's readers are closed:
try {
- if (doMerge) {
- if (infoStream != null) infoStream.print("merging segments");
- merger = new SegmentMerger(this, mergedName);
+ if (infoStream != null) infoStream.print("merging segments");
- for (int i = minSegment; i < end; i++) {
- SegmentInfo si = sourceSegments.info(i);
- if (infoStream != null)
- infoStream.print(" " + si.name + " (" + si.docCount + " docs)");
- IndexReader reader = SegmentReader.get(si); // no need to set deleter (yet)
- merger.add(reader);
- if (reader.directory() == this.ramDirectory) {
- ramSegmentsToDelete.add(si);
- }
- }
+ merger = new SegmentMerger(this, mergedName);
+
+ for (int i = minSegment; i < end; i++) {
+ SegmentInfo si = segmentInfos.info(i);
+ if (infoStream != null)
+ infoStream.print(" " + si.name + " (" + si.docCount + " docs)");
+ IndexReader reader = SegmentReader.get(si); // no need to set deleter (yet)
+ merger.add(reader);
}
SegmentInfos rollback = null;
@@ -1831,99 +1952,57 @@
// if we hit exception when doing the merge:
try {
- if (doMerge) {
- mergedDocCount = merger.merge();
+ mergedDocCount = merger.merge();
- if (infoStream != null) {
- infoStream.println(" into "+mergedName+" ("+mergedDocCount+" docs)");
- }
+ if (infoStream != null) {
+ infoStream.println(" into "+mergedName+" ("+mergedDocCount+" docs)");
+ }
- newSegment = new SegmentInfo(mergedName, mergedDocCount,
- directory, false, true);
- }
+ newSegment = new SegmentInfo(mergedName, mergedDocCount,
+ directory, false, true);
- if (sourceSegments != ramSegmentInfos || anyDeletes) {
- // Now save the SegmentInfo instances that
- // we are replacing:
- rollback = (SegmentInfos) segmentInfos.clone();
- }
+ rollback = (SegmentInfos) segmentInfos.clone();
- if (doMerge) {
- if (sourceSegments == ramSegmentInfos) {
- segmentInfos.addElement(newSegment);
- } else {
- for (int i = end-1; i > minSegment; i--) // remove old infos & add new
- sourceSegments.remove(i);
+ for (int i = end-1; i > minSegment; i--) // remove old infos & add new
+ segmentInfos.remove(i);
- segmentInfos.set(minSegment, newSegment);
- }
- }
+ segmentInfos.set(minSegment, newSegment);
- if (sourceSegments == ramSegmentInfos) {
- maybeApplyDeletes(doMerge);
- doAfterFlush();
- }
-
checkpoint();
success = true;
} finally {
- if (success) {
- // The non-ram-segments case is already committed
- // (above), so all the remains for ram segments case
- // is to clear the ram segments:
- if (sourceSegments == ramSegmentInfos) {
- ramSegmentInfos.removeAllElements();
- }
- } else {
+ if (!success && rollback != null) {
+ // Rollback the individual SegmentInfo
+ // instances, but keep original SegmentInfos
+ // instance (so we don't try to write again the
+ // same segments_N file -- write once):
+ segmentInfos.clear();
+ segmentInfos.addAll(rollback);
- // Must rollback so our state matches index:
- if (sourceSegments == ramSegmentInfos && !anyDeletes) {
- // Simple case: newSegment may or may not have
- // been added to the end of our segment infos,
- // so just check & remove if so:
- if (newSegment != null &&
- segmentInfos.size() > 0 &&
- segmentInfos.info(segmentInfos.size()-1) == newSegment) {
- segmentInfos.remove(segmentInfos.size()-1);
- }
- } else if (rollback != null) {
- // Rollback the individual SegmentInfo
- // instances, but keep original SegmentInfos
- // instance (so we don't try to write again the
- // same segments_N file -- write once):
- segmentInfos.clear();
- segmentInfos.addAll(rollback);
- }
-
// Delete any partially created and now unreferenced files:
deleter.refresh();
}
}
} finally {
// close readers before we attempt to delete now-obsolete segments
- if (doMerge) merger.closeReaders();
+ merger.closeReaders();
}
- // Delete the RAM segments
- deleter.deleteDirect(ramDirectory, ramSegmentsToDelete);
-
// Give deleter a chance to remove files now.
deleter.checkpoint(segmentInfos, autoCommit);
- if (useCompoundFile && doMerge) {
+ if (useCompoundFile) {
boolean success = false;
try {
-
merger.createCompoundFile(mergedName + ".cfs");
newSegment.setUseCompoundFile(true);
checkpoint();
success = true;
-
} finally {
if (!success) {
// Must rollback:
@@ -1936,20 +2015,24 @@
deleter.checkpoint(segmentInfos, autoCommit);
}
+ long t1 = System.currentTimeMillis();
+ netMergeTime += (t1-t0);
+ if (infoStream != null)
+ System.out.println("TIME: merge: " + (netMergeTime/1000.0) + " sec");
return mergedDocCount;
}
// Called during flush to apply any buffered deletes. If
// doMerge is true then a new segment was just created and
// flushed from the ram segments.
- private final void maybeApplyDeletes(boolean doMerge) throws CorruptIndexException, IOException {
+ private final void maybeApplyDeletes(boolean flushedNewSegment) throws CorruptIndexException, IOException {
if (bufferedDeleteTerms.size() > 0) {
if (infoStream != null)
infoStream.println("flush " + numBufferedDeleteTerms + " buffered deleted terms on "
+ segmentInfos.size() + " segments.");
- if (doMerge) {
+ if (flushedNewSegment) {
IndexReader reader = null;
try {
reader = SegmentReader.get(segmentInfos.info(segmentInfos.size() - 1));
@@ -1970,7 +2053,7 @@
}
int infosEnd = segmentInfos.size();
- if (doMerge) {
+ if (flushedNewSegment) {
infosEnd--;
}
@@ -2002,6 +2085,8 @@
private final boolean checkNonDecreasingLevels(int start) {
int lowerBound = -1;
int upperBound = minMergeDocs;
+ if (upperBound == 0)
+ upperBound = 10;
for (int i = segmentInfos.size() - 1; i >= start; i--) {
int docCount = segmentInfos.info(i).docCount;
@@ -2050,10 +2135,11 @@
// well as the disk segments.
private void bufferDeleteTerm(Term term) {
Num num = (Num) bufferedDeleteTerms.get(term);
+ int numDoc = docWriter.docID;
if (num == null) {
- bufferedDeleteTerms.put(term, new Num(ramSegmentInfos.size()));
+ bufferedDeleteTerms.put(term, new Num(numDoc));
} else {
- num.setNum(ramSegmentInfos.size());
+ num.setNum(numDoc);
}
numBufferedDeleteTerms++;
}
@@ -2063,17 +2149,20 @@
// the documents buffered before it, not those buffered after it.
private final void applyDeletesSelectively(HashMap deleteTerms,
IndexReader reader) throws CorruptIndexException, IOException {
+ //System.out.println("now apply selective deletes");
Iterator iter = deleteTerms.entrySet().iterator();
while (iter.hasNext()) {
Entry entry = (Entry) iter.next();
Term term = (Term) entry.getKey();
-
+ //System.out.println(" term " + term);
+
TermDocs docs = reader.termDocs(term);
if (docs != null) {
int num = ((Num) entry.getValue()).getNum();
try {
while (docs.next()) {
int doc = docs.doc();
+ //System.out.println(" doc " + doc + " vs " + num);
if (doc >= num) {
break;
}
Index: src/java/org/apache/lucene/index/IndexFileDeleter.java
===================================================================
--- src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 540135)
+++ src/java/org/apache/lucene/index/IndexFileDeleter.java (working copy)
@@ -97,6 +97,7 @@
private PrintStream infoStream;
private Directory directory;
private IndexDeletionPolicy policy;
+ private DocumentsWriter docWriter;
void setInfoStream(PrintStream infoStream) {
this.infoStream = infoStream;
@@ -116,10 +117,12 @@
* @throws CorruptIndexException if the index is corrupt
* @throws IOException if there is a low-level IO error
*/
- public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream)
+ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, PrintStream infoStream, DocumentsWriter docWriter)
throws CorruptIndexException, IOException {
+ this.docWriter = docWriter;
this.infoStream = infoStream;
+
this.policy = policy;
this.directory = directory;
@@ -310,6 +313,8 @@
// Incref the files:
incRef(segmentInfos, isCommit);
+ if (docWriter != null)
+ incRef(docWriter.files());
if (isCommit) {
// Append to our commits list:
@@ -325,9 +330,8 @@
// DecRef old files from the last checkpoint, if any:
int size = lastFiles.size();
if (size > 0) {
- for(int i=0;i