Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 947247) +++ lucene/CHANGES.txt (working copy) @@ -74,6 +74,14 @@ it cannot delete the lock file, since obtaining the lock does not fail if the file is there. (Shai Erera) +* LUCENE-2455: IndexWriter.addIndexes no longer optimizes the target index + before it adds the new ones. Also, the existing segments are not merged and so + the index will not end up with a single segment (unless it was empty before). + In addition, addIndexesNoOptimize was renamed to addIndexes and no longer + invokes a merge on the incoming and target segments, but instead copies the + segments to the target index. You can call maybeMerge or optimize after this + method completes, if you need to. (Shai Erera) + API Changes * LUCENE-2076: Rename FSDirectory.getFile -> getDirectory. (George Index: lucene/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java (revision 947247) +++ lucene/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java (working copy) @@ -49,7 +49,7 @@ } System.out.println("Merging..."); - writer.addIndexesNoOptimize(indexes); + writer.addIndexes(indexes); System.out.println("Optimizing..."); writer.optimize(); Index: lucene/src/java/org/apache/lucene/index/CompoundFileReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/CompoundFileReader.java (revision 947247) +++ lucene/src/java/org/apache/lucene/index/CompoundFileReader.java (working copy) @@ -55,9 +55,7 @@ this(dir, name, BufferedIndexInput.BUFFER_SIZE); } - public CompoundFileReader(Directory dir, String name, int readBufferSize) - throws IOException - { + public CompoundFileReader(Directory dir, String name, int readBufferSize) throws IOException { directory = dir; fileName = name; this.readBufferSize = readBufferSize; @@ -74,6 +72,12 @@ long offset = stream.readLong(); String id = stream.readString(); + // Fix the id to not include the segment names. When indexes are + // added via addIndexes(Dir), they are bulk-copied, and renamed. + // But the ids in the CFS file are not changed. In 4.0, CFS + // no longer stores the segment name in the file itself. + id = IndexFileNames.stripSegmentName(id); + if (entry != null) { // set length of the previous entry entry.length = offset - entry.offset; @@ -92,7 +96,7 @@ success = true; } finally { - if (! success && (stream != null)) { + if (!success && (stream != null)) { try { stream.close(); } catch (IOException e) { } @@ -132,7 +136,8 @@ { if (stream == null) throw new IOException("Stream closed"); - + + id = IndexFileNames.stripSegmentName(id); FileEntry entry = entries.get(id); if (entry == null) throw new IOException("No sub-file with id " + id + " found"); @@ -143,14 +148,19 @@ /** Returns an array of strings, one for each file in the directory. */ @Override public String[] listAll() { - String res[] = new String[entries.size()]; - return entries.keySet().toArray(res); + String[] res = entries.keySet().toArray(new String[entries.size()]); + // Add the segment name + String seg = fileName.substring(0, fileName.indexOf('.')); + for (int i = 0; i < res.length; i++) { + res[i] = seg + res[i]; + } + return res; } /** Returns true iff a file with the given name exists. */ @Override public boolean fileExists(String name) { - return entries.containsKey(name); + return entries.containsKey(IndexFileNames.stripSegmentName(name)); } /** Returns the time the compound file was last modified. */ @@ -184,7 +194,7 @@ * @throws IOException if the file does not exist */ @Override public long fileLength(String name) throws IOException { - FileEntry e = entries.get(name); + FileEntry e = entries.get(IndexFileNames.stripSegmentName(name)); if (e == null) throw new FileNotFoundException(name); return e.length; Index: lucene/src/java/org/apache/lucene/index/IndexFileNames.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexFileNames.java (revision 947247) +++ lucene/src/java/org/apache/lucene/index/IndexFileNames.java (working copy) @@ -238,5 +238,27 @@ // or not, since there's only 1 '+' operator. return filename.endsWith("." + ext); } + + /** + * Strips the segment file name out of the given one. If you used + * {@link #segmentFileName} or {@link #fileNameFromGeneration} to create your + * files, then this method simply removes whatever comes before the first '.', + * or the second '_' (excluding both), in case of deleted docs. + * + * @return the filename with the segment name removed, or the given filename + * if it does not contain a '.' and '_'. + */ + public static final String stripSegmentName(String filename) { + // If it is a .del file, there's an '_' after the first character + int idx = filename.indexOf('_', 1); + if (idx == -1) { + // If it's not, strip everything that's before the '.' + idx = filename.indexOf('.'); + } + if (idx != -1) { + filename = filename.substring(idx); + } + return filename; + } } Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 947247) +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -3039,26 +3039,34 @@ private void noDupDirs(Directory... dirs) { HashSet dups = new HashSet(); - for(int i=0;iThis may be used to parallelize batch indexing. A large document - * collection can be broken into sub-collections. Each sub-collection can be - * indexed in parallel, on a different thread, process or machine. The + *

This may be used to parallelize batch indexing. A large document + * collection can be broken into sub-collections. Each sub-collection can be + * indexed in parallel, on a different thread, process or machine. The * complete index can then be created by merging sub-collection indexes * with this method. * - *

NOTE: the index in each Directory must not be + *

+ * NOTE: the index in each {@link Directory} must not be * changed (opened by a writer) while this method is * running. This method does not acquire a write lock in * each input Directory, so it is up to the caller to @@ -3072,35 +3080,32 @@ * handled: it does not commit a new segments_N file until * all indexes are added. This means if an Exception * occurs (for example disk full), then either no indexes - * will have been added or they all will have been.

+ * will have been added or they all will have been. * *

Note that this requires temporary free space in the - * Directory up to 2X the sum of all input indexes - * (including the starting index). If readers/searchers + * {@link Directory} up to 2X the sum of all input indexes + * (including the starting index). If readers/searchers * are open against the starting index, then temporary * free space required will be higher by the size of the * starting index (see {@link #optimize()} for details). - *

* *

Once this completes, the final size of the index * will be less than the sum of all input index sizes - * (including the starting index). It could be quite a + * (including the starting index). It could be quite a * bit smaller (if there were many pending deletes) or - * just slightly smaller.

+ * just slightly smaller. * + *

This requires this index not be among those to be added. + * *

- * This requires this index not be among those to be added. + * NOTE: if this method hits an OutOfMemoryError + * you should immediately close the writer. See above for details. * - *

NOTE: if this method hits an OutOfMemoryError - * you should immediately close the writer. See above for details.

- * * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void addIndexesNoOptimize(Directory... dirs) - throws CorruptIndexException, IOException { - + public void addIndexes2(Directory... dirs) throws CorruptIndexException, IOException { ensureOpen(); noDupDirs(dirs); @@ -3110,7 +3115,7 @@ try { if (infoStream != null) - message("flush at addIndexesNoOptimize"); + message("flush at addIndexes(Directory...)"); flush(true, false, true); boolean success = false; @@ -3123,16 +3128,10 @@ synchronized(this) { ensureOpen(); - for (int i = 0; i < dirs.length; i++) { - if (directory == dirs[i]) { - // cannot add this index: segments may be deleted in merge before added - throw new IllegalArgumentException("Cannot add this index to itself"); - } - + for (Directory dir : dirs) { SegmentInfos sis = new SegmentInfos(); // read infos from dir - sis.read(dirs[i]); - for (int j = 0; j < sis.size(); j++) { - SegmentInfo info = sis.info(j); + sis.read(dir); + for (SegmentInfo info : sis) { assert !segmentInfos.contains(info): "dup info dir=" + info.dir + " name=" + info.name; docCount += info.docCount; segmentInfos.add(info); // add each info @@ -3166,7 +3165,7 @@ } } } catch (OutOfMemoryError oom) { - handleOOM(oom, "addIndexesNoOptimize"); + handleOOM(oom, "addIndexes(Directory...)"); } finally { if (docWriter != null) { docWriter.resumeAllThreads(); @@ -3184,7 +3183,7 @@ * currently running merges (in the background) complete. * We don't return until the SegmentInfos has no more * external segments. Currently this is only used by - * addIndexesNoOptimize(). */ + * addIndexes(Directory...). */ private void resolveExternalSegments() throws CorruptIndexException, IOException { boolean any = false; @@ -3253,117 +3252,54 @@ mergeScheduler.merge(this); } - /** Merges the provided indexes into this index. - *

After this completes, the index is optimized.

- *

The provided IndexReaders are not closed.

+ /** + * Merges the provided indexes into this index. This method is useful + * if you use extensions of {@link IndexReader}. Otherwise, using + * {@link #addIndexes(Directory...)} is highly recommended for performance + * reasons. It uses the {@link MergeScheduler} and {@link MergePolicy} set + * on this writer, which may perform merges in parallel. + * + *

The provided IndexReaders are not closed. * - *

NOTE: while this is running, any attempts to - * add or delete documents (with another thread) will be - * paused until this method completes. + *

NOTE: this method does not merge the current segments, + * only the incoming ones. + * + *

See {@link #addIndexes(Directory...)} for details on transactional + * semantics, temporary free space required in the Directory, + * and non-CFS segments on an Exception. * - *

See {@link #addIndexesNoOptimize} for - * details on transactional semantics, temporary free - * space required in the Directory, and non-CFS segments - * on an Exception.

- * *

NOTE: if this method hits an OutOfMemoryError * you should immediately close the writer. See above for details.

+ * href="#OOME">above for details. * * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void addIndexes(IndexReader... readers) - throws CorruptIndexException, IOException { + public void addIndexes(IndexReader... readers) throws CorruptIndexException, IOException { ensureOpen(); - // Do not allow add docs or deletes while we are running: - docWriter.pauseAllThreads(); - - // We must pre-acquire a read lock here (and upgrade to - // write lock in startTransaction below) so that no - // other addIndexes is allowed to start up after we have - // flushed & optimized but before we then start our - // transaction. This is because the merging below - // requires that only one segment is present in the - // index: - acquireRead(); - try { - + String mergedName = newSegmentName(); + SegmentMerger merger = new SegmentMerger(this, mergedName, null); + + for (IndexReader reader : readers) // add new indexes + merger.add(reader); + + int docCount = merger.merge(); // merge 'em + SegmentInfo info = null; - String mergedName = null; - SegmentMerger merger = null; + synchronized(this) { + info = new SegmentInfo(mergedName, docCount, directory, false, true, + -1, null, false, merger.hasProx()); + setDiagnostics(info, "addIndexes(IndexReader...)"); + segmentInfos.add(info); - boolean success = false; - - try { - flush(true, false, true); - optimize(); // start with zero or 1 seg - success = true; - } finally { - // Take care to release the read lock if we hit an - // exception before starting the transaction - if (!success) - releaseRead(); + // Notify DocumentsWriter that the flushed count just increased + docWriter.updateFlushedDocCount(docCount); } - - // true means we already have a read lock; if this - // call hits an exception it will release the write - // lock: - startTransaction(true); - - try { - mergedName = newSegmentName(); - merger = new SegmentMerger(this, mergedName, null); - - SegmentReader sReader = null; - synchronized(this) { - if (segmentInfos.size() == 1) { // add existing index, if any - sReader = readerPool.get(segmentInfos.info(0), true, BufferedIndexInput.BUFFER_SIZE, -1); - } - } - - success = false; - - try { - if (sReader != null) - merger.add(sReader); - - for (int i = 0; i < readers.length; i++) // add new indexes - merger.add(readers[i]); - - int docCount = merger.merge(); // merge 'em - - synchronized(this) { - segmentInfos.clear(); // pop old infos & add new - info = new SegmentInfo(mergedName, docCount, directory, false, true, - -1, null, false, merger.hasProx()); - setDiagnostics(info, "addIndexes(IndexReader...)"); - segmentInfos.add(info); - } - - // Notify DocumentsWriter that the flushed count just increased - docWriter.updateFlushedDocCount(docCount); - - success = true; - - } finally { - if (sReader != null) { - readerPool.release(sReader); - } - } - } finally { - if (!success) { - if (infoStream != null) - message("hit exception in addIndexes during merge"); - rollbackTransaction(); - } else { - commitTransaction(); - } - } - + + // Now create the compound file if needed if (mergePolicy instanceof LogMergePolicy && getUseCompoundFile()) { List files = null; @@ -3371,7 +3307,7 @@ synchronized(this) { // Must incRef our files so that if another thread // is running merge/optimize, it doesn't delete our - // segment's files before we have a change to + // segment's files before we have a chance to // finish making the compound file. if (segmentInfos.contains(info)) { files = info.files(); @@ -3380,43 +3316,85 @@ } if (files != null) { - - success = false; - - startTransaction(false); - try { merger.createCompoundFile(mergedName + ".cfs"); synchronized(this) { info.setUseCompoundFile(true); } - - success = true; - } finally { - deleter.decRef(files); - - if (!success) { - if (infoStream != null) - message("hit exception building compound file in addIndexes during merge"); - - rollbackTransaction(); - } else { - commitTransaction(); - } } } } } catch (OutOfMemoryError oom) { handleOOM(oom, "addIndexes(IndexReader...)"); - } finally { - if (docWriter != null) { - docWriter.resumeAllThreads(); - } } } + public void addIndexes(Directory... dirs) throws CorruptIndexException, IOException { + ensureOpen(); + + noDupDirs(dirs); + + try { + if (infoStream != null) + message("flush at addIndexes(Directory...)"); + flush(true, false, true); + + int docCount = 0; + List infos = new ArrayList(); + for (Directory dir : dirs) { + SegmentInfos sis = new SegmentInfos(); // read infos from dir + sis.read(dir); + Map dsNames = new HashMap(); + for (SegmentInfo info : sis) { + assert !infos.contains(info): "dup info dir=" + info.dir + " name=" + info.name; + docCount += info.docCount; + String newSegName = newSegmentName(); + String dsName = info.getDocStoreSegment(); + + // Determine if the doc store of this segment needs to be copied. It's + // only relevant for segments who share doc store with others, because + // the DS might have been copied already, in which case we just want to + // update the DS name of this SegmentInfo. + String newDsName = dsNames.get(dsName); + boolean docStoreCopied = true; + if (newDsName == null) { + dsNames.put(dsName, newSegName); + newDsName = newSegName; + docStoreCopied = false; + } + + // Copy the segment files + for (String file : info.files()) { + if (docStoreCopied && IndexFileNames.isDocStoreFile(file)) { + continue; + } + dir.copy(directory, file, newSegName + IndexFileNames.stripSegmentName(file)); + } + + // Update SI appropriately + info.setDocStore(info.getDocStoreOffset(), newDsName, info.getDocStoreIsCompoundFile()); + info.dir = directory; + info.name = newSegName; + + infos.add(info); + } + } + + synchronized (this) { + ensureOpen(); + segmentInfos.addAll(infos); + // Notify DocumentsWriter that the flushed count just increased + docWriter.updateFlushedDocCount(docCount); + } + + } catch (OutOfMemoryError oom) { + handleOOM(oom, "addIndexes(Directory...)"); + } + + } + /** * A hook for extending classes to execute operations after pending added and * deleted documents have been flushed to the Directory but before the change @@ -4976,10 +4954,10 @@ * Sets the {@link PayloadProcessorProvider} to use when merging payloads. * Note that the given pcp will be invoked for every segment that * is merged, not only external ones that are given through - * {@link IndexWriter#addIndexes} or {@link IndexWriter#addIndexesNoOptimize}. - * If you want only the payloads of the external segments to be processed, you - * can return null whenever a {@link DirPayloadProcessor} is - * requested for the {@link Directory} of the {@link IndexWriter}. + * {@link #addIndexes}. If you want only the payloads of the external segments + * to be processed, you can return null whenever a + * {@link DirPayloadProcessor} is requested for the {@link Directory} of the + * {@link IndexWriter}. *

* The default is null which means payloads are processed * normally (copied) during segment merges. You can also unset it by passing Index: lucene/src/java/org/apache/lucene/index/SegmentInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfo.java (revision 947247) +++ lucene/src/java/org/apache/lucene/index/SegmentInfo.java (working copy) @@ -521,6 +521,7 @@ docStoreOffset = offset; docStoreSegment = segment; docStoreIsCompoundFile = isCompoundFile; + clearFiles(); } /** Index: lucene/src/java/org/apache/lucene/store/Directory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/Directory.java (revision 947247) +++ lucene/src/java/org/apache/lucene/store/Directory.java (working copy) @@ -21,13 +21,7 @@ import java.io.IOException; import java.io.Closeable; import java.util.Collection; -import java.util.Collections; -import java.util.ArrayList; -import static java.util.Arrays.asList; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; import org.apache.lucene.index.IndexFileNameFilter; import org.apache.lucene.util.IOUtils; @@ -204,84 +198,89 @@ return this.toString(); } - /** - *

Copy all files of this directory to destination directory. All conflicting files at destination are overwritten

- *

NOTE: this method only copies files that look like index files (ie, have extensions matching the known - * extensions of index files). - *

NOTE: the source directory should not change while this method is running. Otherwise the results are - * undefined and you could easily hit a FileNotFoundException.

- * - * @param to destination directory + * Copies the file src to {@link Directory} to under the new + * file name dest. + *

+ * If you want to copy the entire source directory to the destination one, you + * can do so like this: + * + * 

+   * Directory to; // the directory to copy to
+   * for (String file : dir.listAll()) {
+   *   dir.copy(to, file, newFile); // newFile can be either file, or a new name
+   * }
+   *
+ *

+ * NOTE: this method does not check whether dest exist and will + * overwrite it if it does. */ - public final void copyTo(Directory to) throws IOException { - List filenames = new ArrayList(); - IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); - - for (String name : listAll()) - if (filter.accept(null, name)) - filenames.add(name); - - copyTo(to, filenames); - } - - /** - *

Copy given files of this directory to destination directory. All conflicting files at destination are overwritten

- *

NOTE: the source directory should not change while this method is running. Otherwise the results are - * undefined and you could easily hit a FileNotFoundException.

- *

NOTE: implementations can check if destination directory is of the same type as 'this' and perform optimized copy

- * - * @param to destination directory - * @param filenames file names to be copied - */ - public void copyTo(Directory to, Collection filenames) throws IOException { - byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE]; - for (String filename : filenames) { - IndexOutput os = null; - IndexInput is = null; - IOException priorException = null; - try { - // create file in dest directory - os = to.createOutput(filename); - // read current file - is = openInput(filename); - // and copy to dest directory - long len = is.length(); - long readCount = 0; - while (readCount < len) { - int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len ? (int) (len - readCount) : BufferedIndexOutput.BUFFER_SIZE; - is.readBytes(buf, 0, toRead); - os.writeBytes(buf, toRead); - readCount += toRead; - } - } catch (IOException ioe) { - priorException = ioe; - } finally { - IOUtils.closeSafely(priorException, os, is); + public void copy(Directory to, String src, String dest) throws IOException { + IndexOutput os = null; + IndexInput is = null; + IOException priorException = null; + int bufSize = BufferedIndexOutput.BUFFER_SIZE; + byte[] buf = new byte[bufSize]; + try { + // create file in dest directory + os = to.createOutput(dest); + // read current file + is = openInput(src); + // and copy to dest directory + long len = is.length(); + long numRead = 0; + while (numRead < len) { + long left = len - numRead; + int toRead = (int) (bufSize < left ? bufSize : left); + is.readBytes(buf, 0, toRead); + os.writeBytes(buf, toRead); + numRead += toRead; } + } catch (IOException ioe) { + priorException = ioe; + } finally { + IOUtils.closeSafely(priorException, os, is); } } /** - * Copy contents of a directory src to a directory dest. If a file in src already exists in dest then the one in dest - * will be blindly overwritten. - *

- *

NOTE: the source directory cannot change while this method is running. Otherwise the results are - * undefined and you could easily hit a FileNotFoundException. - *

- *

NOTE: this method only copies files that look like index files (ie, have extensions matching the known - * extensions of index files). - * - * @param src source directory - * @param dest destination directory - * @param closeDirSrc if true, call {@link #close()} method on source directory - * @deprecated should be replaced with src.copyTo(dest); [src.close();] + * Copy contents of a directory src to a directory dest. If a file in src + * already exists in dest then the one in dest will be blindly overwritten. + *

+ * NOTE: the source directory cannot change while this method is + * running. Otherwise the results are undefined and you could easily hit a + * FileNotFoundException. + *

+ * NOTE: this method only copies files that look like index files (ie, + * have extensions matching the known extensions of index files). + * + * @param src source directory + * @param dest destination directory + * @param closeDirSrc if true, call {@link #close()} method on + * source directory + * @deprecated should be replaced with calls to + * {@link #copy(Directory, String, String)} for every file that + * needs copying. You can use the the following code: + * + * 

+   * IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
+   * for (String file : src.listAll()) {
+   *   if (filter.accept(null, file)) {
+   *     src.copy(dest, file, file);
+   *   }
+   * }
+   *
*/ - @Deprecated public static void copy(Directory src, Directory dest, boolean closeDirSrc) throws IOException { - src.copyTo(dest); - if (closeDirSrc) + IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); + for (String file : src.listAll()) { + if (filter.accept(null, file)) { + src.copy(dest, file, file); + } + } + if (closeDirSrc) { src.close(); + } } /** Index: lucene/src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/FSDirectory.java (revision 947247) +++ lucene/src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -431,29 +431,27 @@ } @Override - public void copyTo(Directory to, Collection filenames) throws IOException { + public void copy(Directory to, String src, String dest) throws IOException { if (to instanceof FSDirectory) { FSDirectory target = (FSDirectory) to; - - for (String filename : filenames) { - target.ensureCanWrite(filename); - FileChannel input = null; - FileChannel output = null; - IOException priorException = null; - try { - input = new FileInputStream(new File(directory, filename)).getChannel(); - output = new FileOutputStream(new File(target.directory, filename)).getChannel(); - output.transferFrom(input, 0, input.size()); - } catch (IOException ioe) { - priorException = ioe; - } finally { - IOUtils.closeSafely(priorException, input, output); - } + target.ensureCanWrite(dest); + FileChannel input = null; + FileChannel output = null; + IOException priorException = null; + try { + input = new FileInputStream(new File(directory, src)).getChannel(); + output = new FileOutputStream(new File(target.directory, dest)).getChannel(); + output.transferFrom(input, 0, input.size()); + } catch (IOException ioe) { + priorException = ioe; + } finally { + IOUtils.closeSafely(priorException, input, output); } - } else - super.copyTo(to, filenames); + } else { + super.copy(to, src, dest); + } } - + protected static class FSIndexOutput extends BufferedIndexOutput { private final FSDirectory parent; private final String name; Index: lucene/src/java/org/apache/lucene/store/RAMDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/RAMDirectory.java (revision 947247) +++ lucene/src/java/org/apache/lucene/store/RAMDirectory.java (working copy) @@ -23,6 +23,8 @@ import java.util.HashMap; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; + +import org.apache.lucene.index.IndexFileNameFilter; import org.apache.lucene.util.ThreadInterruptedException; /** @@ -68,7 +70,16 @@ private RAMDirectory(Directory dir, boolean closeDir) throws IOException { this(); - Directory.copy(dir, this, closeDir); + + IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); + for (String file : dir.listAll()) { + if (filter.accept(null, file)) { + dir.copy(this, file, file); + } + } + if (closeDir) { + dir.close(); + } } @Override Index: lucene/src/test/org/apache/lucene/index/TestAddIndexes.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (revision 944220) +++ lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (working copy) @@ -30,7 +30,8 @@ import org.apache.lucene.search.PhraseQuery; -public class TestAddIndexesNoOptimize extends LuceneTestCase { +public class TestAddIndexes extends LuceneTestCase { + public void testSimpleCase() throws IOException { // main directory Directory dir = new RAMDirectory(); @@ -65,7 +66,7 @@ // test doc count before segments are merged writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); assertEquals(100, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux, aux2 }); + writer.addIndexes(new Directory[] { aux, aux2 }); assertEquals(190, writer.maxDoc()); writer.close(); @@ -86,7 +87,7 @@ // test doc count before segments are merged/index is optimized writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); assertEquals(190, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux3 }); + writer.addIndexes(new Directory[] { aux3 }); assertEquals(230, writer.maxDoc()); writer.close(); @@ -117,7 +118,7 @@ writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); assertEquals(230, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux4 }); + writer.addIndexes(new Directory[] { aux4 }); assertEquals(231, writer.maxDoc()); writer.close(); @@ -134,7 +135,7 @@ setUpDirs(dir, aux); IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - writer.addIndexesNoOptimize(new Directory[] {aux}); + writer.addIndexes(new Directory[] {aux}); // Adds 10 docs, then replaces them with another 10 // docs, so 10 pending deletes: @@ -177,13 +178,12 @@ for (int i = 0; i < 20; i++) { Document doc = new Document(); doc.add(new Field("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED)); - doc.add(new Field("content", "bbb " + i, Field.Store.NO, - Field.Index.ANALYZED)); + doc.add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED)); writer.updateDocument(new Term("id", "" + (i%10)), doc); } - - writer.addIndexesNoOptimize(new Directory[] {aux}); - + + writer.addIndexes(new Directory[] {aux}); + // Deletes one of the 10 added docs, leaving 9: PhraseQuery q = new PhraseQuery(); q.add(new Term("content", "bbb")); @@ -227,7 +227,7 @@ q.add(new Term("content", "14")); writer.deleteDocuments(q); - writer.addIndexesNoOptimize(new Directory[] {aux}); + writer.addIndexes(new Directory[] {aux}); writer.optimize(); writer.commit(); @@ -271,7 +271,7 @@ writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); try { // cannot add self - writer.addIndexesNoOptimize(new Directory[] { aux, dir }); + writer.addIndexes(new Directory[] { aux, dir }); assertTrue(false); } catch (IllegalArgumentException e) { @@ -284,7 +284,7 @@ } // in all the remaining tests, make the doc count of the oldest segment - // in dir large so that it is never merged in addIndexesNoOptimize() + // in dir large so that it is never merged in addIndexes() // case 1: no tail segments public void testNoTailSegments() throws IOException { // main directory @@ -300,9 +300,8 @@ ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); addDocs(writer, 10); - writer.addIndexesNoOptimize(new Directory[] { aux }); + writer.addIndexes(new Directory[] { aux }); assertEquals(1040, writer.maxDoc()); - assertEquals(2, writer.getSegmentCount()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -323,9 +322,8 @@ ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); addDocs(writer, 2); - writer.addIndexesNoOptimize(new Directory[] { aux }); + writer.addIndexes(new Directory[] { aux }); assertEquals(1032, writer.maxDoc()); - assertEquals(2, writer.getSegmentCount()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -347,7 +345,7 @@ .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - writer.addIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) }); + writer.addIndexes(new Directory[] { aux, new RAMDirectory(aux) }); assertEquals(1060, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -377,13 +375,10 @@ .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(4)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - writer.addIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) }); - assertEquals(1020, writer.maxDoc()); + writer.addIndexes(new Directory[] { aux, new RAMDirectory(aux) }); + assertEquals(1060, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1020); } // case 5: tail segments, invariants not hold @@ -400,9 +395,8 @@ TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(100)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); - writer.addIndexesNoOptimize(new Directory[] { aux }); + writer.addIndexes(new Directory[] { aux }); assertEquals(30, writer.maxDoc()); - assertEquals(3, writer.getSegmentCount()); writer.close(); IndexReader reader = IndexReader.open(aux, false); @@ -423,13 +417,10 @@ .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(6)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - writer.addIndexesNoOptimize(new Directory[] { aux, aux2 }); - assertEquals(1025, writer.maxDoc()); + writer.addIndexes(new Directory[] { aux, aux2 }); + assertEquals(1060, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1025); } private IndexWriter newWriter(Directory dir, IndexWriterConfig conf) @@ -543,7 +534,7 @@ writer = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) .setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(lmp)); - writer.addIndexesNoOptimize(new Directory[] {dir}); + writer.addIndexes(new Directory[] {dir}); writer.close(); dir.close(); dir2.close(); @@ -563,7 +554,8 @@ writer = newWriter(other, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(true); - writer.addIndexesNoOptimize(new Directory[] {dir}); + writer.addIndexes(new Directory[] {dir}); + writer.optimize(); // nocommit assertTrue(writer.newestSegment().getUseCompoundFile()); writer.close(); } Index: lucene/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (revision 947247) +++ lucene/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (working copy) @@ -1,570 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.analysis.WhitespaceAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.store.MockRAMDirectory; - -import org.apache.lucene.search.PhraseQuery; - -public class TestAddIndexesNoOptimize extends LuceneTestCase { - public void testSimpleCase() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // two auxiliary directories - Directory aux = new RAMDirectory(); - Directory aux2 = new RAMDirectory(); - - IndexWriter writer = null; - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, - new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.CREATE)); - // add 100 documents - addDocs(writer, 100); - assertEquals(100, writer.maxDoc()); - writer.close(); - - writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file - // add 40 documents in separate files - addDocs(writer, 40); - assertEquals(40, writer.maxDoc()); - writer.close(); - - writer = newWriter(aux2, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE)); - // add 40 documents in compound files - addDocs2(writer, 50); - assertEquals(50, writer.maxDoc()); - writer.close(); - - // test doc count before segments are merged - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - assertEquals(100, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux, aux2 }); - assertEquals(190, writer.maxDoc()); - writer.close(); - - // make sure the old index is correct - verifyNumDocs(aux, 40); - - // make sure the new index is correct - verifyNumDocs(dir, 190); - - // now add another set in. - Directory aux3 = new RAMDirectory(); - writer = newWriter(aux3, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - // add 40 documents - addDocs(writer, 40); - assertEquals(40, writer.maxDoc()); - writer.close(); - - // test doc count before segments are merged/index is optimized - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - assertEquals(190, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux3 }); - assertEquals(230, writer.maxDoc()); - writer.close(); - - // make sure the new index is correct - verifyNumDocs(dir, 230); - - verifyTermDocs(dir, new Term("content", "aaa"), 180); - - verifyTermDocs(dir, new Term("content", "bbb"), 50); - - // now optimize it. - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - writer.optimize(); - writer.close(); - - // make sure the new index is correct - verifyNumDocs(dir, 230); - - verifyTermDocs(dir, new Term("content", "aaa"), 180); - - verifyTermDocs(dir, new Term("content", "bbb"), 50); - - // now add a single document - Directory aux4 = new RAMDirectory(); - writer = newWriter(aux4, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - addDocs2(writer, 1); - writer.close(); - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - assertEquals(230, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux4 }); - assertEquals(231, writer.maxDoc()); - writer.close(); - - verifyNumDocs(dir, 231); - - verifyTermDocs(dir, new Term("content", "bbb"), 51); - } - - public void testWithPendingDeletes() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - writer.addIndexesNoOptimize(new Directory[] {aux}); - - // Adds 10 docs, then replaces them with another 10 - // docs, so 10 pending deletes: - for (int i = 0; i < 20; i++) { - Document doc = new Document(); - doc.add(new Field("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED)); - doc.add(new Field("content", "bbb " + i, Field.Store.NO, - Field.Index.ANALYZED)); - writer.updateDocument(new Term("id", "" + (i%10)), doc); - } - // Deletes one of the 10 added docs, leaving 9: - PhraseQuery q = new PhraseQuery(); - q.add(new Term("content", "bbb")); - q.add(new Term("content", "14")); - writer.deleteDocuments(q); - - writer.optimize(); - writer.commit(); - - verifyNumDocs(dir, 1039); - verifyTermDocs(dir, new Term("content", "aaa"), 1030); - verifyTermDocs(dir, new Term("content", "bbb"), 9); - - writer.close(); - dir.close(); - aux.close(); - } - - public void testWithPendingDeletes2() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - - // Adds 10 docs, then replaces them with another 10 - // docs, so 10 pending deletes: - for (int i = 0; i < 20; i++) { - Document doc = new Document(); - doc.add(new Field("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED)); - doc.add(new Field("content", "bbb " + i, Field.Store.NO, - Field.Index.ANALYZED)); - writer.updateDocument(new Term("id", "" + (i%10)), doc); - } - - writer.addIndexesNoOptimize(new Directory[] {aux}); - - // Deletes one of the 10 added docs, leaving 9: - PhraseQuery q = new PhraseQuery(); - q.add(new Term("content", "bbb")); - q.add(new Term("content", "14")); - writer.deleteDocuments(q); - - writer.optimize(); - writer.commit(); - - verifyNumDocs(dir, 1039); - verifyTermDocs(dir, new Term("content", "aaa"), 1030); - verifyTermDocs(dir, new Term("content", "bbb"), 9); - - writer.close(); - dir.close(); - aux.close(); - } - - public void testWithPendingDeletes3() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - - // Adds 10 docs, then replaces them with another 10 - // docs, so 10 pending deletes: - for (int i = 0; i < 20; i++) { - Document doc = new Document(); - doc.add(new Field("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED)); - doc.add(new Field("content", "bbb " + i, Field.Store.NO, - Field.Index.ANALYZED)); - writer.updateDocument(new Term("id", "" + (i%10)), doc); - } - - // Deletes one of the 10 added docs, leaving 9: - PhraseQuery q = new PhraseQuery(); - q.add(new Term("content", "bbb")); - q.add(new Term("content", "14")); - writer.deleteDocuments(q); - - writer.addIndexesNoOptimize(new Directory[] {aux}); - - writer.optimize(); - writer.commit(); - - verifyNumDocs(dir, 1039); - verifyTermDocs(dir, new Term("content", "aaa"), 1030); - verifyTermDocs(dir, new Term("content", "bbb"), 9); - - writer.close(); - dir.close(); - aux.close(); - } - - // case 0: add self or exceed maxMergeDocs, expect exception - public void testAddSelf() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - IndexWriter writer = null; - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - // add 100 documents - addDocs(writer, 100); - assertEquals(100, writer.maxDoc()); - writer.close(); - - writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file - // add 140 documents in separate files - addDocs(writer, 40); - writer.close(); - writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file - addDocs(writer, 100); - writer.close(); - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - try { - // cannot add self - writer.addIndexesNoOptimize(new Directory[] { aux, dir }); - assertTrue(false); - } - catch (IllegalArgumentException e) { - assertEquals(100, writer.maxDoc()); - } - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 100); - } - - // in all the remaining tests, make the doc count of the oldest segment - // in dir large so that it is never merged in addIndexesNoOptimize() - // case 1: no tail segments - public void testNoTailSegments() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - - IndexWriter writer = newWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - addDocs(writer, 10); - - writer.addIndexesNoOptimize(new Directory[] { aux }); - assertEquals(1040, writer.maxDoc()); - assertEquals(2, writer.getSegmentCount()); - assertEquals(1000, writer.getDocCount(0)); - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1040); - } - - // case 2: tail segments, invariants hold, no copy - public void testNoCopySegments() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - - IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(9)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - addDocs(writer, 2); - - writer.addIndexesNoOptimize(new Directory[] { aux }); - assertEquals(1032, writer.maxDoc()); - assertEquals(2, writer.getSegmentCount()); - assertEquals(1000, writer.getDocCount(0)); - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1032); - } - - // case 3: tail segments, invariants hold, copy, invariants hold - public void testNoMergeAfterCopy() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - - IndexWriter writer = newWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - - writer.addIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) }); - assertEquals(1060, writer.maxDoc()); - assertEquals(1000, writer.getDocCount(0)); - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1060); - } - - // case 4: tail segments, invariants hold, copy, invariants not hold - public void testMergeAfterCopy() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - - IndexReader reader = IndexReader.open(aux, false); - for (int i = 0; i < 20; i++) { - reader.deleteDocument(i); - } - assertEquals(10, reader.numDocs()); - reader.close(); - - IndexWriter writer = newWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(4)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - - writer.addIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) }); - assertEquals(1020, writer.maxDoc()); - assertEquals(1000, writer.getDocCount(0)); - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1020); - } - - // case 5: tail segments, invariants not hold - public void testMoreMerges() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - Directory aux2 = new RAMDirectory(); - - setUpDirs(dir, aux); - - IndexWriter writer = newWriter(aux2, new IndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(100)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); - writer.addIndexesNoOptimize(new Directory[] { aux }); - assertEquals(30, writer.maxDoc()); - assertEquals(3, writer.getSegmentCount()); - writer.close(); - - IndexReader reader = IndexReader.open(aux, false); - for (int i = 0; i < 27; i++) { - reader.deleteDocument(i); - } - assertEquals(3, reader.numDocs()); - reader.close(); - - reader = IndexReader.open(aux2, false); - for (int i = 0; i < 8; i++) { - reader.deleteDocument(i); - } - assertEquals(22, reader.numDocs()); - reader.close(); - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(6)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - - writer.addIndexesNoOptimize(new Directory[] { aux, aux2 }); - assertEquals(1025, writer.maxDoc()); - assertEquals(1000, writer.getDocCount(0)); - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1025); - } - - private IndexWriter newWriter(Directory dir, IndexWriterConfig conf) - throws IOException { - conf.setMergePolicy(new LogDocMergePolicy()); - final IndexWriter writer = new IndexWriter(dir, conf); - return writer; - } - - private void addDocs(IndexWriter writer, int numDocs) throws IOException { - for (int i = 0; i < numDocs; i++) { - Document doc = new Document(); - doc.add(new Field("content", "aaa", Field.Store.NO, - Field.Index.ANALYZED)); - writer.addDocument(doc); - } - } - - private void addDocs2(IndexWriter writer, int numDocs) throws IOException { - for (int i = 0; i < numDocs; i++) { - Document doc = new Document(); - doc.add(new Field("content", "bbb", Field.Store.NO, - Field.Index.ANALYZED)); - writer.addDocument(doc); - } - } - - private void verifyNumDocs(Directory dir, int numDocs) throws IOException { - IndexReader reader = IndexReader.open(dir, true); - assertEquals(numDocs, reader.maxDoc()); - assertEquals(numDocs, reader.numDocs()); - reader.close(); - } - - private void verifyTermDocs(Directory dir, Term term, int numDocs) - throws IOException { - IndexReader reader = IndexReader.open(dir, true); - TermDocs termDocs = reader.termDocs(term); - int count = 0; - while (termDocs.next()) - count++; - assertEquals(numDocs, count); - reader.close(); - } - - private void setUpDirs(Directory dir, Directory aux) throws IOException { - IndexWriter writer = null; - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); - // add 1000 documents in 1 segment - addDocs(writer, 1000); - assertEquals(1000, writer.maxDoc()); - assertEquals(1, writer.getSegmentCount()); - writer.close(); - - writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(100)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); - // add 30 documents in 3 segments - for (int i = 0; i < 3; i++) { - addDocs(writer, 10); - writer.close(); - writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(100)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); - } - assertEquals(30, writer.maxDoc()); - assertEquals(3, writer.getSegmentCount()); - writer.close(); - } - - // LUCENE-1270 - public void testHangOnClose() throws IOException { - - Directory dir = new MockRAMDirectory(); - LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy(); - lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); - lmp.setMergeFactor(100); - IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setMaxBufferedDocs(5).setMergePolicy(lmp)); - - Document doc = new Document(); - doc.add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); - for(int i=0;i<60;i++) - writer.addDocument(doc); - - Document doc2 = new Document(); - doc2.add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - doc2.add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - doc2.add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - doc2.add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - for(int i=0;i<10;i++) - writer.addDocument(doc2); - writer.close(); - - Directory dir2 = new MockRAMDirectory(); - lmp = new LogByteSizeMergePolicy(); - lmp.setMinMergeMB(0.0001); - lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); - lmp.setMergeFactor(4); - writer = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, - new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(lmp)); - writer.addIndexesNoOptimize(new Directory[] {dir}); - writer.close(); - dir.close(); - dir2.close(); - } - - // LUCENE-1642: make sure CFS of destination indexwriter - // is respected when copying tail segments - public void testTargetCFS() throws IOException { - Directory dir = new RAMDirectory(); - IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); - addDocs(writer, 1); - writer.close(); - - Directory other = new RAMDirectory(); - writer = newWriter(other, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(true); - writer.addIndexesNoOptimize(new Directory[] {dir}); - assertTrue(writer.newestSegment().getUseCompoundFile()); - writer.close(); - } -} Index: lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (revision 947247) +++ lucene/src/test/org/apache/lucene/index/TestIndexReaderCloneNorms.java (working copy) @@ -117,7 +117,7 @@ TEST_VERSION_CURRENT, anlzr).setOpenMode(OpenMode.APPEND) .setMaxBufferedDocs(5)); ((LogMergePolicy) iw.getConfig().getMergePolicy()).setMergeFactor(3); - iw.addIndexesNoOptimize(new Directory[] { dir1, dir2 }); + iw.addIndexes(new Directory[] { dir1, dir2 }); iw.optimize(); iw.close(); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 947247) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -159,7 +159,7 @@ /* Test: make sure when we run out of disk space or hit - random IOExceptions in any of the addIndexesNoOptimize(*) calls + random IOExceptions in any of the addIndexes(*) calls that 1) index is not corrupt (searcher can open/search it) and 2) transactional semantics are followed: either all or none of the incoming documents were in @@ -172,7 +172,7 @@ int END_COUNT = START_COUNT + NUM_DIR*25; // Build up a bunch of dirs that have indexes which we - // will then merge together by calling addIndexesNoOptimize(*): + // will then merge together by calling addIndexes(*): Directory[] dirs = new Directory[NUM_DIR]; long inputDiskUsage = 0; for(int i=0;i 0) { openWriter(); - writer.addIndexesNoOptimize(dirs); + writer.addIndexes(dirs); rc = 1; } else { rc = 0;