Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 947247) +++ lucene/CHANGES.txt (working copy) @@ -74,6 +74,17 @@ it cannot delete the lock file, since obtaining the lock does not fail if the file is there. (Shai Erera) +* LUCENE-2455: IndexWriter.addIndexes no longer optimizes the target index + before it adds the new ones. Also, the existing segments are not merged and so + the index will not end up with a single segment (unless it was empty before). + In addition, addIndexesNoOptimize was renamed to addIndexes and no longer + invokes a merge on the incoming and target segments, but instead copies the + segments to the target index. You can call maybeMerge or optimize after this + method completes, if you need to. + In addition, Directory.copyTo* were removed in favor of copy which takes the + target Directory, source and target files as arguments, and copies the source + file to the target Directory under the target file name. (Shai Erera) + API Changes * LUCENE-2076: Rename FSDirectory.getFile -> getDirectory. (George Index: lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java =================================================================== --- lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (revision 947247) +++ lucene/backwards/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (working copy) @@ -298,7 +298,6 @@ writer.addIndexesNoOptimize(new Directory[] { aux }); assertEquals(1040, writer.maxDoc()); - assertEquals(2, writer.getSegmentCount()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -322,7 +321,6 @@ writer.addIndexesNoOptimize(new Directory[] { aux }); assertEquals(1032, writer.maxDoc()); - assertEquals(2, writer.getSegmentCount()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -373,12 +371,9 @@ writer.setMergeFactor(4); writer.addIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) }); - assertEquals(1020, writer.maxDoc()); + assertEquals(1060, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1020); } // case 5: tail segments, invariants not hold @@ -418,12 +413,9 @@ writer.setMergeFactor(4); writer.addIndexesNoOptimize(new Directory[] { aux, aux2 }); - assertEquals(1025, writer.maxDoc()); + assertEquals(1060, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1025); } private IndexWriter newWriter(Directory dir, boolean create) @@ -541,20 +533,4 @@ dir2.close(); } - // LUCENE-1642: make sure CFS of destination indexwriter - // is respected when copying tail segments - public void testTargetCFS() throws IOException { - Directory dir = new RAMDirectory(); - IndexWriter writer = newWriter(dir, true); - writer.setUseCompoundFile(false); - addDocs(writer, 1); - writer.close(); - - Directory other = new RAMDirectory(); - writer = newWriter(other, true); - writer.setUseCompoundFile(true); - writer.addIndexesNoOptimize(new Directory[] {dir}); - assertTrue(writer.newestSegment().getUseCompoundFile()); - writer.close(); - } } Index: lucene/backwards/src/test/org/apache/lucene/index/index.30.cfs.zip =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: lucene\backwards\src\test\org\apache\lucene\index\index.30.cfs.zip ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: lucene/backwards/src/test/org/apache/lucene/index/index.30.nocfs.zip =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: lucene\backwards\src\test\org\apache\lucene\index\index.30.nocfs.zip ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: lucene/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java (revision 947247) +++ lucene/contrib/misc/src/java/org/apache/lucene/misc/IndexMergeTool.java (working copy) @@ -49,7 +49,7 @@ } System.out.println("Merging..."); - writer.addIndexesNoOptimize(indexes); + writer.addIndexes(indexes); System.out.println("Optimizing..."); writer.optimize(); Index: lucene/src/java/org/apache/lucene/index/CompoundFileReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/CompoundFileReader.java (revision 947247) +++ lucene/src/java/org/apache/lucene/index/CompoundFileReader.java (working copy) @@ -55,9 +55,7 @@ this(dir, name, BufferedIndexInput.BUFFER_SIZE); } - public CompoundFileReader(Directory dir, String name, int readBufferSize) - throws IOException - { + public CompoundFileReader(Directory dir, String name, int readBufferSize) throws IOException { directory = dir; fileName = name; this.readBufferSize = readBufferSize; @@ -67,13 +65,32 @@ try { stream = dir.openInput(name, readBufferSize); + // read the first VInt. If it is negative, it's the version number + // otherwise it's the count (pre-3.1 indexes) + int firstInt = stream.readVInt(); + + // Currently the comparison is !=. If more versions are created, we + // can change it accordingly + if (firstInt < CompoundFileWriter.PRE_VERSION && firstInt != CompoundFileWriter.FORMAT_CURRENT) { + throw new CorruptIndexException("Incompatible format version: " + + firstInt + " expected " + CompoundFileWriter.FORMAT_CURRENT); + } + + // If it's a post-3.1 index, read the count. + int count = firstInt < CompoundFileWriter.PRE_VERSION ? stream.readVInt() : firstInt; + // read the directory and init files - int count = stream.readVInt(); FileEntry entry = null; for (int i=0; i 0) { + // Fix the id to not include the segment names. This is relevant for + // pre-3.1 indexes. + id = IndexFileNames.stripSegmentName(id); + } + if (entry != null) { // set length of the previous entry entry.length = offset - entry.offset; @@ -92,7 +109,7 @@ success = true; } finally { - if (! success && (stream != null)) { + if (!success && (stream != null)) { try { stream.close(); } catch (IOException e) { } @@ -132,7 +149,8 @@ { if (stream == null) throw new IOException("Stream closed"); - + + id = IndexFileNames.stripSegmentName(id); FileEntry entry = entries.get(id); if (entry == null) throw new IOException("No sub-file with id " + id + " found"); @@ -143,14 +161,19 @@ /** Returns an array of strings, one for each file in the directory. */ @Override public String[] listAll() { - String res[] = new String[entries.size()]; - return entries.keySet().toArray(res); + String[] res = entries.keySet().toArray(new String[entries.size()]); + // Add the segment name + String seg = fileName.substring(0, fileName.indexOf('.')); + for (int i = 0; i < res.length; i++) { + res[i] = seg + res[i]; + } + return res; } /** Returns true iff a file with the given name exists. */ @Override public boolean fileExists(String name) { - return entries.containsKey(name); + return entries.containsKey(IndexFileNames.stripSegmentName(name)); } /** Returns the time the compound file was last modified. */ @@ -184,7 +207,7 @@ * @throws IOException if the file does not exist */ @Override public long fileLength(String name) throws IOException { - FileEntry e = entries.get(name); + FileEntry e = entries.get(IndexFileNames.stripSegmentName(name)); if (e == null) throw new FileNotFoundException(name); return e.length; Index: lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java (revision 947247) +++ lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java (working copy) @@ -59,7 +59,17 @@ long dataOffset; } + // Before versioning started. + static final int PRE_VERSION = 0; + + // Segment name is not written in the file names. + static final int FORMAT_START = -1; + // NOTE: if you introduce a new format, make it 1 lower + // than the current one, and always change this if you + // switch to a new format! + static final int FORMAT_CURRENT = FORMAT_START; + private Directory directory; private String fileName; private HashSet ids; @@ -146,6 +156,10 @@ try { os = directory.createOutput(fileName); + // Write the Version info - must be a VInt because CFR reads a VInt + // in older versions! + os.writeVInt(FORMAT_CURRENT); + // Write the number of entries os.writeVInt(entries.size()); @@ -156,7 +170,7 @@ for (FileEntry fe : entries) { fe.directoryOffset = os.getFilePointer(); os.writeLong(0); // for now - os.writeString(fe.file); + os.writeString(IndexFileNames.stripSegmentName(fe.file)); totalSize += directory.fileLength(fe.file); } Index: lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 947247) +++ lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (working copy) @@ -613,8 +613,6 @@ files = Collections.unmodifiableCollection(segmentInfos.files(directory, true)); gen = segmentInfos.getGeneration(); isOptimized = segmentInfos.size() == 1 && !segmentInfos.info(0).hasDeletions(); - - assert !segmentInfos.hasExternalSegments(directory); } @Override Index: lucene/src/java/org/apache/lucene/index/IndexFileNames.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexFileNames.java (revision 947247) +++ lucene/src/java/org/apache/lucene/index/IndexFileNames.java (working copy) @@ -238,5 +238,27 @@ // or not, since there's only 1 '+' operator. return filename.endsWith("." + ext); } + + /** + * Strips the segment file name out of the given one. If you used + * {@link #segmentFileName} or {@link #fileNameFromGeneration} to create your + * files, then this method simply removes whatever comes before the first '.', + * or the second '_' (excluding both), in case of deleted docs. + * + * @return the filename with the segment name removed, or the given filename + * if it does not contain a '.' and '_'. + */ + public static final String stripSegmentName(String filename) { + // If it is a .del file, there's an '_' after the first character + int idx = filename.indexOf('_', 1); + if (idx == -1) { + // If it's not, strip everything that's before the '.' + idx = filename.indexOf('.'); + } + if (idx != -1) { + filename = filename.substring(idx); + } + return filename; + } } Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 947247) +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -269,9 +269,6 @@ volatile SegmentInfos pendingCommit; // set when a commit is pending (after prepareCommit() & before commit()) volatile long pendingCommitChangeCount; - private SegmentInfos localRollbackSegmentInfos; // segmentInfos we will fallback to if the commit fails - private int localFlushedDocCount; // saved docWriter.getFlushedDocCount during local transaction - private SegmentInfos segmentInfos = new SegmentInfos(); // the segments private DocumentsWriter docWriter; @@ -1181,7 +1178,6 @@ private synchronized void setRollbackSegmentInfos(SegmentInfos infos) { rollbackSegmentInfos = (SegmentInfos) infos.clone(); - assert !rollbackSegmentInfos.hasExternalSegments(directory); rollbackSegments = new HashMap(); final int size = rollbackSegmentInfos.size(); for(int i=0;i(); - final int numSegments = segmentInfos.size(); - for(int i=0;i(segmentInfos); // Now mark all pending & running merges as optimize // merge: @@ -2626,169 +2619,6 @@ } } - /** Like getNextMerge() except only returns a merge if it's - * external. */ - private synchronized MergePolicy.OneMerge getNextExternalMerge() { - if (pendingMerges.size() == 0) - return null; - else { - Iterator it = pendingMerges.iterator(); - while(it.hasNext()) { - MergePolicy.OneMerge merge = it.next(); - if (merge.isExternal) { - // Advance the merge from pending to running - it.remove(); - runningMerges.add(merge); - return merge; - } - } - - // All existing merges do not involve external segments - return null; - } - } - - /* - * Begin a transaction. During a transaction, any segment - * merges that happen (or ram segments flushed) will not - * write a new segments file and will not remove any files - * that were present at the start of the transaction. You - * must make a matched (try/finally) call to - * commitTransaction() or rollbackTransaction() to finish - * the transaction. - * - * Note that buffered documents and delete terms are not handled - * within the transactions, so they must be flushed before the - * transaction is started. - */ - private synchronized void startTransaction(boolean haveReadLock) throws IOException { - - boolean success = false; - try { - if (infoStream != null) - message("now start transaction"); - - assert docWriter.getNumBufferedDeleteTerms() == 0 : - "calling startTransaction with buffered delete terms not supported: numBufferedDeleteTerms=" + docWriter.getNumBufferedDeleteTerms(); - assert docWriter.getNumDocsInRAM() == 0 : - "calling startTransaction with buffered documents not supported: numDocsInRAM=" + docWriter.getNumDocsInRAM(); - - ensureOpen(); - - // If a transaction is trying to roll back (because - // addIndexes hit an exception) then wait here until - // that's done: - synchronized(this) { - while(stopMerges) - doWait(); - } - success = true; - } finally { - // Release the write lock if our caller held it, on - // hitting an exception - if (!success && haveReadLock) - releaseRead(); - } - - if (haveReadLock) { - upgradeReadToWrite(); - } else { - acquireWrite(); - } - - success = false; - try { - localRollbackSegmentInfos = (SegmentInfos) segmentInfos.clone(); - - assert !hasExternalSegments(); - - localFlushedDocCount = docWriter.getFlushedDocCount(); - - // We must "protect" our files at this point from - // deletion in case we need to rollback: - deleter.incRef(segmentInfos, false); - - success = true; - } finally { - if (!success) - finishAddIndexes(); - } - } - - /* - * Rolls back the transaction and restores state to where - * we were at the start. - */ - private synchronized void rollbackTransaction() throws IOException { - - if (infoStream != null) - message("now rollback transaction"); - - if (docWriter != null) { - docWriter.setFlushedDocCount(localFlushedDocCount); - } - - // Must finish merges before rolling back segmentInfos - // so merges don't hit exceptions on trying to commit - // themselves, don't get files deleted out from under - // them, etc: - finishMerges(false); - - // Keep the same segmentInfos instance but replace all - // of its SegmentInfo instances. This is so the next - // attempt to commit using this instance of IndexWriter - // will always write to a new generation ("write once"). - segmentInfos.clear(); - segmentInfos.addAll(localRollbackSegmentInfos); - localRollbackSegmentInfos = null; - - // This must come after we rollback segmentInfos, so - // that if a commit() kicks off it does not see the - // segmentInfos with external segments - finishAddIndexes(); - - // Ask deleter to locate unreferenced files we had - // created & remove them: - deleter.checkpoint(segmentInfos, false); - - // Remove the incRef we did in startTransaction: - deleter.decRef(segmentInfos); - - // Also ask deleter to remove any newly created files - // that were never incref'd; this "garbage" is created - // when a merge kicks off but aborts part way through - // before it had a chance to incRef the files it had - // partially created - deleter.refresh(); - - notifyAll(); - - assert !hasExternalSegments(); - } - - /* - * Commits the transaction. This will write the new - * segments file and remove and pending deletions we have - * accumulated during the transaction - */ - private synchronized void commitTransaction() throws IOException { - - if (infoStream != null) - message("now commit transaction"); - - // Give deleter a chance to remove files now: - checkpoint(); - - // Remove the incRef we did in startTransaction. - deleter.decRef(localRollbackSegmentInfos); - - localRollbackSegmentInfos = null; - - assert !hasExternalSegments(); - - finishAddIndexes(); - } - /** * Close the IndexWriter without committing * any changes that have occurred since the last commit @@ -2840,8 +2670,6 @@ segmentInfos.clear(); segmentInfos.addAll(rollbackSegmentInfos); - assert !hasExternalSegments(); - docWriter.abort(); assert testPoint("rollback before checkpoint"); @@ -2997,7 +2825,7 @@ assert 0 == mergingSegments.size(); } - /* + /** * Called whenever the SegmentInfos has been updated and * the index files referenced exist (correctly) in the * index directory. @@ -3007,31 +2835,6 @@ deleter.checkpoint(segmentInfos, false); } - private void finishAddIndexes() { - releaseWrite(); - } - - private void blockAddIndexes() { - - acquireRead(); - - boolean success = false; - try { - - // Make sure we are still open since we could have - // waited quite a while for last addIndexes to finish - ensureOpen(false); - success = true; - } finally { - if (!success) - releaseRead(); - } - } - - private void resumeAddIndexes() { - releaseRead(); - } - private synchronized void resetMergeExceptions() { mergeExceptions = new ArrayList(); mergeGen++; @@ -3039,331 +2842,72 @@ private void noDupDirs(Directory... dirs) { HashSet dups = new HashSet(); - for(int i=0;iThis may be used to parallelize batch indexing. A large document - * collection can be broken into sub-collections. Each sub-collection can be - * indexed in parallel, on a different thread, process or machine. The - * complete index can then be created by merging sub-collection indexes - * with this method. - * - *

NOTE: the index in each Directory must not be - * changed (opened by a writer) while this method is - * running. This method does not acquire a write lock in - * each input Directory, so it is up to the caller to - * enforce this. - * - *

NOTE: while this is running, any attempts to - * add or delete documents (with another thread) will be - * paused until this method completes. - * - *

This method is transactional in how Exceptions are - * handled: it does not commit a new segments_N file until - * all indexes are added. This means if an Exception - * occurs (for example disk full), then either no indexes - * will have been added or they all will have been.

- * - *

Note that this requires temporary free space in the - * Directory up to 2X the sum of all input indexes - * (including the starting index). If readers/searchers - * are open against the starting index, then temporary - * free space required will be higher by the size of the - * starting index (see {@link #optimize()} for details). - *

- * - *

Once this completes, the final size of the index - * will be less than the sum of all input index sizes - * (including the starting index). It could be quite a - * bit smaller (if there were many pending deletes) or - * just slightly smaller.

- * - *

- * This requires this index not be among those to be added. - * - *

NOTE: if this method hits an OutOfMemoryError - * you should immediately close the writer. See above for details.

- * - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error + * @deprecated use {@link #addIndexes(Directory...)} instead */ public void addIndexesNoOptimize(Directory... dirs) throws CorruptIndexException, IOException { - - ensureOpen(); - - noDupDirs(dirs); - - // Do not allow add docs or deletes while we are running: - docWriter.pauseAllThreads(); - - try { - if (infoStream != null) - message("flush at addIndexesNoOptimize"); - flush(true, false, true); - - boolean success = false; - - startTransaction(false); - - try { - - int docCount = 0; - synchronized(this) { - ensureOpen(); - - for (int i = 0; i < dirs.length; i++) { - if (directory == dirs[i]) { - // cannot add this index: segments may be deleted in merge before added - throw new IllegalArgumentException("Cannot add this index to itself"); - } - - SegmentInfos sis = new SegmentInfos(); // read infos from dir - sis.read(dirs[i]); - for (int j = 0; j < sis.size(); j++) { - SegmentInfo info = sis.info(j); - assert !segmentInfos.contains(info): "dup info dir=" + info.dir + " name=" + info.name; - docCount += info.docCount; - segmentInfos.add(info); // add each info - } - } - } - - // Notify DocumentsWriter that the flushed count just increased - docWriter.updateFlushedDocCount(docCount); - - maybeMerge(); - - ensureOpen(); - - // If after merging there remain segments in the index - // that are in a different directory, just copy these - // over into our index. This is necessary (before - // finishing the transaction) to avoid leaving the - // index in an unusable (inconsistent) state. - resolveExternalSegments(); - - ensureOpen(); - - success = true; - - } finally { - if (success) { - commitTransaction(); - } else { - rollbackTransaction(); - } - } - } catch (OutOfMemoryError oom) { - handleOOM(oom, "addIndexesNoOptimize"); - } finally { - if (docWriter != null) { - docWriter.resumeAllThreads(); - } - } + addIndexes(dirs); } - private boolean hasExternalSegments() { - return segmentInfos.hasExternalSegments(directory); - } - - /* If any of our segments are using a directory != ours - * then we have to either copy them over one by one, merge - * them (if merge policy has chosen to) or wait until - * currently running merges (in the background) complete. - * We don't return until the SegmentInfos has no more - * external segments. Currently this is only used by - * addIndexesNoOptimize(). */ - private void resolveExternalSegments() throws CorruptIndexException, IOException { - - boolean any = false; - - boolean done = false; - - while(!done) { - SegmentInfo info = null; - MergePolicy.OneMerge merge = null; - synchronized(this) { - - if (stopMerges) - throw new MergePolicy.MergeAbortedException("rollback() was called or addIndexes* hit an unhandled exception"); - - final int numSegments = segmentInfos.size(); - - done = true; - for(int i=0;iAfter this completes, the index is optimized.

- *

The provided IndexReaders are not closed.

+ /** + * Merges the provided indexes into this index. This method is useful + * if you use extensions of {@link IndexReader}. Otherwise, using + * {@link #addIndexes(Directory...)} is highly recommended for performance + * reasons. It uses the {@link MergeScheduler} and {@link MergePolicy} set + * on this writer, which may perform merges in parallel. + * + *

The provided IndexReaders are not closed. * - *

NOTE: while this is running, any attempts to - * add or delete documents (with another thread) will be - * paused until this method completes. + *

NOTE: this method does not merge the current segments, + * only the incoming ones. + * + *

See {@link #addIndexes(Directory...)} for details on transactional + * semantics, temporary free space required in the Directory, + * and non-CFS segments on an Exception. * - *

See {@link #addIndexesNoOptimize} for - * details on transactional semantics, temporary free - * space required in the Directory, and non-CFS segments - * on an Exception.

- * *

NOTE: if this method hits an OutOfMemoryError * you should immediately close the writer. See above for details.

+ * href="#OOME">above for details. * * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public void addIndexes(IndexReader... readers) - throws CorruptIndexException, IOException { + public void addIndexes(IndexReader... readers) throws CorruptIndexException, IOException { ensureOpen(); - // Do not allow add docs or deletes while we are running: - docWriter.pauseAllThreads(); - - // We must pre-acquire a read lock here (and upgrade to - // write lock in startTransaction below) so that no - // other addIndexes is allowed to start up after we have - // flushed & optimized but before we then start our - // transaction. This is because the merging below - // requires that only one segment is present in the - // index: - acquireRead(); - try { - + String mergedName = newSegmentName(); + SegmentMerger merger = new SegmentMerger(this, mergedName, null); + + for (IndexReader reader : readers) // add new indexes + merger.add(reader); + + int docCount = merger.merge(); // merge 'em + SegmentInfo info = null; - String mergedName = null; - SegmentMerger merger = null; - - boolean success = false; - - try { - flush(true, false, true); - optimize(); // start with zero or 1 seg - success = true; - } finally { - // Take care to release the read lock if we hit an - // exception before starting the transaction - if (!success) - releaseRead(); - } - - // true means we already have a read lock; if this - // call hits an exception it will release the write - // lock: - startTransaction(true); - - try { - mergedName = newSegmentName(); - merger = new SegmentMerger(this, mergedName, null); - - SegmentReader sReader = null; - synchronized(this) { - if (segmentInfos.size() == 1) { // add existing index, if any - sReader = readerPool.get(segmentInfos.info(0), true, BufferedIndexInput.BUFFER_SIZE, -1); - } - } + synchronized(this) { + info = new SegmentInfo(mergedName, docCount, directory, false, true, + -1, null, false, merger.hasProx()); + setDiagnostics(info, "addIndexes(IndexReader...)"); + segmentInfos.add(info); + checkpoint(); - success = false; - - try { - if (sReader != null) - merger.add(sReader); - - for (int i = 0; i < readers.length; i++) // add new indexes - merger.add(readers[i]); - - int docCount = merger.merge(); // merge 'em - - synchronized(this) { - segmentInfos.clear(); // pop old infos & add new - info = new SegmentInfo(mergedName, docCount, directory, false, true, - -1, null, false, merger.hasProx()); - setDiagnostics(info, "addIndexes(IndexReader...)"); - segmentInfos.add(info); - } - - // Notify DocumentsWriter that the flushed count just increased - docWriter.updateFlushedDocCount(docCount); - - success = true; - - } finally { - if (sReader != null) { - readerPool.release(sReader); - } - } - } finally { - if (!success) { - if (infoStream != null) - message("hit exception in addIndexes during merge"); - rollbackTransaction(); - } else { - commitTransaction(); - } + // Notify DocumentsWriter that the flushed count just increased + docWriter.updateFlushedDocCount(docCount); } - + + // Now create the compound file if needed if (mergePolicy instanceof LogMergePolicy && getUseCompoundFile()) { List files = null; @@ -3371,7 +2915,7 @@ synchronized(this) { // Must incRef our files so that if another thread // is running merge/optimize, it doesn't delete our - // segment's files before we have a change to + // segment's files before we have a chance to // finish making the compound file. if (segmentInfos.contains(info)) { files = info.files(); @@ -3380,44 +2924,146 @@ } if (files != null) { - - success = false; - - startTransaction(false); - try { merger.createCompoundFile(mergedName + ".cfs"); synchronized(this) { info.setUseCompoundFile(true); } - - success = true; - } finally { - deleter.decRef(files); - - if (!success) { - if (infoStream != null) - message("hit exception building compound file in addIndexes during merge"); - - rollbackTransaction(); - } else { - commitTransaction(); - } } } } } catch (OutOfMemoryError oom) { handleOOM(oom, "addIndexes(IndexReader...)"); - } finally { - if (docWriter != null) { - docWriter.resumeAllThreads(); - } } } /** + * Adds all segments from an array of indexes into this index. + * + *

This may be used to parallelize batch indexing. A large document + * collection can be broken into sub-collections. Each sub-collection can be + * indexed in parallel, on a different thread, process or machine. The + * complete index can then be created by merging sub-collection indexes + * with this method. + * + *

+ * NOTE: the index in each {@link Directory} must not be + * changed (opened by a writer) while this method is + * running. This method does not acquire a write lock in + * each input Directory, so it is up to the caller to + * enforce this. + * + *

This method is transactional in how Exceptions are + * handled: it does not commit a new segments_N file until + * all indexes are added. This means if an Exception + * occurs (for example disk full), then either no indexes + * will have been added or they all will have been. + * + *

Note that this requires temporary free space in the + * {@link Directory} up to 2X the sum of all input indexes + * (including the starting index). If readers/searchers + * are open against the starting index, then temporary + * free space required will be higher by the size of the + * starting index (see {@link #optimize()} for details). + * + *

+ * NOTE: this method only copies the segments of the incomning indexes + * and does not merge them. Therefore deleted documents are not removed and + * the new segments are not merged with the existing ones. Alos, the segments + * are copied as-is, meaning they are not converted to CFS if they aren't, + * and vice-versa. If you wish to do that, you can call {@link #maybeMerge} + * or {@link #optimize} afterwards. + * + *

This requires this index not be among those to be added. + * + *

+ * NOTE: if this method hits an OutOfMemoryError + * you should immediately close the writer. See above for details. + * + * @throws CorruptIndexException if the index is corrupt + * @throws IOException if there is a low-level IO error + */ + public void addIndexes(Directory... dirs) throws CorruptIndexException, IOException { + ensureOpen(); + + noDupDirs(dirs); + + try { + if (infoStream != null) + message("flush at addIndexes(Directory...)"); + flush(true, false, true); + + int docCount = 0; + List infos = new ArrayList(); + for (Directory dir : dirs) { + if (infoStream != null) { + message("process directory " + dir); + } + SegmentInfos sis = new SegmentInfos(); // read infos from dir + sis.read(dir); + Map dsNames = new HashMap(); + for (SegmentInfo info : sis) { + assert !infos.contains(info): "dup info dir=" + info.dir + " name=" + info.name; + + if (infoStream != null) { + message("process segment=" + info.name); + } + docCount += info.docCount; + String newSegName = newSegmentName(); + String dsName = info.getDocStoreSegment(); + + // Determine if the doc store of this segment needs to be copied. It's + // only relevant for segments who share doc store with others, because + // the DS might have been copied already, in which case we just want + // to update the DS name of this SegmentInfo. + // NOTE: pre-3x segments include a null DSName if they don't share doc + // store. So the following code ensures we don't accidentally insert + // 'null' to the map. + String newDsName = newSegName; + boolean docStoreCopied = false; + if (dsNames.containsKey(dsName)) { + newDsName = dsNames.get(dsName); + docStoreCopied = true; + } else if (dsName != null) { + dsNames.put(dsName, newSegName); + docStoreCopied = false; + } + + // Copy the segment files + for (String file : info.files()) { + if (docStoreCopied && IndexFileNames.isDocStoreFile(file)) { + continue; + } + dir.copy(directory, file, newSegName + IndexFileNames.stripSegmentName(file)); + } + + // Update SI appropriately + info.setDocStore(info.getDocStoreOffset(), newDsName, info.getDocStoreIsCompoundFile()); + info.dir = directory; + info.name = newSegName; + + infos.add(info); + } + } + + synchronized (this) { + ensureOpen(); + segmentInfos.addAll(infos); + // Notify DocumentsWriter that the flushed count just increased + docWriter.updateFlushedDocCount(docCount); + + checkpoint(); + } + + } catch (OutOfMemoryError oom) { + handleOOM(oom, "addIndexes(Directory...)"); + } + } + + /** * A hook for extending classes to execute operations after pending added and * deleted documents have been flushed to the Directory but before the change * is committed (new segments_N file written). @@ -4661,52 +4307,37 @@ synchronized(this) { - // Wait for any running addIndexes to complete - // first, then block any from running until we've - // copied the segmentInfos we intend to sync: - blockAddIndexes(); - - // On commit the segmentInfos must never - // reference a segment in another directory: - assert !hasExternalSegments(); - - try { - - assert lastCommitChangeCount <= changeCount; - - if (changeCount == lastCommitChangeCount) { - if (infoStream != null) - message(" skip startCommit(): no changes pending"); - return; - } - - // First, we clone & incref the segmentInfos we intend - // to sync, then, without locking, we sync() each file - // referenced by toSync, in the background. Multiple - // threads can be doing this at once, if say a large - // merge and a small merge finish at the same time: - + assert lastCommitChangeCount <= changeCount; + + if (changeCount == lastCommitChangeCount) { if (infoStream != null) - message("startCommit index=" + segString(segmentInfos) + " changeCount=" + changeCount); - - readerPool.commit(); - - toSync = (SegmentInfos) segmentInfos.clone(); - - if (commitUserData != null) - toSync.setUserData(commitUserData); - - deleter.incRef(toSync, false); - myChangeCount = changeCount; - - Collection files = toSync.files(directory, false); - for(final String fileName: files) { - assert directory.fileExists(fileName): "file " + fileName + " does not exist"; - } - - } finally { - resumeAddIndexes(); + message(" skip startCommit(): no changes pending"); + return; } + + // First, we clone & incref the segmentInfos we intend + // to sync, then, without locking, we sync() each file + // referenced by toSync, in the background. Multiple + // threads can be doing this at once, if say a large + // merge and a small merge finish at the same time: + + if (infoStream != null) + message("startCommit index=" + segString(segmentInfos) + " changeCount=" + changeCount); + + readerPool.commit(); + + toSync = (SegmentInfos) segmentInfos.clone(); + + if (commitUserData != null) + toSync.setUserData(commitUserData); + + deleter.incRef(toSync, false); + myChangeCount = changeCount; + + Collection files = toSync.files(directory, false); + for(final String fileName: files) { + assert directory.fileExists(fileName): "file " + fileName + " does not exist"; + } } assert testPoint("midStartCommit"); @@ -4976,10 +4607,10 @@ * Sets the {@link PayloadProcessorProvider} to use when merging payloads. * Note that the given pcp will be invoked for every segment that * is merged, not only external ones that are given through - * {@link IndexWriter#addIndexes} or {@link IndexWriter#addIndexesNoOptimize}. - * If you want only the payloads of the external segments to be processed, you - * can return null whenever a {@link DirPayloadProcessor} is - * requested for the {@link Directory} of the {@link IndexWriter}. + * {@link #addIndexes}. If you want only the payloads of the external segments + * to be processed, you can return null whenever a + * {@link DirPayloadProcessor} is requested for the {@link Directory} of the + * {@link IndexWriter}. *

* The default is null which means payloads are processed * normally (copied) during segment merges. You can also unset it by passing Index: lucene/src/java/org/apache/lucene/index/SegmentInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfo.java (revision 947247) +++ lucene/src/java/org/apache/lucene/index/SegmentInfo.java (working copy) @@ -521,6 +521,7 @@ docStoreOffset = offset; docStoreSegment = segment; docStoreIsCompoundFile = isCompoundFile; + clearFiles(); } /** Index: lucene/src/java/org/apache/lucene/index/SegmentInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfos.java (revision 947247) +++ lucene/src/java/org/apache/lucene/index/SegmentInfos.java (working copy) @@ -924,15 +924,6 @@ lastGeneration = other.lastGeneration; } - // Used only for testing - public boolean hasExternalSegments(Directory dir) { - final int numSegments = size(); - for(int i=0;iCopy all files of this directory to destination directory. All conflicting files at destination are overwritten

- *

NOTE: this method only copies files that look like index files (ie, have extensions matching the known - * extensions of index files). - *

NOTE: the source directory should not change while this method is running. Otherwise the results are - * undefined and you could easily hit a FileNotFoundException.

- * - * @param to destination directory + * Copies the file src to {@link Directory} to under the new + * file name dest. + *

+ * If you want to copy the entire source directory to the destination one, you + * can do so like this: + * + * 

+   * Directory to; // the directory to copy to
+   * for (String file : dir.listAll()) {
+   *   dir.copy(to, file, newFile); // newFile can be either file, or a new name
+   * }
+   *
+ *

+ * NOTE: this method does not check whether dest exist and will + * overwrite it if it does. */ - public final void copyTo(Directory to) throws IOException { - List filenames = new ArrayList(); - IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); - - for (String name : listAll()) - if (filter.accept(null, name)) - filenames.add(name); - - copyTo(to, filenames); - } - - /** - *

Copy given files of this directory to destination directory. All conflicting files at destination are overwritten

- *

NOTE: the source directory should not change while this method is running. Otherwise the results are - * undefined and you could easily hit a FileNotFoundException.

- *

NOTE: implementations can check if destination directory is of the same type as 'this' and perform optimized copy

- * - * @param to destination directory - * @param filenames file names to be copied - */ - public void copyTo(Directory to, Collection filenames) throws IOException { - byte[] buf = new byte[BufferedIndexOutput.BUFFER_SIZE]; - for (String filename : filenames) { - IndexOutput os = null; - IndexInput is = null; - IOException priorException = null; - try { - // create file in dest directory - os = to.createOutput(filename); - // read current file - is = openInput(filename); - // and copy to dest directory - long len = is.length(); - long readCount = 0; - while (readCount < len) { - int toRead = readCount + BufferedIndexOutput.BUFFER_SIZE > len ? (int) (len - readCount) : BufferedIndexOutput.BUFFER_SIZE; - is.readBytes(buf, 0, toRead); - os.writeBytes(buf, toRead); - readCount += toRead; - } - } catch (IOException ioe) { - priorException = ioe; - } finally { - IOUtils.closeSafely(priorException, os, is); + public void copy(Directory to, String src, String dest) throws IOException { + IndexOutput os = null; + IndexInput is = null; + IOException priorException = null; + int bufSize = BufferedIndexOutput.BUFFER_SIZE; + byte[] buf = new byte[bufSize]; + try { + // create file in dest directory + os = to.createOutput(dest); + // read current file + is = openInput(src); + // and copy to dest directory + long len = is.length(); + long numRead = 0; + while (numRead < len) { + long left = len - numRead; + int toRead = (int) (bufSize < left ? bufSize : left); + is.readBytes(buf, 0, toRead); + os.writeBytes(buf, toRead); + numRead += toRead; } + } catch (IOException ioe) { + priorException = ioe; + } finally { + IOUtils.closeSafely(priorException, os, is); } } /** - * Copy contents of a directory src to a directory dest. If a file in src already exists in dest then the one in dest - * will be blindly overwritten. - *

- *

NOTE: the source directory cannot change while this method is running. Otherwise the results are - * undefined and you could easily hit a FileNotFoundException. - *

- *

NOTE: this method only copies files that look like index files (ie, have extensions matching the known - * extensions of index files). - * - * @param src source directory - * @param dest destination directory - * @param closeDirSrc if true, call {@link #close()} method on source directory - * @deprecated should be replaced with src.copyTo(dest); [src.close();] + * Copy contents of a directory src to a directory dest. If a file in src + * already exists in dest then the one in dest will be blindly overwritten. + *

+ * NOTE: the source directory cannot change while this method is + * running. Otherwise the results are undefined and you could easily hit a + * FileNotFoundException. + *

+ * NOTE: this method only copies files that look like index files (ie, + * have extensions matching the known extensions of index files). + * + * @param src source directory + * @param dest destination directory + * @param closeDirSrc if true, call {@link #close()} method on + * source directory + * @deprecated should be replaced with calls to + * {@link #copy(Directory, String, String)} for every file that + * needs copying. You can use the the following code: + * + * 

+   * IndexFileNameFilter filter = IndexFileNameFilter.getFilter();
+   * for (String file : src.listAll()) {
+   *   if (filter.accept(null, file)) {
+   *     src.copy(dest, file, file);
+   *   }
+   * }
+   *
*/ - @Deprecated public static void copy(Directory src, Directory dest, boolean closeDirSrc) throws IOException { - src.copyTo(dest); - if (closeDirSrc) + IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); + for (String file : src.listAll()) { + if (filter.accept(null, file)) { + src.copy(dest, file, file); + } + } + if (closeDirSrc) { src.close(); + } } /** Index: lucene/src/java/org/apache/lucene/store/FSDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/FSDirectory.java (revision 947247) +++ lucene/src/java/org/apache/lucene/store/FSDirectory.java (working copy) @@ -431,29 +431,27 @@ } @Override - public void copyTo(Directory to, Collection filenames) throws IOException { + public void copy(Directory to, String src, String dest) throws IOException { if (to instanceof FSDirectory) { FSDirectory target = (FSDirectory) to; - - for (String filename : filenames) { - target.ensureCanWrite(filename); - FileChannel input = null; - FileChannel output = null; - IOException priorException = null; - try { - input = new FileInputStream(new File(directory, filename)).getChannel(); - output = new FileOutputStream(new File(target.directory, filename)).getChannel(); - output.transferFrom(input, 0, input.size()); - } catch (IOException ioe) { - priorException = ioe; - } finally { - IOUtils.closeSafely(priorException, input, output); - } + target.ensureCanWrite(dest); + FileChannel input = null; + FileChannel output = null; + IOException priorException = null; + try { + input = new FileInputStream(new File(directory, src)).getChannel(); + output = new FileOutputStream(new File(target.directory, dest)).getChannel(); + output.transferFrom(input, 0, input.size()); + } catch (IOException ioe) { + priorException = ioe; + } finally { + IOUtils.closeSafely(priorException, input, output); } - } else - super.copyTo(to, filenames); + } else { + super.copy(to, src, dest); + } } - + protected static class FSIndexOutput extends BufferedIndexOutput { private final FSDirectory parent; private final String name; Index: lucene/src/java/org/apache/lucene/store/RAMDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/RAMDirectory.java (revision 947247) +++ lucene/src/java/org/apache/lucene/store/RAMDirectory.java (working copy) @@ -23,6 +23,8 @@ import java.util.HashMap; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; + +import org.apache.lucene.index.IndexFileNameFilter; import org.apache.lucene.util.ThreadInterruptedException; /** @@ -68,7 +70,16 @@ private RAMDirectory(Directory dir, boolean closeDir) throws IOException { this(); - Directory.copy(dir, this, closeDir); + + IndexFileNameFilter filter = IndexFileNameFilter.getFilter(); + for (String file : dir.listAll()) { + if (filter.accept(null, file)) { + dir.copy(this, file, file); + } + } + if (closeDir) { + dir.close(); + } } @Override Index: lucene/src/test/org/apache/lucene/index/TestAddIndexes.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (revision 944220) +++ lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (working copy) @@ -30,7 +30,8 @@ import org.apache.lucene.search.PhraseQuery; -public class TestAddIndexesNoOptimize extends LuceneTestCase { +public class TestAddIndexes extends LuceneTestCase { + public void testSimpleCase() throws IOException { // main directory Directory dir = new RAMDirectory(); @@ -65,7 +66,7 @@ // test doc count before segments are merged writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); assertEquals(100, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux, aux2 }); + writer.addIndexes(new Directory[] { aux, aux2 }); assertEquals(190, writer.maxDoc()); writer.close(); @@ -86,7 +87,7 @@ // test doc count before segments are merged/index is optimized writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); assertEquals(190, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux3 }); + writer.addIndexes(new Directory[] { aux3 }); assertEquals(230, writer.maxDoc()); writer.close(); @@ -117,7 +118,7 @@ writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); assertEquals(230, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux4 }); + writer.addIndexes(new Directory[] { aux4 }); assertEquals(231, writer.maxDoc()); writer.close(); @@ -134,7 +135,7 @@ setUpDirs(dir, aux); IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - writer.addIndexesNoOptimize(new Directory[] {aux}); + writer.addIndexes(new Directory[] {aux}); // Adds 10 docs, then replaces them with another 10 // docs, so 10 pending deletes: @@ -177,13 +178,12 @@ for (int i = 0; i < 20; i++) { Document doc = new Document(); doc.add(new Field("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED)); - doc.add(new Field("content", "bbb " + i, Field.Store.NO, - Field.Index.ANALYZED)); + doc.add(new Field("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED)); writer.updateDocument(new Term("id", "" + (i%10)), doc); } - - writer.addIndexesNoOptimize(new Directory[] {aux}); - + + writer.addIndexes(new Directory[] {aux}); + // Deletes one of the 10 added docs, leaving 9: PhraseQuery q = new PhraseQuery(); q.add(new Term("content", "bbb")); @@ -227,7 +227,7 @@ q.add(new Term("content", "14")); writer.deleteDocuments(q); - writer.addIndexesNoOptimize(new Directory[] {aux}); + writer.addIndexes(new Directory[] {aux}); writer.optimize(); writer.commit(); @@ -271,7 +271,7 @@ writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); try { // cannot add self - writer.addIndexesNoOptimize(new Directory[] { aux, dir }); + writer.addIndexes(new Directory[] { aux, dir }); assertTrue(false); } catch (IllegalArgumentException e) { @@ -284,7 +284,7 @@ } // in all the remaining tests, make the doc count of the oldest segment - // in dir large so that it is never merged in addIndexesNoOptimize() + // in dir large so that it is never merged in addIndexes() // case 1: no tail segments public void testNoTailSegments() throws IOException { // main directory @@ -300,9 +300,8 @@ ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); addDocs(writer, 10); - writer.addIndexesNoOptimize(new Directory[] { aux }); + writer.addIndexes(new Directory[] { aux }); assertEquals(1040, writer.maxDoc()); - assertEquals(2, writer.getSegmentCount()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -323,9 +322,8 @@ ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); addDocs(writer, 2); - writer.addIndexesNoOptimize(new Directory[] { aux }); + writer.addIndexes(new Directory[] { aux }); assertEquals(1032, writer.maxDoc()); - assertEquals(2, writer.getSegmentCount()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -347,7 +345,7 @@ .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - writer.addIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) }); + writer.addIndexes(new Directory[] { aux, new RAMDirectory(aux) }); assertEquals(1060, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -377,13 +375,10 @@ .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(4)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - writer.addIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) }); - assertEquals(1020, writer.maxDoc()); + writer.addIndexes(new Directory[] { aux, new RAMDirectory(aux) }); + assertEquals(1060, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1020); } // case 5: tail segments, invariants not hold @@ -400,9 +395,8 @@ TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(100)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); - writer.addIndexesNoOptimize(new Directory[] { aux }); + writer.addIndexes(new Directory[] { aux }); assertEquals(30, writer.maxDoc()); - assertEquals(3, writer.getSegmentCount()); writer.close(); IndexReader reader = IndexReader.open(aux, false); @@ -423,13 +417,10 @@ .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(6)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - writer.addIndexesNoOptimize(new Directory[] { aux, aux2 }); - assertEquals(1025, writer.maxDoc()); + writer.addIndexes(new Directory[] { aux, aux2 }); + assertEquals(1060, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1025); } private IndexWriter newWriter(Directory dir, IndexWriterConfig conf) @@ -543,28 +534,10 @@ writer = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) .setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(lmp)); - writer.addIndexesNoOptimize(new Directory[] {dir}); + writer.addIndexes(new Directory[] {dir}); writer.close(); dir.close(); dir2.close(); } - // LUCENE-1642: make sure CFS of destination indexwriter - // is respected when copying tail segments - public void testTargetCFS() throws IOException { - Directory dir = new RAMDirectory(); - IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); - addDocs(writer, 1); - writer.close(); - - Directory other = new RAMDirectory(); - writer = newWriter(other, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(true); - writer.addIndexesNoOptimize(new Directory[] {dir}); - assertTrue(writer.newestSegment().getUseCompoundFile()); - writer.close(); - } } Index: lucene/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (revision 947247) +++ lucene/src/test/org/apache/lucene/index/TestAddIndexesNoOptimize.java (working copy) @@ -1,570 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.analysis.WhitespaceAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.store.MockRAMDirectory; - -import org.apache.lucene.search.PhraseQuery; - -public class TestAddIndexesNoOptimize extends LuceneTestCase { - public void testSimpleCase() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // two auxiliary directories - Directory aux = new RAMDirectory(); - Directory aux2 = new RAMDirectory(); - - IndexWriter writer = null; - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, - new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.CREATE)); - // add 100 documents - addDocs(writer, 100); - assertEquals(100, writer.maxDoc()); - writer.close(); - - writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file - // add 40 documents in separate files - addDocs(writer, 40); - assertEquals(40, writer.maxDoc()); - writer.close(); - - writer = newWriter(aux2, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE)); - // add 40 documents in compound files - addDocs2(writer, 50); - assertEquals(50, writer.maxDoc()); - writer.close(); - - // test doc count before segments are merged - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - assertEquals(100, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux, aux2 }); - assertEquals(190, writer.maxDoc()); - writer.close(); - - // make sure the old index is correct - verifyNumDocs(aux, 40); - - // make sure the new index is correct - verifyNumDocs(dir, 190); - - // now add another set in. - Directory aux3 = new RAMDirectory(); - writer = newWriter(aux3, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - // add 40 documents - addDocs(writer, 40); - assertEquals(40, writer.maxDoc()); - writer.close(); - - // test doc count before segments are merged/index is optimized - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - assertEquals(190, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux3 }); - assertEquals(230, writer.maxDoc()); - writer.close(); - - // make sure the new index is correct - verifyNumDocs(dir, 230); - - verifyTermDocs(dir, new Term("content", "aaa"), 180); - - verifyTermDocs(dir, new Term("content", "bbb"), 50); - - // now optimize it. - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - writer.optimize(); - writer.close(); - - // make sure the new index is correct - verifyNumDocs(dir, 230); - - verifyTermDocs(dir, new Term("content", "aaa"), 180); - - verifyTermDocs(dir, new Term("content", "bbb"), 50); - - // now add a single document - Directory aux4 = new RAMDirectory(); - writer = newWriter(aux4, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - addDocs2(writer, 1); - writer.close(); - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - assertEquals(230, writer.maxDoc()); - writer.addIndexesNoOptimize(new Directory[] { aux4 }); - assertEquals(231, writer.maxDoc()); - writer.close(); - - verifyNumDocs(dir, 231); - - verifyTermDocs(dir, new Term("content", "bbb"), 51); - } - - public void testWithPendingDeletes() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - writer.addIndexesNoOptimize(new Directory[] {aux}); - - // Adds 10 docs, then replaces them with another 10 - // docs, so 10 pending deletes: - for (int i = 0; i < 20; i++) { - Document doc = new Document(); - doc.add(new Field("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED)); - doc.add(new Field("content", "bbb " + i, Field.Store.NO, - Field.Index.ANALYZED)); - writer.updateDocument(new Term("id", "" + (i%10)), doc); - } - // Deletes one of the 10 added docs, leaving 9: - PhraseQuery q = new PhraseQuery(); - q.add(new Term("content", "bbb")); - q.add(new Term("content", "14")); - writer.deleteDocuments(q); - - writer.optimize(); - writer.commit(); - - verifyNumDocs(dir, 1039); - verifyTermDocs(dir, new Term("content", "aaa"), 1030); - verifyTermDocs(dir, new Term("content", "bbb"), 9); - - writer.close(); - dir.close(); - aux.close(); - } - - public void testWithPendingDeletes2() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - - // Adds 10 docs, then replaces them with another 10 - // docs, so 10 pending deletes: - for (int i = 0; i < 20; i++) { - Document doc = new Document(); - doc.add(new Field("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED)); - doc.add(new Field("content", "bbb " + i, Field.Store.NO, - Field.Index.ANALYZED)); - writer.updateDocument(new Term("id", "" + (i%10)), doc); - } - - writer.addIndexesNoOptimize(new Directory[] {aux}); - - // Deletes one of the 10 added docs, leaving 9: - PhraseQuery q = new PhraseQuery(); - q.add(new Term("content", "bbb")); - q.add(new Term("content", "14")); - writer.deleteDocuments(q); - - writer.optimize(); - writer.commit(); - - verifyNumDocs(dir, 1039); - verifyTermDocs(dir, new Term("content", "aaa"), 1030); - verifyTermDocs(dir, new Term("content", "bbb"), 9); - - writer.close(); - dir.close(); - aux.close(); - } - - public void testWithPendingDeletes3() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - - // Adds 10 docs, then replaces them with another 10 - // docs, so 10 pending deletes: - for (int i = 0; i < 20; i++) { - Document doc = new Document(); - doc.add(new Field("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED)); - doc.add(new Field("content", "bbb " + i, Field.Store.NO, - Field.Index.ANALYZED)); - writer.updateDocument(new Term("id", "" + (i%10)), doc); - } - - // Deletes one of the 10 added docs, leaving 9: - PhraseQuery q = new PhraseQuery(); - q.add(new Term("content", "bbb")); - q.add(new Term("content", "14")); - writer.deleteDocuments(q); - - writer.addIndexesNoOptimize(new Directory[] {aux}); - - writer.optimize(); - writer.commit(); - - verifyNumDocs(dir, 1039); - verifyTermDocs(dir, new Term("content", "aaa"), 1030); - verifyTermDocs(dir, new Term("content", "bbb"), 9); - - writer.close(); - dir.close(); - aux.close(); - } - - // case 0: add self or exceed maxMergeDocs, expect exception - public void testAddSelf() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - IndexWriter writer = null; - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - // add 100 documents - addDocs(writer, 100); - assertEquals(100, writer.maxDoc()); - writer.close(); - - writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file - // add 140 documents in separate files - addDocs(writer, 40); - writer.close(); - writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file - addDocs(writer, 100); - writer.close(); - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND)); - try { - // cannot add self - writer.addIndexesNoOptimize(new Directory[] { aux, dir }); - assertTrue(false); - } - catch (IllegalArgumentException e) { - assertEquals(100, writer.maxDoc()); - } - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 100); - } - - // in all the remaining tests, make the doc count of the oldest segment - // in dir large so that it is never merged in addIndexesNoOptimize() - // case 1: no tail segments - public void testNoTailSegments() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - - IndexWriter writer = newWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - addDocs(writer, 10); - - writer.addIndexesNoOptimize(new Directory[] { aux }); - assertEquals(1040, writer.maxDoc()); - assertEquals(2, writer.getSegmentCount()); - assertEquals(1000, writer.getDocCount(0)); - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1040); - } - - // case 2: tail segments, invariants hold, no copy - public void testNoCopySegments() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - - IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(9)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - addDocs(writer, 2); - - writer.addIndexesNoOptimize(new Directory[] { aux }); - assertEquals(1032, writer.maxDoc()); - assertEquals(2, writer.getSegmentCount()); - assertEquals(1000, writer.getDocCount(0)); - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1032); - } - - // case 3: tail segments, invariants hold, copy, invariants hold - public void testNoMergeAfterCopy() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - - IndexWriter writer = newWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - - writer.addIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) }); - assertEquals(1060, writer.maxDoc()); - assertEquals(1000, writer.getDocCount(0)); - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1060); - } - - // case 4: tail segments, invariants hold, copy, invariants not hold - public void testMergeAfterCopy() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - - setUpDirs(dir, aux); - - IndexReader reader = IndexReader.open(aux, false); - for (int i = 0; i < 20; i++) { - reader.deleteDocument(i); - } - assertEquals(10, reader.numDocs()); - reader.close(); - - IndexWriter writer = newWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(4)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - - writer.addIndexesNoOptimize(new Directory[] { aux, new RAMDirectory(aux) }); - assertEquals(1020, writer.maxDoc()); - assertEquals(1000, writer.getDocCount(0)); - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1020); - } - - // case 5: tail segments, invariants not hold - public void testMoreMerges() throws IOException { - // main directory - Directory dir = new RAMDirectory(); - // auxiliary directory - Directory aux = new RAMDirectory(); - Directory aux2 = new RAMDirectory(); - - setUpDirs(dir, aux); - - IndexWriter writer = newWriter(aux2, new IndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(100)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); - writer.addIndexesNoOptimize(new Directory[] { aux }); - assertEquals(30, writer.maxDoc()); - assertEquals(3, writer.getSegmentCount()); - writer.close(); - - IndexReader reader = IndexReader.open(aux, false); - for (int i = 0; i < 27; i++) { - reader.deleteDocument(i); - } - assertEquals(3, reader.numDocs()); - reader.close(); - - reader = IndexReader.open(aux2, false); - for (int i = 0; i < 8; i++) { - reader.deleteDocument(i); - } - assertEquals(22, reader.numDocs()); - reader.close(); - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(6)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - - writer.addIndexesNoOptimize(new Directory[] { aux, aux2 }); - assertEquals(1025, writer.maxDoc()); - assertEquals(1000, writer.getDocCount(0)); - writer.close(); - - // make sure the index is correct - verifyNumDocs(dir, 1025); - } - - private IndexWriter newWriter(Directory dir, IndexWriterConfig conf) - throws IOException { - conf.setMergePolicy(new LogDocMergePolicy()); - final IndexWriter writer = new IndexWriter(dir, conf); - return writer; - } - - private void addDocs(IndexWriter writer, int numDocs) throws IOException { - for (int i = 0; i < numDocs; i++) { - Document doc = new Document(); - doc.add(new Field("content", "aaa", Field.Store.NO, - Field.Index.ANALYZED)); - writer.addDocument(doc); - } - } - - private void addDocs2(IndexWriter writer, int numDocs) throws IOException { - for (int i = 0; i < numDocs; i++) { - Document doc = new Document(); - doc.add(new Field("content", "bbb", Field.Store.NO, - Field.Index.ANALYZED)); - writer.addDocument(doc); - } - } - - private void verifyNumDocs(Directory dir, int numDocs) throws IOException { - IndexReader reader = IndexReader.open(dir, true); - assertEquals(numDocs, reader.maxDoc()); - assertEquals(numDocs, reader.numDocs()); - reader.close(); - } - - private void verifyTermDocs(Directory dir, Term term, int numDocs) - throws IOException { - IndexReader reader = IndexReader.open(dir, true); - TermDocs termDocs = reader.termDocs(term); - int count = 0; - while (termDocs.next()) - count++; - assertEquals(numDocs, count); - reader.close(); - } - - private void setUpDirs(Directory dir, Directory aux) throws IOException { - IndexWriter writer = null; - - writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); - // add 1000 documents in 1 segment - addDocs(writer, 1000); - assertEquals(1000, writer.maxDoc()); - assertEquals(1, writer.getSegmentCount()); - writer.close(); - - writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(100)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); - // add 30 documents in 3 segments - for (int i = 0; i < 3; i++) { - addDocs(writer, 10); - writer.close(); - writer = newWriter(aux, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(100)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); - } - assertEquals(30, writer.maxDoc()); - assertEquals(3, writer.getSegmentCount()); - writer.close(); - } - - // LUCENE-1270 - public void testHangOnClose() throws IOException { - - Directory dir = new MockRAMDirectory(); - LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy(); - lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); - lmp.setMergeFactor(100); - IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setMaxBufferedDocs(5).setMergePolicy(lmp)); - - Document doc = new Document(); - doc.add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); - for(int i=0;i<60;i++) - writer.addDocument(doc); - - Document doc2 = new Document(); - doc2.add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - doc2.add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - doc2.add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - doc2.add(new Field("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - for(int i=0;i<10;i++) - writer.addDocument(doc2); - writer.close(); - - Directory dir2 = new MockRAMDirectory(); - lmp = new LogByteSizeMergePolicy(); - lmp.setMinMergeMB(0.0001); - lmp.setUseCompoundFile(false); - lmp.setUseCompoundDocStore(false); - lmp.setMergeFactor(4); - writer = new IndexWriter(dir2, new IndexWriterConfig(TEST_VERSION_CURRENT, - new WhitespaceAnalyzer(TEST_VERSION_CURRENT)) - .setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(lmp)); - writer.addIndexesNoOptimize(new Directory[] {dir}); - writer.close(); - dir.close(); - dir2.close(); - } - - // LUCENE-1642: make sure CFS of destination indexwriter - // is respected when copying tail segments - public void testTargetCFS() throws IOException { - Directory dir = new RAMDirectory(); - IndexWriter writer = newWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); - addDocs(writer, 1); - writer.close(); - - Directory other = new RAMDirectory(); - writer = newWriter(other, new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(true); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(true); - writer.addIndexesNoOptimize(new Directory[] {dir}); - assertTrue(writer.newestSegment().getUseCompoundFile()); - writer.close(); - } -} Index: lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 947247) +++ lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy) @@ -44,6 +44,7 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -134,6 +135,8 @@ "24.nocfs", "29.cfs", "29.nocfs", + "30.cfs", + "30.nocfs", }; private void assertCompressedFields29(Directory dir, boolean shouldStillBeCompressed) throws IOException { @@ -233,6 +236,46 @@ assertEquals("test for compressed field should have run 4 times", 4, hasTested29); } + public void testAddOldIndexes() throws IOException { + for (String name : oldNames) { + unzip(getDataFile("index." + name + ".zip"), name); + String fullPath = fullDir(name); + Directory dir = FSDirectory.open(new File(fullPath)); + + Directory targetDir = new RAMDirectory(); + IndexWriter w = new IndexWriter(targetDir, new IndexWriterConfig( + TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); + w.addIndexes(new Directory[] { dir }); + w.close(); + + _TestUtil.checkIndex(targetDir); + + dir.close(); + rmDir(name); + } + } + + public void testAddOldIndexesReader() throws IOException { + for (String name : oldNames) { + unzip(getDataFile("index." + name + ".zip"), name); + String fullPath = fullDir(name); + Directory dir = FSDirectory.open(new File(fullPath)); + IndexReader reader = IndexReader.open(dir); + + Directory targetDir = new RAMDirectory(); + IndexWriter w = new IndexWriter(targetDir, new IndexWriterConfig( + TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); + w.addIndexes(new IndexReader[] { reader }); + w.close(); + reader.close(); + + _TestUtil.checkIndex(targetDir); + + dir.close(); + rmDir(name); + } + } + public void testSearchOldIndex() throws IOException { for(int i=0;i 0) { openWriter(); - writer.addIndexesNoOptimize(dirs); + writer.addIndexes(dirs); rc = 1; } else { rc = 0;