Index: lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java =================================================================== --- lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java (revision 0) +++ lucene/src/java/org/apache/lucene/index/UpgradeIndexMergePolicy.java (revision 0) @@ -0,0 +1,122 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.util.Constants; + +import java.io.IOException; +import java.util.HashSet; +import java.util.Set; + +/** This {@link MergePolicy} is solely used for upgrading all existing segments of an index when calling {@link IndexWriter#optimize} */ +public final class UpgradeIndexMergePolicy extends MergePolicy { + + private final MergePolicy base; + + public UpgradeIndexMergePolicy(MergePolicy base) { + this.base = base; + } + + @Override + public void setIndexWriter(IndexWriter writer) { + super.setIndexWriter(writer); + base.setIndexWriter(writer); + } + + @Override + public MergeSpecification findMerges(SegmentInfos segmentInfos) { + message("UpgradeIndexMergePolicy should only be used for upgrading old indexes through IndexWriter.optimize(), this class will not do any conventional merges"); + return null; + } + + @Override + public MergeSpecification findMergesForOptimize(SegmentInfos segmentInfos, int maxSegmentCount, Set segmentsToOptimize) throws CorruptIndexException, IOException { + // first find all old segments + final HashSet oldSegments = new HashSet(); + for(int i = 0, numSegments = segmentInfos.size(); i < numSegments; i++) { + final SegmentInfo si = segmentInfos.info(i); + if (segmentsToOptimize.contains(si) && !Constants.LUCENE_MAIN_VERSION.equals(si.getVersion())) { + oldSegments.add(si); + } + } + + if (verbose()) message("findMergesForOptimize: segmentsToUpgrade=" + oldSegments); + + if (oldSegments.isEmpty()) + return null; + + MergeSpecification spec = base.findMergesForOptimize(segmentInfos, maxSegmentCount, oldSegments); + + if (spec == null) { + spec = new MergeSpecification(); + // all segments left over in oldSegments, we add them later + } else { + // remove all segments that are in merge specification from oldSegments, + // the resulting set contains all segments that need to be upgraded one-by-one + for (final OneMerge om : spec.merges) { + oldSegments.removeAll(om.segments); + } + } + + if (!oldSegments.isEmpty()) { + if (verbose()) + message("findMergesForOptimize: " + base.getClass().getSimpleName() + + " does not want to merge all old segments, so merge remaining one-by-one: " + oldSegments); + for(int i = 0, numSegments = segmentInfos.size(); i < numSegments; i++) { + final SegmentInfo si = segmentInfos.info(i); + if (oldSegments.contains(si)) { + spec.add(new OneMerge(segmentInfos.range(i, i+1))); + } + } + } + + return spec.merges.isEmpty() ? null : spec; + } + + @Override + public MergeSpecification findMergesToExpungeDeletes(SegmentInfos segmentInfos) { + message("UpgradeIndexMergePolicy should only be used for upgrading old indexes through IndexWriter.optimize(), this class will not do any conventional merges"); + return null; + } + + @Override + public boolean useCompoundFile(SegmentInfos segments, SegmentInfo newSegment) throws IOException { + return base.useCompoundFile(segments, newSegment); + } + + @Override + public void close() { + base.close(); + } + + @Override + public String toString() { + return "[" + getClass().getSimpleName() + "]"; + } + + private boolean verbose() { + IndexWriter w = writer.get(); + return w != null && w.verbose(); + } + + private void message(String message) { + if (verbose()) + writer.get().message("UPGMP: " + message); + } + +} Property changes on: lucene\src\java\org\apache\lucene\index\UpgradeIndexMergePolicy.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: lucene/src/test/org/apache/lucene/index/index.31.optimized.cfs.zip =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: lucene\src\test\org\apache\lucene\index\index.31.optimized.cfs.zip ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: lucene/src/test/org/apache/lucene/index/index.31.optimized.nocfs.zip =================================================================== Cannot display: file marked as a binary type. svn:mime-type = application/octet-stream Property changes on: lucene\src\test\org\apache\lucene\index\index.31.optimized.nocfs.zip ___________________________________________________________________ Added: svn:mime-type + application/octet-stream Index: lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 1100748) +++ lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy) @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.PrintStream; import java.util.Arrays; +import java.util.ArrayList; import java.util.List; import java.util.Random; @@ -45,6 +46,7 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.Constants; /* Verify we can read the pre-4.0 file format, do searches @@ -110,6 +112,10 @@ "29.nocfs", }; + final String[] oldOptimizedNames = {"31.optimized.cfs", + "31.optimized.nocfs", + }; + /** This test checks that *only* IndexFormatTooOldExceptions are throws when you open and operate on too old indexes! */ public void testUnsupportedOldIndexes() throws Exception { for(int i=0;i names = new ArrayList(oldNames.length + oldOptimizedNames.length); + names.addAll(Arrays.asList(oldNames)); + names.addAll(Arrays.asList(oldOptimizedNames)); + for(String name : names) { + if (VERBOSE) { + System.out.println("testUpgradeOldIndex: index=" +name); + } + File oldIndxeDir = _TestUtil.getTempDir(name); + _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir); + Directory dir = newFSDirectory(oldIndxeDir); + + IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); + iwc = iwc.setMergePolicy(new UpgradeIndexMergePolicy(iwc.getMergePolicy())); + IndexWriter w = new IndexWriter(dir, iwc); + w.setInfoStream(VERBOSE ? System.out : null); + w.optimize(); + w.close(); + + assertEquals("Index must be optimized, as all previous segments were old format", + 1, checkAllSegmentsUptodate(dir)); + + _TestUtil.checkIndex(dir); + + dir.close(); + _TestUtil.rmDir(oldIndxeDir); + } + } + + public void testUpgradeOldOptimizedIndexWithAdditions() throws Exception { + for (String name : oldOptimizedNames) { + if (VERBOSE) { + System.out.println("testUpgradeOldOptimizedIndexWithAdditions: index=" +name); + } + File oldIndxeDir = _TestUtil.getTempDir(name); + _TestUtil.unzip(getDataFile("index." + name + ".zip"), oldIndxeDir); + Directory dir = newFSDirectory(oldIndxeDir); + + IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + w.setInfoStream(VERBOSE ? System.out : null); + // add 10 docs + for(int i=0;i<10;i++) { + addDoc(w, 35+i); + } + w.close(); + + IndexWriterConfig iwc = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); + iwc = iwc.setMergePolicy(new UpgradeIndexMergePolicy(iwc.getMergePolicy())); + w = new IndexWriter(dir, iwc); + w.setInfoStream(VERBOSE ? System.out : null); + w.optimize(); + w.close(); + + assertTrue("Index must contain more than one segment", + checkAllSegmentsUptodate(dir) > 1); + + dir.close(); + _TestUtil.rmDir(oldIndxeDir); + } + } + }