Index: lucene/codecs/src/java/org/apache/lucene/codecs/appending/AppendingCodec.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/appending/AppendingCodec.java (révision 1387486) +++ lucene/codecs/src/java/org/apache/lucene/codecs/appending/AppendingCodec.java (copie de travail) @@ -20,6 +20,8 @@ import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.FilterCodec; import org.apache.lucene.codecs.PostingsFormat; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.filter.FilterCodecSegmentInfoFormat; import org.apache.lucene.codecs.lucene40.Lucene40Codec; /** @@ -31,15 +33,24 @@ */ public final class AppendingCodec extends FilterCodec { + public static final String CODEC_NAME = "Appending"; + + private Codec delegate; + public AppendingCodec() { - super("Appending"); + this(new Lucene40Codec()); } + private AppendingCodec(Codec delegate) { + super(CODEC_NAME); + this.delegate = delegate; + } + private final PostingsFormat postings = new AppendingPostingsFormat(); @Override protected Codec delegate() { - return Codec.forName("Lucene40"); + return delegate; } @Override @@ -47,4 +58,14 @@ return postings; } + @Override + public SegmentInfoFormat segmentInfoFormat() { + return new FilterCodecSegmentInfoFormat(delegate()) { + @Override + protected Codec newCodec(Codec delegate) { + return new AppendingCodec(delegate); + } + }; + } + } Index: lucene/codecs/src/java/org/apache/lucene/codecs/filter/FilterCodecSegmentInfoFormat.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/filter/FilterCodecSegmentInfoFormat.java (révision 0) +++ lucene/codecs/src/java/org/apache/lucene/codecs/filter/FilterCodecSegmentInfoFormat.java (révision 0) @@ -0,0 +1,146 @@ +package org.apache.lucene.codecs.filter; + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.SegmentInfoFormat; +import org.apache.lucene.codecs.SegmentInfoReader; +import org.apache.lucene.codecs.SegmentInfoWriter; +import org.apache.lucene.codecs.appending.AppendingCodec; +import org.apache.lucene.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.IOUtils; + +/** + * A {@link SegmentInfoFormat} that stores and restores the delegate codec name. + *

+ * The purpose of this class is that contrib codecs such {@link AppendingCodec} + * don't break backwards compatibility when a new version of the default Lucene + * {@link Codec} gets released. To do so, this {@link SegmentInfoFormat} writes + * the name of the delegate codec when flushing segments and reads it when + * loading a segment in order to known what the delegate codec was. This way, if + * the {@link AppendingCodec} changes its delegate codec from + * {@link Lucene40Codec} to, say, Lucene43Codec, it will still be able to read + * indices generated with {@link Lucene40Codec} as a delegate codec. + *

+ * Don't use this class with user-provided delegate codecs since this format + * will fail at working with delegate codecs that use this + * {@link SegmentInfoFormat} too. + + * @lucene.internal + */ +public abstract class FilterCodecSegmentInfoFormat extends SegmentInfoFormat { + + private static final String CODEC_NAME = "FilterCodecSegmentInfo"; + private static final int VERSION_START = 0; + private static final int VERSION_CURRENT = VERSION_START; + private static final String DELEGATE_CODEC_EXTENSION = "dlg"; + + private final Codec delegate; + + public FilterCodecSegmentInfoFormat(Codec delegate) { + if (delegate.segmentInfoFormat() instanceof FilterCodecSegmentInfoFormat) { + throw new IllegalArgumentException("Cannot wrap a codec that already uses FilterCodecSegmentInfoFormat"); + } + this.delegate = delegate; + } + + private final SegmentInfoWriter segmentWriter = new SegmentInfoWriter() { + + @Override + public void write(Directory dir, SegmentInfo info, FieldInfos fis, IOContext context) throws IOException { + final String dlgFileName = IndexFileNames.segmentFileName( + info.name, "", DELEGATE_CODEC_EXTENSION); + if (dir.fileExists(dlgFileName)) { + throw new IllegalStateException(dlgFileName + " already exists"); + } + + IndexOutput dlgOut = dir.createOutput(dlgFileName, context); + boolean success = false; + try { + CodecUtil.writeHeader(dlgOut, CODEC_NAME, VERSION_CURRENT); + dlgOut.writeString(delegate.getName()); + dlgOut.close(); + dlgOut = null; + info.addFile(dlgFileName); + delegate.segmentInfoFormat().getSegmentInfoWriter().write(dir, info, fis, context); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(dlgOut); + IOUtils.deleteFilesIgnoringExceptions(dir, dlgFileName); + } + } + } + + }; + + private final SegmentInfoReader segmentReader = new SegmentInfoReader() { + + @Override + public SegmentInfo read(Directory dir, String segmentName, IOContext context) + throws IOException { + final String dlgFileName = IndexFileNames.segmentFileName( + segmentName, "", DELEGATE_CODEC_EXTENSION); + + final Codec delegateCodec; + IndexInput dlgIn = dir.openInput(dlgFileName, context); + boolean success = false; + try { + CodecUtil.checkHeader(dlgIn, CODEC_NAME, VERSION_START, VERSION_CURRENT); + delegateCodec = Codec.forName(dlgIn.readString()); + dlgIn.close(); + dlgIn = null; + + final SegmentInfo segmentInfo = delegateCodec.segmentInfoFormat() + .getSegmentInfoReader().read(dir, segmentName, context); + segmentInfo.setCodec(newCodec(delegateCodec)); + success = true; + return segmentInfo; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(dlgIn); + } + } + + } + + }; + + protected abstract Codec newCodec(Codec delegate); + + @Override + public SegmentInfoReader getSegmentInfoReader() { + return segmentReader; + } + + @Override + public SegmentInfoWriter getSegmentInfoWriter() { + return segmentWriter; + } + +} Modification de propriétés sur lucene/codecs/src/java/org/apache/lucene/codecs/filter/FilterCodecSegmentInfoFormat.java ___________________________________________________________________ Ajouté : svn:eol-style + native Index: lucene/codecs/src/java/org/apache/lucene/codecs/filter/package.html =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/filter/package.html (révision 0) +++ lucene/codecs/src/java/org/apache/lucene/codecs/filter/package.html (révision 0) @@ -0,0 +1,25 @@ + + + + + + + +Utility classes to work with {@link org.apache.lucene.codecs.FilterCodec}. + + \ No newline at end of file Modification de propriétés sur lucene/codecs/src/java/org/apache/lucene/codecs/filter/package.html ___________________________________________________________________ Ajouté : svn:eol-style + native Index: lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (révision 1387486) +++ lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java (copie de travail) @@ -24,15 +24,17 @@ import java.util.Map; import java.util.TreeMap; +import org.apache.lucene.codecs.CodecUtil; import org.apache.lucene.codecs.FieldsConsumer; import org.apache.lucene.codecs.FieldsProducer; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; // javadocs +import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.OrdTermState; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; @@ -40,10 +42,13 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.RAMOutputStream; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; import org.apache.lucene.util.automaton.CompiledAutomaton; import org.apache.lucene.util.automaton.RunAutomaton; import org.apache.lucene.util.automaton.Transition; @@ -52,7 +57,7 @@ // - build depth-N prefix hash? // - or: longer dense skip lists than just next byte? -/** Wraps {@link Lucene40PostingsFormat} format for on-disk +/** Wraps any postings format for on-disk * storage, but then at read time loads and stores all * terms & postings directly in RAM as byte[], int[]. * @@ -73,6 +78,13 @@ public final class DirectPostingsFormat extends PostingsFormat { + private static final String CODEC_NAME = "Direct"; + private static final int VERSION_START = 0; + private static final int VERSION_CURRENT = VERSION_START; + private static final String WRAPPED_CODEC_EXTENSION = "wrp"; + + private final PostingsFormat wrappedPostingsFormat; + private final int minSkipCount; private final int lowFreqCutoff; @@ -81,42 +93,80 @@ //private static final boolean DEBUG = true; - // TODO: allow passing/wrapping arbitrary postings format? - + /** + * Create a new instance with the default {@link PostingsFormat}, + * minSkipCount and lowFreqCutOff. + */ public DirectPostingsFormat() { - this(DEFAULT_MIN_SKIP_COUNT, DEFAULT_LOW_FREQ_CUTOFF); + this(new Lucene40PostingsFormat(), + DEFAULT_MIN_SKIP_COUNT, DEFAULT_LOW_FREQ_CUTOFF); } - + /** minSkipCount is how many terms in a row must have the * same prefix before we put a skip pointer down. Terms * with docFreq <= lowFreqCutoff will use a single int[] * to hold all docs, freqs, position and offsets; terms * with higher docFreq will use separate arrays. */ - public DirectPostingsFormat(int minSkipCount, int lowFreqCutoff) { + public DirectPostingsFormat(PostingsFormat wrappedPostingsFormat, int minSkipCount, int lowFreqCutoff) { super("Direct"); + this.wrappedPostingsFormat = wrappedPostingsFormat; this.minSkipCount = minSkipCount; this.lowFreqCutoff = lowFreqCutoff; } @Override public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - return PostingsFormat.forName("Lucene40").fieldsConsumer(state); + final String wrpFileName = IndexFileNames.segmentFileName( + state.segmentInfo.name, state.segmentSuffix, WRAPPED_CODEC_EXTENSION); + IndexOutput wrpOut = state.directory.createOutput(wrpFileName, state.context); + boolean success = false; + try { + CodecUtil.writeHeader(wrpOut, CODEC_NAME, VERSION_CURRENT); + wrpOut.writeString(wrappedPostingsFormat.getName()); + wrpOut.close(); + wrpOut = null; + FieldsConsumer consumer = wrappedPostingsFormat.fieldsConsumer(state); + success = true; + return consumer; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(wrpOut); + IOUtils.deleteFilesIgnoringExceptions(state.directory, wrpFileName); + } + } } @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - FieldsProducer postings = PostingsFormat.forName("Lucene40").fieldsProducer(state); - if (state.context.context != IOContext.Context.MERGE) { - FieldsProducer loadedPostings; - try { - loadedPostings = new DirectFields(state, postings, minSkipCount, lowFreqCutoff); - } finally { - postings.close(); + final String wrpFileName = IndexFileNames.segmentFileName( + state.segmentInfo.name, state.segmentSuffix, WRAPPED_CODEC_EXTENSION); + final PostingsFormat wrappedPostingsFormat; + IndexInput wrpIn = state.dir.openInput(wrpFileName, state.context); + boolean success = false; + try { + CodecUtil.checkHeader(wrpIn, CODEC_NAME, VERSION_START, VERSION_CURRENT); + wrappedPostingsFormat = PostingsFormat.forName(wrpIn.readString()); + wrpIn.close(); + wrpIn = null; + + FieldsProducer postings = wrappedPostingsFormat.fieldsProducer(state); + + if (state.context.context != IOContext.Context.MERGE) { // Don't load postings for merge: + FieldsProducer loadedPostings; + try { + loadedPostings = new DirectFields(state, postings, minSkipCount, lowFreqCutoff); + } finally { + postings.close(); + } + postings = loadedPostings; } - return loadedPostings; - } else { - // Don't load postings for merge: + + success = true; return postings; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(wrpIn); + } } } Index: lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java =================================================================== --- lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (révision 1387486) +++ lucene/codecs/src/java/org/apache/lucene/codecs/bloom/BloomFilteringPostingsFormat.java (copie de travail) @@ -34,6 +34,7 @@ import org.apache.lucene.codecs.TermStats; import org.apache.lucene.codecs.TermsConsumer; import org.apache.lucene.codecs.bloom.FuzzySet.ContainsResult; +import org.apache.lucene.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; @@ -126,19 +127,18 @@ public BloomFilteringPostingsFormat(PostingsFormat delegatePostingsFormat) { this(delegatePostingsFormat, new DefaultBloomFilterFactory()); } - - // Used only by core Lucene at read-time via Service Provider instantiation - - // do not use at Write-time in application code. + + /** + * Create a new instance of {@link BloomFilteringPostingsFormat} that wraps + * the default postings format. + * @see #BloomFilteringPostingsFormat(PostingsFormat) + */ public BloomFilteringPostingsFormat() { - super(BLOOM_CODEC_NAME); + this(new Lucene40PostingsFormat()); } public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - if (delegatePostingsFormat == null) { - throw new UnsupportedOperationException("Error - " + getClass().getName() - + " has been constructed without a choice of PostingsFormat"); - } return new BloomFilteredFieldsConsumer( delegatePostingsFormat.fieldsConsumer(state), state, delegatePostingsFormat); Index: lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (révision 1387486) +++ lucene/test-framework/src/java/org/apache/lucene/index/RandomCodec.java (copie de travail) @@ -96,8 +96,10 @@ add(avoidCodecs, new Lucene40PostingsFormat(minItemsPerBlock, maxItemsPerBlock), new BlockPostingsFormat(minItemsPerBlock, maxItemsPerBlock), - new DirectPostingsFormat(LuceneTestCase.rarely(random) ? 1 : (LuceneTestCase.rarely(random) ? Integer.MAX_VALUE : maxItemsPerBlock), - LuceneTestCase.rarely(random) ? 1 : (LuceneTestCase.rarely(random) ? Integer.MAX_VALUE : lowFreqCutoff)), + new DirectPostingsFormat( + PostingsFormat.forName("Lucene40"), + LuceneTestCase.rarely(random) ? 1 : (LuceneTestCase.rarely(random) ? Integer.MAX_VALUE : maxItemsPerBlock), + LuceneTestCase.rarely(random) ? 1 : (LuceneTestCase.rarely(random) ? Integer.MAX_VALUE : lowFreqCutoff)), new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock), // add pulsing again with (usually) different parameters new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock), Index: lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (révision 1387486) +++ lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (copie de travail) @@ -1103,10 +1103,23 @@ w3.close(); // we should now see segments_X, // segments.gen,_Y.cfs,_Y.cfe, _Z.si - assertEquals("Only one compound segment should exist, but got: " + Arrays.toString(dir.listAll()), 5, dir.listAll().length); + assertEquals("Only one compound segment should exist, but got: " + + Arrays.toString(dir.listAll()), 5, dir.listAll().length - numDlg(dir)); dir.close(); } - + + // required because of FilterCodecs that serialize the name of the delegate + // codec to a separate .dlg file + private static final int numDlg(Directory dir) throws IOException { + int n = 0; + for (String file : dir.listAll()) { + if (file.endsWith(".dlg")) { + ++n; + } + } + return n; + } + private static final class UnRegisteredCodec extends FilterCodec { public UnRegisteredCodec() { super("NotRegistered"); Index: lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (révision 1387486) +++ lucene/core/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (copie de travail) @@ -77,7 +77,7 @@ // we won't even be running the actual code, only the impostor // @SuppressCodecs("Lucene4x") // Sep codec cannot yet handle the offsets in our 4.x index! -@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom"}) +@SuppressCodecs({"MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Appending"}) public class TestBackwardsCompatibility extends LuceneTestCase { // Uncomment these cases & run them on an older Lucene Index: lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java =================================================================== --- lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (révision 1387486) +++ lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java (copie de travail) @@ -290,7 +290,11 @@ Codec codec = Codec.forName(input.readString()); //System.out.println("SIS.read seg=" + seg + " codec=" + codec); SegmentInfo info = codec.segmentInfoFormat().getSegmentInfoReader().read(directory, segName, IOContext.READ); - info.setCodec(codec); + if (info.getCodec() == null) { + info.setCodec(codec); + } else if (!codec.getClass().isInstance(info.getCodec())) { + throw new IllegalStateException("SegmentInfoFormat of " + codec.getName() + " returned a different codec instance"); + } long delGen = input.readLong(); int delCount = input.readInt(); if (delCount < 0 || delCount > info.getDocCount()) {