Index: lucene/src/test/org/apache/lucene/TestExternalCodecs.java =================================================================== --- lucene/src/test/org/apache/lucene/TestExternalCodecs.java (revision 1035198) +++ lucene/src/test/org/apache/lucene/TestExternalCodecs.java (working copy) @@ -480,7 +480,7 @@ } @Override - public void files(Directory dir, SegmentInfo segmentInfo, Set files) { + public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set files) { } } @@ -549,7 +549,7 @@ @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize); + PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId); PostingsReaderBase pulsingReader = new PulsingPostingsReaderImpl(docsReader); // Terms dict index reader @@ -561,7 +561,8 @@ state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - reverseUnicodeComparator); + reverseUnicodeComparator, + state.codecId); success = true; } finally { if (!success) { @@ -579,7 +580,8 @@ pulsingReader, state.readBufferSize, reverseUnicodeComparator, - StandardCodec.TERMS_CACHE_SIZE); + StandardCodec.TERMS_CACHE_SIZE, + state.codecId); success = true; return ret; } finally { @@ -594,10 +596,10 @@ } @Override - public void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, files); - PrefixCodedTermsReader.files(dir, segmentInfo, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, files); + public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set files) throws IOException { + StandardPostingsReader.files(dir, segmentInfo, codecId, files); + PrefixCodedTermsReader.files(dir, segmentInfo, codecId, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); } @Override Index: lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java =================================================================== --- lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (revision 1035198) +++ lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (working copy) @@ -162,7 +162,7 @@ PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, state.readBufferSize, - new MockIntFactory()); + new MockIntFactory(), state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -171,7 +171,8 @@ state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator()); + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.codecId); success = true; } finally { if (!success) { @@ -188,7 +189,8 @@ postingsReader, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - StandardCodec.TERMS_CACHE_SIZE); + StandardCodec.TERMS_CACHE_SIZE, + state.codecId); success = true; return ret; } finally { @@ -203,10 +205,10 @@ } @Override - public void files(Directory dir, SegmentInfo segmentInfo, Set files) { - SepPostingsReaderImpl.files(segmentInfo, files); - PrefixCodedTermsReader.files(dir, segmentInfo, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, files); + public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set files) { + SepPostingsReaderImpl.files(segmentInfo, codecId, files); + PrefixCodedTermsReader.files(dir, segmentInfo, codecId, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); } @Override Index: lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java =================================================================== --- lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (revision 1035198) +++ lucene/src/test/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (working copy) @@ -139,7 +139,7 @@ PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, state.readBufferSize, - new MockIntFactory()); + new MockIntFactory(), state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -148,7 +148,7 @@ state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator()); + BytesRef.getUTF8SortedAsUnicodeComparator(), state.codecId); success = true; } finally { if (!success) { @@ -165,7 +165,8 @@ postingsReader, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - StandardCodec.TERMS_CACHE_SIZE); + StandardCodec.TERMS_CACHE_SIZE, + state.codecId); success = true; return ret; } finally { @@ -180,10 +181,10 @@ } @Override - public void files(Directory dir, SegmentInfo segmentInfo, Set files) { - SepPostingsReaderImpl.files(segmentInfo, files); - PrefixCodedTermsReader.files(dir, segmentInfo, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, files); + public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set files) { + SepPostingsReaderImpl.files(segmentInfo, codecId, files); + PrefixCodedTermsReader.files(dir, segmentInfo, codecId, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); } @Override Index: lucene/src/test/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java =================================================================== --- lucene/src/test/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (revision 1035198) +++ lucene/src/test/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (working copy) @@ -87,7 +87,8 @@ @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, state.readBufferSize, new MockSingleIntFactory()); + PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, + state.readBufferSize, new MockSingleIntFactory(), state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -96,7 +97,8 @@ state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator()); + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.codecId); success = true; } finally { if (!success) { @@ -113,7 +115,8 @@ postingsReader, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - StandardCodec.TERMS_CACHE_SIZE); + StandardCodec.TERMS_CACHE_SIZE, + state.codecId); success = true; return ret; } finally { @@ -128,10 +131,10 @@ } @Override - public void files(Directory dir, SegmentInfo segmentInfo, Set files) { - SepPostingsReaderImpl.files(segmentInfo, files); - PrefixCodedTermsReader.files(dir, segmentInfo, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, files); + public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set files) { + SepPostingsReaderImpl.files(segmentInfo, codecId, files); + PrefixCodedTermsReader.files(dir, segmentInfo, codecId, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); } @Override Index: lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java (revision 1035198) +++ lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java (working copy) @@ -28,7 +28,10 @@ import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec; +import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec; import org.apache.lucene.index.codecs.mocksep.MockSepCodec; +import org.apache.lucene.index.codecs.pulsing.PulsingCodec; import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.search.IndexSearcher; @@ -255,15 +258,16 @@ final int docsPerRound = 97; for (int i = 0; i < 5; i++) { CodecProvider provider = new CodecProvider(); - provider.register(new StandardCodec()); - provider.register(new SimpleTextCodec()); - // provider.register(new MockSepCodec()); // TODO enable once we have - // files per codec - // provider.register(new PulsingCodec()); - + Codec[] codecs = new Codec[] { new StandardCodec(), + new SimpleTextCodec(), new MockSepCodec(), + new PulsingCodec(1 + random.nextInt(10)), + new MockVariableIntBlockCodec(1 + random.nextInt(10)), + new MockFixedIntBlockCodec(1 + random.nextInt(10)) }; + for (Codec codec : codecs) { + provider.register(codec); + } for (int j = 0; j < 30 * RANDOM_MULTIPLIER; j++) { - provider.setFieldCodec("" + j, random.nextBoolean() ? "SimpleText" - : "Standard"); // TODO enable other codecs once possible + provider.setFieldCodec("" + j, codecs[random.nextInt(codecs.length)].name); } IndexWriterConfig config = newIndexWriterConfig(random, TEST_VERSION_CURRENT, new MockAnalyzer()); Index: lucene/src/java/org/apache/lucene/index/SegmentCodecs.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentCodecs.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/SegmentCodecs.java (working copy) @@ -18,7 +18,6 @@ */ import java.io.IOException; import java.util.ArrayList; -import java.util.HashSet; import java.util.IdentityHashMap; import java.util.Map; import java.util.Set; @@ -120,14 +119,11 @@ void files(Directory dir, SegmentInfo info, Set files) throws IOException { - final Set seen = new HashSet(); final Codec[] codecArray = codecs; - for (Codec codec : codecArray) { - if (!seen.contains(codec)) { - seen.add(codec); - codec.files(dir, info, files); - } - } + for (int i = 0; i < codecArray.length; i++) { + codecArray[i].files(dir, info, ""+i, files); + } + } @Override Index: lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (working copy) @@ -61,9 +61,7 @@ assert segmentCodecs == state.segmentCodecs; final Codec[] codecs = segmentCodecs.codecs; for (int i = 0; i < codecs.length; i++) { - state.currentCodecId = i; // actual codec should use that to create its - // files - consumers.add(codecs[i].fieldsConsumer(state)); + consumers.add(codecs[i].fieldsConsumer(new SegmentWriteState(state, "" + i))); } } @@ -111,7 +109,7 @@ Codec codec = segmentCodecs.codecs[fi.codecId]; if (!producers.containsKey(codec)) { producers.put(codec, codec.fieldsProducer(new SegmentReadState(dir, - si, fieldInfos, readBufferSize, indexDivisor))); + si, fieldInfos, readBufferSize, indexDivisor, ""+fi.codecId))); } codecs.put(fi.name, producers.get(codec)); } @@ -195,8 +193,9 @@ } @Override - public void files(Directory dir, SegmentInfo info, Set files) + public void files(Directory dir, SegmentInfo info, String codecId, Set files) throws IOException { + // ignore codecid sicne segmentCodec will assign it per codec segmentCodecs.files(dir, info, files); } Index: lucene/src/java/org/apache/lucene/index/SegmentReadState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentReadState.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/SegmentReadState.java (working copy) @@ -34,16 +34,24 @@ // that must do so), then it should negate this value to // get the app's terms divisor: public final int termsIndexDivisor; + public final String codecId; + public SegmentReadState(Directory dir, SegmentInfo info, + FieldInfos fieldInfos, int readBufferSize, int termsIndexDivisor) { + this(dir, info, fieldInfos, readBufferSize, termsIndexDivisor, ""); + } + public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, int readBufferSize, - int termsIndexDivisor) { + int termsIndexDivisor, + String codecId) { this.dir = dir; this.segmentInfo = info; this.fieldInfos = fieldInfos; this.readBufferSize = readBufferSize; this.termsIndexDivisor = termsIndexDivisor; + this.codecId = codecId; } } \ No newline at end of file Index: lucene/src/java/org/apache/lucene/index/SegmentWriteState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentWriteState.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/SegmentWriteState.java (working copy) @@ -37,7 +37,7 @@ public final Collection flushedFiles; final SegmentCodecs segmentCodecs; - public int currentCodecId; + public final String codecId; /** Expert: The fraction of terms in the "dictionary" which should be stored * in RAM. Smaller values use more memory, but make searching slightly @@ -73,5 +73,23 @@ this.termIndexInterval = termIndexInterval; this.segmentCodecs = segmentCodecs; flushedFiles = new HashSet(); + codecId = ""; } + + /** + * Create a shallow {@link SegmentWriteState} copy final a codec ID + */ + SegmentWriteState(SegmentWriteState state, String codecId) { + infoStream = state.infoStream; + directory = state.directory; + segmentName = state.segmentName; + fieldInfos = state.fieldInfos; + docStoreSegmentName = state.docStoreSegmentName; + numDocs = state.numDocs; + numDocsInStore = state.numDocsInStore; + termIndexInterval = state.termIndexInterval; + segmentCodecs = state.segmentCodecs; + flushedFiles = state.flushedFiles; + this.codecId = codecId; + } } Index: lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (working copy) @@ -108,7 +108,7 @@ // We wrap StandardPostingsReader, but any StandardPostingsReader // will work: - PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize); + PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId); PostingsReaderBase pulsingReader = new PulsingPostingsReaderImpl(docsReader); // Terms dict index reader @@ -120,7 +120,8 @@ state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator()); + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.codecId); success = true; } finally { if (!success) { @@ -136,7 +137,8 @@ pulsingReader, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - StandardCodec.TERMS_CACHE_SIZE); + StandardCodec.TERMS_CACHE_SIZE, + state.codecId); success = true; return ret; } finally { @@ -151,10 +153,10 @@ } @Override - public void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, files); - PrefixCodedTermsReader.files(dir, segmentInfo, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, files); + public void files(Directory dir, SegmentInfo segmentInfo, String id, Set files) throws IOException { + StandardPostingsReader.files(dir, segmentInfo, id, files); + PrefixCodedTermsReader.files(dir, segmentInfo, id, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, id, files); } @Override Index: lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsWriter.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsWriter.java (working copy) @@ -69,7 +69,7 @@ PostingsWriterBase postingsWriter, Comparator termComp) throws IOException { - final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, "", TERMS_EXTENSION); + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); this.termsIndexWriter = termsIndexWriter; this.termComp = termComp; out = state.directory.createOutput(termsFileName); Index: lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java (working copy) @@ -90,12 +90,12 @@ // start of the field info data protected long dirOffset; - public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp) + public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, String codecId) throws IOException { this.termComp = termComp; - IndexInput in = dir.openInput(IndexFileNames.segmentFileName(segment, "", FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + IndexInput in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); boolean success = false; @@ -436,8 +436,8 @@ return fields.get(fieldInfo); } - public static void files(Directory dir, SegmentInfo info, Collection files) { - files.add(IndexFileNames.segmentFileName(info.name, "", FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + public static void files(Directory dir, SegmentInfo info, String id, Collection files) { + files.add(IndexFileNames.segmentFileName(info.name, id, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); } public static void getIndexExtensions(Collection extensions) { Index: lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java (working copy) @@ -44,7 +44,7 @@ final static BytesRef PAYLOAD = new BytesRef(" payload "); public SimpleTextFieldsWriter(SegmentWriteState state) throws IOException { - final String fileName = SimpleTextCodec.getPostingsFileName(state.segmentName); + final String fileName = SimpleTextCodec.getPostingsFileName(state.segmentName, state.codecId); out = state.directory.createOutput(fileName); state.flushedFiles.add(fileName); } Index: lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (working copy) @@ -56,7 +56,8 @@ final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD; public SimpleTextFieldsReader(SegmentReadState state) throws IOException { - in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name)); + in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, ""+state.codecId)); + fieldInfos = state.fieldInfos; } Index: lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java (working copy) @@ -56,13 +56,13 @@ /** Extension of freq postings file */ static final String POSTINGS_EXTENSION = "pst"; - static String getPostingsFileName(String segment) { + static String getPostingsFileName(String segment, String id) { return IndexFileNames.segmentFileName(segment, "", POSTINGS_EXTENSION); } @Override - public void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException { - files.add(getPostingsFileName(segmentInfo.name)); + public void files(Directory dir, SegmentInfo segmentInfo, String id, Set files) throws IOException { + files.add(getPostingsFileName(segmentInfo.name, id)); } @Override Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java (working copy) @@ -60,14 +60,14 @@ public StandardPostingsWriter(SegmentWriteState state) throws IOException { super(); - String fileName = IndexFileNames.segmentFileName(state.segmentName, "", StandardCodec.FREQ_EXTENSION); + String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.FREQ_EXTENSION); state.flushedFiles.add(fileName); freqOut = state.directory.createOutput(fileName); if (state.fieldInfos.hasProx()) { // At least one field does not omit TF, so create the // prox file - fileName = IndexFileNames.segmentFileName(state.segmentName, "", StandardCodec.PROX_EXTENSION); + fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION); state.flushedFiles.add(fileName); proxOut = state.directory.createOutput(fileName); } else { Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (working copy) @@ -45,13 +45,13 @@ int skipInterval; int maxSkipLevels; - public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize) throws IOException { - freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, "", StandardCodec.FREQ_EXTENSION), + public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize, String codecId) throws IOException { + freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.FREQ_EXTENSION), readBufferSize); if (segmentInfo.getHasProx()) { boolean success = false; try { - proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, "", StandardCodec.PROX_EXTENSION), + proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.PROX_EXTENSION), readBufferSize); success = true; } finally { @@ -64,10 +64,10 @@ } } - public static void files(Directory dir, SegmentInfo segmentInfo, Collection files) throws IOException { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", StandardCodec.FREQ_EXTENSION)); + public static void files(Directory dir, SegmentInfo segmentInfo, String id, Collection files) throws IOException { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, StandardCodec.FREQ_EXTENSION)); if (segmentInfo.getHasProx()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", StandardCodec.PROX_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, StandardCodec.PROX_EXTENSION)); } } Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (working copy) @@ -84,7 +84,7 @@ @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize); + PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -93,7 +93,8 @@ state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator()); + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.codecId); success = true; } finally { if (!success) { @@ -110,7 +111,8 @@ postings, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - TERMS_CACHE_SIZE); + TERMS_CACHE_SIZE, + state.codecId); success = true; return ret; } finally { @@ -131,10 +133,10 @@ static final String PROX_EXTENSION = "prx"; @Override - public void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, files); - PrefixCodedTermsReader.files(dir, segmentInfo, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, files); + public void files(Directory dir, SegmentInfo segmentInfo, String id, Set files) throws IOException { + StandardPostingsReader.files(dir, segmentInfo, id, files); + PrefixCodedTermsReader.files(dir, segmentInfo, id, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, id, files); } @Override Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (working copy) @@ -66,7 +66,8 @@ } @Override - public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + public void files(Directory dir, SegmentInfo info, String id, Set files) throws IOException { + // preflex fields have no codec ID - we ignore it here PreFlexFields.files(dir, info, files); } Index: lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java (working copy) @@ -49,7 +49,7 @@ private IndexOutput termsOut; public FixedGapTermsIndexWriter(SegmentWriteState state) throws IOException { - final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, "", TERMS_INDEX_EXTENSION); + final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); state.flushedFiles.add(indexFileName); termIndexInterval = state.termIndexInterval; out = state.directory.createOutput(indexFileName); Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java (working copy) @@ -54,20 +54,20 @@ int skipInterval; int maxSkipLevels; - public SepPostingsReaderImpl(Directory dir, SegmentInfo segmentInfo, int readBufferSize, IntStreamFactory intFactory) throws IOException { + public SepPostingsReaderImpl(Directory dir, SegmentInfo segmentInfo, int readBufferSize, IntStreamFactory intFactory, String codecId) throws IOException { boolean success = false; try { - final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.DOC_EXTENSION); + final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.DOC_EXTENSION); docIn = intFactory.openInput(dir, docFileName); - skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.SKIP_EXTENSION), readBufferSize); + skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.SKIP_EXTENSION), readBufferSize); if (segmentInfo.getHasProx()) { - freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.FREQ_EXTENSION)); - posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.POS_EXTENSION), readBufferSize); - payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.PAYLOAD_EXTENSION), readBufferSize); + freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.FREQ_EXTENSION)); + posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.POS_EXTENSION), readBufferSize); + payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.PAYLOAD_EXTENSION), readBufferSize); } else { posIn = null; payloadIn = null; @@ -81,14 +81,14 @@ } } - public static void files(SegmentInfo segmentInfo, Collection files) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.DOC_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.SKIP_EXTENSION)); + public static void files(SegmentInfo segmentInfo, String codecId, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.DOC_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.SKIP_EXTENSION)); if (segmentInfo.getHasProx()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.FREQ_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.POS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", SepPostingsWriterImpl.PAYLOAD_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.FREQ_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.POS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.PAYLOAD_EXTENSION)); } } Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java (working copy) @@ -84,24 +84,24 @@ public SepPostingsWriterImpl(SegmentWriteState state, IntStreamFactory factory) throws IOException { super(); - final String docFileName = IndexFileNames.segmentFileName(state.segmentName, "", DOC_EXTENSION); + final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, DOC_EXTENSION); state.flushedFiles.add(docFileName); docOut = factory.createOutput(state.directory, docFileName); docIndex = docOut.index(); if (state.fieldInfos.hasProx()) { - final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, "", FREQ_EXTENSION); + final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, FREQ_EXTENSION); state.flushedFiles.add(frqFileName); freqOut = factory.createOutput(state.directory, frqFileName); freqIndex = freqOut.index(); - final String posFileName = IndexFileNames.segmentFileName(state.segmentName, "", POS_EXTENSION); + final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, POS_EXTENSION); posOut = factory.createOutput(state.directory, posFileName); state.flushedFiles.add(posFileName); posIndex = posOut.index(); // TODO: -- only if at least one field stores payloads? - final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, "", PAYLOAD_EXTENSION); + final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, PAYLOAD_EXTENSION); state.flushedFiles.add(payloadFileName); payloadOut = state.directory.createOutput(payloadFileName); @@ -113,7 +113,7 @@ payloadOut = null; } - final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, "", SKIP_EXTENSION); + final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SKIP_EXTENSION); state.flushedFiles.add(skipFileName); skipOut = state.directory.createOutput(skipFileName); Index: lucene/src/java/org/apache/lucene/index/codecs/Codec.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/Codec.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/Codec.java (working copy) @@ -51,8 +51,15 @@ * use; else, those files may be deleted. */ public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException; - /** Gathers files associated with this segment */ - public abstract void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException; + /** + * Gathers files associated with this segment + * + * @param dir the {@link Directory} this segment was written to + * @param segmentInfo the {@link SegmentInfo} for this segment + * @param id the codec id within this segment + * @param files the of files to add the codec files to. + */ + public abstract void files(Directory dir, SegmentInfo segmentInfo, String id, Set files) throws IOException; /** Records all file extensions this codec uses */ public abstract void getExtensions(Set extensions); Index: lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java (revision 1035198) +++ lucene/src/java/org/apache/lucene/index/codecs/PrefixCodedTermsReader.java (working copy) @@ -112,7 +112,7 @@ } public PrefixCodedTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, int readBufferSize, - Comparator termComp, int termsCacheSize) + Comparator termComp, int termsCacheSize, String codecId) throws IOException { this.postingsReader = postingsReader; @@ -120,7 +120,7 @@ this.termComp = termComp; - in = dir.openInput(IndexFileNames.segmentFileName(segment, "", PrefixCodedTermsWriter.TERMS_EXTENSION), + in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, PrefixCodedTermsWriter.TERMS_EXTENSION), readBufferSize); boolean success = false; @@ -203,8 +203,8 @@ } } - public static void files(Directory dir, SegmentInfo segmentInfo, Collection files) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, "", PrefixCodedTermsWriter.TERMS_EXTENSION)); + public static void files(Directory dir, SegmentInfo segmentInfo, String id, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, PrefixCodedTermsWriter.TERMS_EXTENSION)); } public static void getExtensions(Collection extensions) { Index: lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java (revision 1035198) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java (working copy) @@ -1,6 +1,6 @@ package org.apache.lucene.index.codecs.appending; -/* +/** * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. @@ -88,7 +88,7 @@ @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize); + PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -97,7 +97,8 @@ state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator()); + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.codecId); success = true; } finally { if (!success) { @@ -111,7 +112,8 @@ docsReader, state.readBufferSize, BytesRef.getUTF8SortedAsUnicodeComparator(), - StandardCodec.TERMS_CACHE_SIZE); + StandardCodec.TERMS_CACHE_SIZE, + state.codecId); success = true; return ret; } finally { @@ -126,11 +128,11 @@ } @Override - public void files(Directory dir, SegmentInfo segmentInfo, Set files) + public void files(Directory dir, SegmentInfo segmentInfo, String codecId, Set files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, files); - PrefixCodedTermsReader.files(dir, segmentInfo, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, files); + StandardPostingsReader.files(dir, segmentInfo, codecId, files); + PrefixCodedTermsReader.files(dir, segmentInfo, codecId, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); } @Override Index: lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java (revision 1035198) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java (working copy) @@ -35,9 +35,9 @@ public AppendingTermsDictReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, int readBufferSize, - Comparator termComp, int termsCacheSize) throws IOException { + Comparator termComp, int termsCacheSize, String codecId) throws IOException { super(indexReader, dir, fieldInfos, segment, postingsReader, readBufferSize, - termComp, termsCacheSize); + termComp, termsCacheSize, codecId); } @Override Index: lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java (revision 1035198) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java (working copy) @@ -30,9 +30,9 @@ public class AppendingTermsIndexReader extends FixedGapTermsIndexReader { public AppendingTermsIndexReader(Directory dir, FieldInfos fieldInfos, - String segment, int indexDivisor, Comparator termComp) + String segment, int indexDivisor, Comparator termComp, String codecId) throws IOException { - super(dir, fieldInfos, segment, indexDivisor, termComp); + super(dir, fieldInfos, segment, indexDivisor, termComp, codecId); } @Override