Index: lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java (revision 1084539) +++ lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java (working copy) @@ -19,6 +19,8 @@ import java.io.IOException; +import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; + abstract class DocConsumerPerThread { /** Process the document. If there is @@ -27,7 +29,7 @@ * DocumentsWriter.DocWriter and return it. * DocumentsWriter then calls finish() on this object * when it's its turn. */ - abstract DocumentsWriter.DocWriter processDocument(FieldInfos fieldInfos) throws IOException; + abstract DocumentsWriter.DocWriter processDocument(FieldInfos fieldInfos, SegmentCodecsBuilder builder) throws IOException; abstract void doAfterFlush(); abstract void abort(); Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (revision 1084539) +++ lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (working copy) @@ -24,6 +24,7 @@ import java.io.IOException; import org.apache.lucene.document.Document; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; @@ -124,7 +125,7 @@ } @Override - public DocumentsWriter.DocWriter processDocument(FieldInfos fieldInfos) throws IOException { + public DocumentsWriter.DocWriter processDocument(FieldInfos fieldInfos, SegmentCodecsBuilder segmentCodecsBuilder) throws IOException { consumer.startDocument(); fieldsWriter.startDocument(); @@ -165,6 +166,9 @@ FieldInfo fi = fieldInfos.add(fieldName, field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions()); + if (fi.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) { + segmentCodecsBuilder.tryAddAndSet(fi); + } fp = new DocFieldProcessorPerField(this, fi); fp.next = fieldHash[hashPos]; fieldHash[hashPos] = fp; @@ -176,6 +180,9 @@ fp.fieldInfo.update(field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(), field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions()); + if (fp.fieldInfo.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) { + segmentCodecsBuilder.tryAddAndSet(fp.fieldInfo); + } if (thisFieldGen != fp.lastGen) { Index: lucene/src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (revision 1084539) +++ lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -29,6 +29,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; import org.apache.lucene.search.Query; import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.store.AlreadyClosedException; @@ -270,6 +271,7 @@ private boolean closed; private FieldInfos fieldInfos; + private final SegmentCodecsBuilder segmentCodecsBuilder; private final BufferedDeletesStream bufferedDeletesStream; private final IndexWriter.FlushControl flushControl; @@ -283,7 +285,7 @@ this.fieldInfos = fieldInfos; this.bufferedDeletesStream = bufferedDeletesStream; flushControl = writer.flushControl; - + segmentCodecsBuilder = SegmentCodecsBuilder.create(writer.codecs); consumer = config.getIndexingChain().getChain(this); this.config = config; } @@ -439,6 +441,7 @@ waitQueue.reset(); segment = null; fieldInfos = new FieldInfos(fieldInfos); + segmentCodecsBuilder.clear(); numDocs = 0; nextDocID = 0; bufferIsFull = false; @@ -542,7 +545,7 @@ final SegmentWriteState flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos, numDocs, writer.getConfig().getTermIndexInterval(), - SegmentCodecs.build(fieldInfos, writer.codecs), + segmentCodecsBuilder.build(), pendingDeletes); // Apply delete-by-docID now (delete-byDocID only // happens when an exception is hit processing that @@ -750,7 +753,7 @@ // work final DocWriter perDoc; try { - perDoc = state.consumer.processDocument(fieldInfos); + perDoc = state.consumer.processDocument(fieldInfos, segmentCodecsBuilder); } finally { docState.clear(); } Index: lucene/src/java/org/apache/lucene/index/FieldInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInfo.java (revision 1084539) +++ lucene/src/java/org/apache/lucene/index/FieldInfo.java (working copy) @@ -19,6 +19,7 @@ /** @lucene.experimental */ public final class FieldInfo { + public static final int UNASSIGNED_CODEC_ID = -1; public String name; public boolean isIndexed; public int number; @@ -32,7 +33,7 @@ public boolean omitTermFreqAndPositions; public boolean storePayloads; // whether this field stores payloads together with term positions - private int codecId = -1; // set inside SegmentCodecs#build() during segment flush - this is used to identify the codec used to write this field + private int codecId = UNASSIGNED_CODEC_ID; // set inside SegmentCodecs#build() during segment flush - this is used to identify the codec used to write this field FieldInfo(String na, boolean tk, int nu, boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, @@ -58,7 +59,7 @@ } public void setCodecId(int codecId) { - assert this.codecId == -1 : "CodecId can only be set once."; + assert this.codecId == UNASSIGNED_CODEC_ID : "CodecId can only be set once."; this.codecId = codecId; } Index: lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java =================================================================== --- lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (revision 1084539) +++ lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (working copy) @@ -67,6 +67,7 @@ @Override public TermsConsumer addField(FieldInfo field) throws IOException { + assert field.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID; final FieldsConsumer fields = consumers.get(field.getCodecId()); return fields.addField(field); } @@ -106,6 +107,7 @@ for (FieldInfo fi : fieldInfos) { if (fi.isIndexed) { // TODO this does not work for non-indexed fields fields.add(fi.name); + assert fi.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID; Codec codec = segmentCodecs.codecs[fi.getCodecId()]; if (!producers.containsKey(codec)) { producers.put(codec, codec.fieldsProducer(new SegmentReadState(dir, Index: lucene/src/java/org/apache/lucene/index/SegmentCodecs.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentCodecs.java (revision 1084539) +++ lucene/src/java/org/apache/lucene/index/SegmentCodecs.java (working copy) @@ -64,36 +64,19 @@ * internal structure to map codecs to fields - don't modify this from outside * of this class! */ - Codec[] codecs; + final Codec[] codecs; final CodecProvider provider; private final Codec codec = new PerFieldCodecWrapper(this); - + + SegmentCodecs(CodecProvider provider, IndexInput input) throws IOException { + this(provider, read(input, provider)); + } + SegmentCodecs(CodecProvider provider, Codec... codecs) { this.provider = provider; this.codecs = codecs; } - static SegmentCodecs build(FieldInfos infos, CodecProvider provider) { - final Map codecRegistry = new IdentityHashMap(); - final ArrayList codecs = new ArrayList(); - - for (FieldInfo fi : infos) { - if (fi.isIndexed) { - final Codec fieldCodec = provider.lookup(provider - .getFieldCodec(fi.name)); - Integer ord = codecRegistry.get(fieldCodec); - if (ord == null) { - ord = Integer.valueOf(codecs.size()); - codecRegistry.put(fieldCodec, ord); - codecs.add(fieldCodec); - } - fi.setCodecId(ord.intValue()); - } - } - return new SegmentCodecs(provider, codecs.toArray(Codec.EMPTY)); - - } - Codec codec() { return codec; } @@ -105,7 +88,7 @@ } } - void read(IndexInput in) throws IOException { + private static Codec[] read(IndexInput in, CodecProvider provider) throws IOException { final int size = in.readVInt(); final ArrayList list = new ArrayList(); for (int i = 0; i < size; i++) { @@ -113,7 +96,7 @@ final Codec lookup = provider.lookup(codecName); list.add(i, lookup); } - codecs = list.toArray(Codec.EMPTY); + return list.toArray(Codec.EMPTY); } void files(Directory dir, SegmentInfo info, Set files) @@ -129,4 +112,55 @@ public String toString() { return "SegmentCodecs [codecs=" + Arrays.toString(codecs) + ", provider=" + provider + "]"; } + + static class SegmentCodecsBuilder { + private final Map codecRegistry = new IdentityHashMap(); + private final ArrayList codecs = new ArrayList(); + private final CodecProvider provider; + + private SegmentCodecsBuilder(CodecProvider provider) { + this.provider = provider; + } + + public static SegmentCodecsBuilder create(CodecProvider provider) { + return new SegmentCodecsBuilder(provider); + } + + public SegmentCodecsBuilder tryAddAndSet(FieldInfo fi) { + if (fi.isIndexed) { + synchronized (this) { + if (fi.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) { + final Codec fieldCodec = provider.lookup(provider + .getFieldCodec(fi.name)); + Integer ord = codecRegistry.get(fieldCodec); + if (ord == null) { + ord = Integer.valueOf(codecs.size()); + codecRegistry.put(fieldCodec, ord); + codecs.add(fieldCodec); + } + fi.setCodecId(ord.intValue()); + } + } + } + return this; + } + + public synchronized SegmentCodecsBuilder addAll(FieldInfos infos) { + for (FieldInfo fieldInfo : infos) { + tryAddAndSet(fieldInfo); + } + return this; + } + + public synchronized SegmentCodecs build() { + return new SegmentCodecs(provider, codecs.toArray(Codec.EMPTY)); + } + + public synchronized SegmentCodecsBuilder clear() { + codecRegistry.clear(); + codecs.clear(); + return this; + } + + } } \ No newline at end of file Index: lucene/src/java/org/apache/lucene/index/SegmentInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfo.java (revision 1084539) +++ lucene/src/java/org/apache/lucene/index/SegmentInfo.java (working copy) @@ -209,13 +209,12 @@ hasProx = input.readByte() == YES; // System.out.println(Thread.currentThread().getName() + ": si.read hasProx=" + hasProx + " seg=" + name); - segmentCodecs = new SegmentCodecs(codecs); if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { - segmentCodecs.read(input); + segmentCodecs = new SegmentCodecs(codecs, input); } else { // codec ID on FieldInfo is 0 so it will simply use the first codec available // TODO what todo if preflex is not available in the provider? register it or fail? - segmentCodecs.codecs = new Codec[] { codecs.lookup("PreFlex")}; + segmentCodecs = new SegmentCodecs(codecs, new Codec[] { codecs.lookup("PreFlex")}); } diagnostics = input.readStringStringMap(); Index: lucene/src/java/org/apache/lucene/index/SegmentMerger.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentMerger.java (revision 1084539) +++ lucene/src/java/org/apache/lucene/index/SegmentMerger.java (working copy) @@ -26,6 +26,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader.FieldOption; import org.apache.lucene.index.MergePolicy.MergeAbortedException; +import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.FieldsConsumer; @@ -221,7 +222,7 @@ fieldInfos.add(reader.getFieldNames(FieldOption.UNINDEXED), false); } } - final SegmentCodecs codecInfo = SegmentCodecs.build(fieldInfos, this.codecs); + final SegmentCodecs codecInfo = SegmentCodecsBuilder.create(codecs).addAll(fieldInfos).build(); fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION); int docCount = 0; Index: lucene/src/test/org/apache/lucene/index/TestCodecs.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestCodecs.java (revision 1084539) +++ lucene/src/test/org/apache/lucene/index/TestCodecs.java (working copy) @@ -25,6 +25,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.Field.Store; +import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; @@ -240,7 +241,7 @@ final Directory dir = newDirectory(); FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone(); this.write(fieldInfos, dir, fields, true); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(clonedFieldInfos, CodecProvider.getDefault()), clonedFieldInfos.hasVectors(), clonedFieldInfos); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecsBuilder.create(CodecProvider.getDefault()).addAll(clonedFieldInfos).build(), clonedFieldInfos.hasVectors(), clonedFieldInfos); si.setHasProx(false); final FieldsProducer reader = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, 64, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); @@ -293,7 +294,7 @@ FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone(); this.write(fieldInfos, dir, fields, false); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecs.build(clonedFieldInfos, CodecProvider.getDefault()), clonedFieldInfos.hasVectors(), clonedFieldInfos); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, true, SegmentCodecsBuilder.create(CodecProvider.getDefault()).addAll(clonedFieldInfos).build(), clonedFieldInfos.hasVectors(), clonedFieldInfos); if (VERBOSE) { System.out.println("TEST: now read postings"); @@ -442,7 +443,7 @@ for(int iter=0;iter