Index: lucene/src/test/org/apache/lucene/index/TestDoc.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDoc.java (revision 1049149) +++ lucene/src/test/org/apache/lucene/index/TestDoc.java (working copy) @@ -192,7 +192,7 @@ SegmentReader r1 = SegmentReader.get(true, si1, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); SegmentReader r2 = SegmentReader.get(true, si2, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); - SegmentMerger merger = new SegmentMerger(si1.dir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, merged, null, CodecProvider.getDefault(), null); + SegmentMerger merger = new SegmentMerger(si1.dir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, merged, null, CodecProvider.getDefault(), null, new FieldInfos()); merger.add(r1); merger.add(r2); Index: lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (revision 1049149) +++ lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (working copy) @@ -73,7 +73,7 @@ } public void testMerge() throws IOException { - SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, CodecProvider.getDefault(), null); + SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, CodecProvider.getDefault(), null, new FieldInfos()); merger.add(reader1); merger.add(reader2); int docsMerged = merger.merge(); Index: lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 1049149) +++ lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy) @@ -293,6 +293,9 @@ public void testIndexOldIndex() throws IOException { for(int i=0;i readers = new ArrayList(); - private FieldInfos fieldInfos; + private final FieldInfos fieldInfos; private int mergedDocs; @@ -79,10 +79,11 @@ private PayloadProcessorProvider payloadProcessorProvider; - SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, CodecProvider codecs, PayloadProcessorProvider payloadProcessorProvider) { + SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, CodecProvider codecs, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos) { this.payloadProcessorProvider = payloadProcessorProvider; directory = dir; this.codecs = codecs; + this.fieldInfos = fieldInfos; segment = name; if (merge != null) { checkAbort = new CheckAbort(merge, directory); @@ -216,7 +217,12 @@ private SegmentReader[] matchingSegmentReaders; private int[] rawDocLengths; private int[] rawDocLengths2; + private int matchedCount; + public int getMatchedSubReaderCount() { + return matchedCount; + } + private void setMatchingSegmentReaders() { // If the i'th reader is a SegmentReader and has // identical fieldName -> number mapping, then this @@ -240,6 +246,7 @@ } if (same) { matchingSegmentReaders[i] = segmentReader; + matchedCount++; } } } @@ -257,28 +264,13 @@ */ private final int mergeFields() throws CorruptIndexException, IOException { - if (!mergeDocStores) { - // When we are not merging by doc stores, their field - // name -> number mapping are the same. So, we start - // with the fieldInfos of the last segment in this - // case, to keep that numbering. - final SegmentReader sr = (SegmentReader) readers.get(readers.size()-1); - fieldInfos = (FieldInfos) sr.core.fieldInfos.clone(); - } else { - fieldInfos = new FieldInfos();// merge field names - } - for (IndexReader reader : readers) { if (reader instanceof SegmentReader) { SegmentReader segmentReader = (SegmentReader) reader; FieldInfos readerFieldInfos = segmentReader.fieldInfos(); int numReaderFieldInfos = readerFieldInfos.size(); for (int j = 0; j < numReaderFieldInfos; j++) { - FieldInfo fi = readerFieldInfos.fieldInfo(j); - fieldInfos.add(fi.name, fi.isIndexed, fi.storeTermVector, - fi.storePositionWithTermVector, fi.storeOffsetWithTermVector, - !reader.hasNorms(fi.name), fi.storePayloads, - fi.omitTermFreqAndPositions); + fieldInfos.add(readerFieldInfos.fieldInfo(j)); } } else { addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, false); Index: lucene/src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (revision 1049149) +++ lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -129,8 +129,6 @@ boolean bufferIsFull; // True when it's time to write segment private boolean aborting; // True if an abort is pending - private DocFieldProcessor docFieldProcessor; - PrintStream infoStream; int maxFieldLength = IndexWriterConfig.UNLIMITED_FIELD_LENGTH; Similarity similarity; @@ -299,9 +297,6 @@ flushControl = writer.flushControl; consumer = indexingChain.getChain(this); - if (consumer instanceof DocFieldProcessor) { - docFieldProcessor = (DocFieldProcessor) consumer; - } } // Buffer a specific docID for deletion. Currently only @@ -359,13 +354,6 @@ return fieldInfos; } - /** Returns true if any of the fields in the current - * buffered docs have omitTermFreqAndPositions==false */ - boolean hasProx() { - return (docFieldProcessor != null) ? fieldInfos.hasProx() - : true; - } - /** If non-null, various details of indexing are printed * here. */ synchronized void setInfoStream(PrintStream infoStream) { @@ -720,7 +708,7 @@ docStoreSegment, numDocsInRAM, numDocsInStore, writer.getConfig().getTermIndexInterval(), SegmentCodecs.build(fieldInfos, writer.codecs)); - newSegment = new SegmentInfo(segment, numDocsInRAM, directory, false, -1, null, false, hasProx(), flushState.segmentCodecs, false); + newSegment = new SegmentInfo(segment, numDocsInRAM, directory, false, -1, null, false, fieldInfos.hasProx(), flushState.segmentCodecs, false); if (!closeDocStore || docStoreOffset != 0) { newSegment.setDocStoreSegment(docStoreSegment); Index: lucene/src/java/org/apache/lucene/index/SegmentInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfos.java (revision 1049149) +++ lucene/src/java/org/apache/lucene/index/SegmentInfos.java (working copy) @@ -74,6 +74,8 @@ private CodecProvider codecs; + private int format; + /** * If non-null, information about loading segments_N files * will be printed here. @see #setInfoStream. @@ -88,6 +90,14 @@ this.codecs = codecs; } + public void setFormat(int format) { + this.format = format; + } + + public int getFormat() { + return format; + } + public final SegmentInfo info(int i) { return get(i); } Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 1049149) +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -17,36 +17,37 @@ * limitations under the License. */ +import java.io.Closeable; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.search.Query; +import org.apache.lucene.store.AlreadyClosedException; +import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.store.AlreadyClosedException; -import org.apache.lucene.store.BufferedIndexInput; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.Constants; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.ThreadInterruptedException; -import org.apache.lucene.util.Bits; -import java.io.IOException; -import java.io.Closeable; -import java.io.PrintStream; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.List; -import java.util.Collection; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Set; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.Iterator; -import java.util.Map; -import java.util.Date; - /** An IndexWriter creates and maintains an index. @@ -810,20 +811,41 @@ } } - private FieldInfos getCurrentFieldInfos() throws IOException { - final FieldInfos fieldInfos; - if (segmentInfos.size() > 0) { - SegmentInfo info = segmentInfos.info(segmentInfos.size()-1); - Directory cfsDir; + private FieldInfos getFieldInfos(SegmentInfo info) throws IOException { + Directory cfsDir = null; + try { if (info.getUseCompoundFile()) { cfsDir = new CompoundFileReader(directory, IndexFileNames.segmentFileName(info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION)); } else { cfsDir = directory; } - fieldInfos = new FieldInfos(cfsDir, IndexFileNames.segmentFileName(info.name, "", IndexFileNames.FIELD_INFOS_EXTENSION)); - if (info.getUseCompoundFile()) { + return new FieldInfos(cfsDir, IndexFileNames.segmentFileName(info.name, "", IndexFileNames.FIELD_INFOS_EXTENSION)); + } finally { + if (info.getUseCompoundFile() && cfsDir != null) { cfsDir.close(); } + } + } + + private FieldInfos getCurrentFieldInfos() throws IOException { + final FieldInfos fieldInfos; + if (segmentInfos.size() > 0) { + if (segmentInfos.getFormat() > DefaultSegmentInfosWriter.FORMAT_4_0) { + // Pre-4.0 index. In this case we sweep all + // segments, merging their FieldInfos: + fieldInfos = new FieldInfos(); + for(SegmentInfo info : segmentInfos) { + final FieldInfos segFieldInfos = getFieldInfos(info); + final int fieldCount = segFieldInfos.size(); + for(int fieldNumber=0;fieldNumber DefaultSegmentInfosWriter.FORMAT_MINIMUM)