Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1083701) +++ lucene/CHANGES.txt (working copy) @@ -162,6 +162,13 @@ say "in order". If this is a problem then you can use either of the LogMergePolicy impls, and call setRequireContiguousMerge(true). (Mike McCandless) + +* LUCENE-2881: FieldInfos is now tracked per segment. Before it was tracked + per IndexWriter session, which resulted in FieldInfos that had the FieldInfo + properties from all previous segments combined. Field numbers are now tracked + globally across IndexWriter sessions and persistend into a _X.fnx file on + successful commit The corresponding file format changes are backwards- + compatible. (Michael Busch, Simon Willnauer) API Changes Index: lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/DocConsumerPerThread.java (working copy) @@ -27,7 +27,8 @@ * DocumentsWriter.DocWriter and return it. * DocumentsWriter then calls finish() on this object * when it's its turn. */ - abstract DocumentsWriter.DocWriter processDocument() throws IOException; + abstract DocumentsWriter.DocWriter processDocument(FieldInfos fieldInfos) throws IOException; + abstract void doAfterFlush(); abstract void abort(); } Index: lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/DocFieldConsumer.java (working copy) @@ -22,9 +22,6 @@ import java.util.Map; abstract class DocFieldConsumer { - - FieldInfos fieldInfos; - /** Called when DocumentsWriter decides to create a new * segment */ abstract void flush(Map> threadsAndFields, SegmentWriteState state) throws IOException; @@ -39,8 +36,4 @@ * The consumer should free RAM, if possible, returning * true if any RAM was in fact freed. */ abstract boolean freeRAM(); - - void setFieldInfos(FieldInfos fieldInfos) { - this.fieldInfos = fieldInfos; } -} Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (working copy) @@ -34,16 +34,13 @@ final class DocFieldProcessor extends DocConsumer { final DocumentsWriter docWriter; - final FieldInfos fieldInfos; final DocFieldConsumer consumer; final StoredFieldsWriter fieldsWriter; public DocFieldProcessor(DocumentsWriter docWriter, DocFieldConsumer consumer) { this.docWriter = docWriter; this.consumer = consumer; - fieldInfos = docWriter.getFieldInfos(); - consumer.setFieldInfos(fieldInfos); - fieldsWriter = new StoredFieldsWriter(docWriter, fieldInfos); + fieldsWriter = new StoredFieldsWriter(docWriter); } @Override @@ -53,7 +50,6 @@ for ( DocConsumerPerThread thread : threads) { DocFieldProcessorPerThread perThread = (DocFieldProcessorPerThread) thread; childThreadsAndFields.put(perThread.consumer, perThread.fields()); - perThread.trimFields(state); } fieldsWriter.flush(state); consumer.flush(childThreadsAndFields, state); @@ -63,7 +59,7 @@ // FreqProxTermsWriter does this with // FieldInfo.storePayload. final String fileName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.FIELD_INFOS_EXTENSION); - fieldInfos.write(state.directory, fileName); + state.fieldInfos.write(state.directory, fileName); } @Override Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/DocFieldProcessorPerThread.java (working copy) @@ -41,14 +41,13 @@ float docBoost; int fieldGen; final DocFieldProcessor docFieldProcessor; - final FieldInfos fieldInfos; final DocFieldConsumerPerThread consumer; // Holds all fields seen in current doc DocFieldProcessorPerField[] fields = new DocFieldProcessorPerField[1]; int fieldCount; - // Hash table for all fields ever seen + // Hash table for all fields seen in current segment DocFieldProcessorPerField[] fieldHash = new DocFieldProcessorPerField[2]; int hashMask = 1; int totalFieldCount; @@ -60,7 +59,6 @@ public DocFieldProcessorPerThread(DocumentsWriterThreadState threadState, DocFieldProcessor docFieldProcessor) throws IOException { this.docState = threadState.docState; this.docFieldProcessor = docFieldProcessor; - this.fieldInfos = docFieldProcessor.fieldInfos; this.consumer = docFieldProcessor.consumer.addThread(this); fieldsWriter = docFieldProcessor.fieldsWriter.addThread(docState); } @@ -75,6 +73,7 @@ field = next; } } + doAfterFlush(); fieldsWriter.abort(); consumer.abort(); } @@ -92,45 +91,15 @@ return fields; } - /** If there are fields we've seen but did not see again - * in the last run, then free them up. */ - - void trimFields(SegmentWriteState state) { - - for(int i=0;i threads = new HashSet(); for (DocumentsWriterThreadState threadState : threadStates) { @@ -749,7 +750,7 @@ // work final DocWriter perDoc; try { - perDoc = state.consumer.processDocument(); + perDoc = state.consumer.processDocument(fieldInfos); } finally { docState.clear(); } Index: lucene/src/java/org/apache/lucene/index/FieldInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInfo.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/FieldInfo.java (working copy) @@ -32,7 +32,7 @@ public boolean omitTermFreqAndPositions; public boolean storePayloads; // whether this field stores payloads together with term positions - int codecId = 0; // set inside SegmentCodecs#build() during segment flush - this is used to identify the codec used to write this field + private int codecId = -1; // set inside SegmentCodecs#build() during segment flush - this is used to identify the codec used to write this field FieldInfo(String na, boolean tk, int nu, boolean storeTermVector, boolean storePositionWithTermVector, boolean storeOffsetWithTermVector, @@ -57,10 +57,21 @@ } } + public void setCodecId(int codecId) { + assert this.codecId == -1 : "CodecId can only be set once."; + this.codecId = codecId; + } + + public int getCodecId() { + return codecId; + } + @Override public Object clone() { - return new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, + FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions); + clone.codecId = this.codecId; + return clone; } void update(boolean isIndexed, boolean storeTermVector, boolean storePositionWithTermVector, Index: lucene/src/java/org/apache/lucene/index/FieldInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInfos.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/FieldInfos.java (working copy) @@ -17,16 +17,23 @@ * limitations under the License. */ -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Fieldable; +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.Map.Entry; + import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.StringHelper; -import java.io.IOException; -import java.util.*; - /** Access to the Fieldable Info file that describes document fields and whether or * not they are indexed. Each segment has a separate Fieldable Info file. Objects * of this class are thread-safe for multiple readers, but only one thread can @@ -34,8 +41,159 @@ * accessing this object. * @lucene.experimental */ -public final class FieldInfos { +public final class FieldInfos implements Iterable { + static final class FieldNumberBiMap { + + final static String CODEC_NAME = "GLOBAL_FIELD_MAP"; + + // Initial format + private static final int VERSION_START = 0; + private static final int VERSION_CURRENT = VERSION_START; + + private final Map numberToName; + private final Map nameToNumber; + private int lowestUnassignedFieldNumber = -1; + private long lastVersion = 0; + private long version = 0; + + FieldNumberBiMap() { + this.nameToNumber = new HashMap(); + this.numberToName = new HashMap(); + } + + /** + * Returns the global field number for the given field name. If the name + * does not exist yet it tries to add it with the given preferred field + * number assigned if possible otherwise the first unassigned field number + * is used as the field number. + */ + synchronized int addOrGet(String fieldName, int preferredFieldNumber) { + Integer fieldNumber = nameToNumber.get(fieldName); + if (fieldNumber == null) { + final Integer preferredBoxed = Integer.valueOf(preferredFieldNumber); + + if (preferredFieldNumber != -1 && !numberToName.containsKey(preferredBoxed)) { + // cool - we can use this number globally + fieldNumber = preferredBoxed; + } else { + // find a new FieldNumber + while (numberToName.containsKey(++lowestUnassignedFieldNumber)) { + // might not be up to date - lets do the work once needed + } + fieldNumber = lowestUnassignedFieldNumber; + } + + version++; + numberToName.put(fieldNumber, fieldName); + nameToNumber.put(fieldName, fieldNumber); + + } + + return fieldNumber.intValue(); + } + + /** + * Sets the given field number and name if not yet set. + */ + synchronized void setIfNotSet(int fieldNumber, String fieldName) { + final Integer boxedFieldNumber = Integer.valueOf(fieldNumber); + if (!numberToName.containsKey(boxedFieldNumber) + && !nameToNumber.containsKey(fieldName)) { + version++; + numberToName.put(boxedFieldNumber, fieldName); + nameToNumber.put(fieldName, boxedFieldNumber); + } else { + assert containsConsistent(boxedFieldNumber, fieldName); + } + } + + /** + * Writes this {@link FieldNumberBiMap} to the given output and returns its + * version. + */ + public synchronized long write(IndexOutput output) throws IOException{ + Set> entrySet = nameToNumber.entrySet(); + CodecUtil.writeHeader(output, CODEC_NAME, VERSION_CURRENT); + output.writeVInt(entrySet.size()); + for (Entry entry : entrySet) { + output.writeVInt(entry.getValue().intValue()); + output.writeString(entry.getKey()); + } + return version; + } + + /** + * Reads the {@link FieldNumberBiMap} from the given input and resets the + * version to 0. + */ + public synchronized void read(IndexInput input) throws IOException{ + CodecUtil.checkHeader(input, CODEC_NAME, + VERSION_START, + VERSION_CURRENT); + final int size = input.readVInt(); + for (int i = 0; i < size; i++) { + final int num = input.readVInt(); + final String name = input.readString(); + setIfNotSet(num, name); + } + version = lastVersion = 0; + } + + /** + * Returns a new {@link FieldInfos} instance with this as the global field + * map + * + * @return a new {@link FieldInfos} instance with this as the global field + * map + */ + public FieldInfos newFieldInfos() { + return new FieldInfos(this); + } + + /** + * Returns true iff the last committed version differs from the + * current version, otherwise false + * + * @return true iff the last committed version differs from the + * current version, otherwise false + */ + public synchronized boolean isDirty() { + return lastVersion != version; + } + + /** + * commits the given version if the given version is greater than the previous committed version + * + * @param version + * the version to commit + * @return true iff the version was successfully committed otherwise false + * @see #write(IndexOutput) + */ + public synchronized boolean commitLastVersion(long version) { + if (version > lastVersion) { + lastVersion = version; + return true; + } + return false; + } + + // just for testing + Set> entries() { + return new HashSet>(nameToNumber.entrySet()); + } + + // used by assert + boolean containsConsistent(Integer number, String name) { + return name.equals(numberToName.get(number)) + && number.equals(nameToNumber.get(name)); + } + } + + private final SortedMap byNumber = new TreeMap(); + private final HashMap byName = new HashMap(); + private final FieldNumberBiMap globalFieldNumbers; + // First used in 2.9; prior to 2.9 there was no format header public static final int FORMAT_START = -2; public static final int FORMAT_PER_FIELD_CODEC = -3; @@ -52,14 +210,21 @@ static final byte OMIT_NORMS = 0x10; static final byte STORE_PAYLOADS = 0x20; static final byte OMIT_TERM_FREQ_AND_POSITIONS = 0x40; - - private final ArrayList byNumber = new ArrayList(); - private final HashMap byName = new HashMap(); + private int format; public FieldInfos() { + this(new FieldNumberBiMap()); } + + FieldInfos(FieldInfos other) { + this(other.globalFieldNumbers); + } + FieldInfos(FieldNumberBiMap globalFieldNumbers) { + this.globalFieldNumbers = globalFieldNumbers; + } + /** * Construct a FieldInfos object using the directory and the name of the file * IndexInput @@ -68,6 +233,7 @@ * @throws IOException */ public FieldInfos(Directory d, String name) throws IOException { + this(new FieldNumberBiMap()); IndexInput input = d.openInput(name); try { read(input, name); @@ -75,36 +241,45 @@ input.close(); } } + + /** + * adds the given field to this FieldInfos name / number mapping. The given FI + * must be present in the global field number mapping before this method it + * called + */ + private void putInternal(FieldInfo fi) { + assert !byNumber.containsKey(fi.number); + assert !byName.containsKey(fi.name); + assert globalFieldNumbers.containsConsistent(Integer.valueOf(fi.number), fi.name); + byNumber.put(fi.number, fi); + byName.put(fi.name, fi); + } + + private int nextFieldNumber(String name, int preferredFieldNumber) { + // get a global number for this field + final int fieldNumber = globalFieldNumbers.addOrGet(name, + preferredFieldNumber); + assert byNumber.get(fieldNumber) == null : "field number " + fieldNumber + + " already taken"; + return fieldNumber; + } /** * Returns a deep clone of this FieldInfos instance. */ @Override synchronized public Object clone() { - FieldInfos fis = new FieldInfos(); - final int numField = byNumber.size(); - for(int i=0;i fields = doc.getFields(); - for (Fieldable field : fields) { - add(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(), - field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions()); - } - } - /** Returns true if any fields do not omitTermFreqAndPositions */ public boolean hasProx() { - final int numFields = byNumber.size(); - for(int i=0;i= 0) ? byNumber.get(fieldNumber) : null; } + public Iterator iterator() { + return byNumber.values().iterator(); + } + public int size() { + assert byNumber.size() == byName.size(); return byNumber.size(); } public boolean hasVectors() { - for (int i = 0; i < size(); i++) { - if (fieldInfo(i).storeTermVector) { + for (FieldInfo fi : this) { + if (fi.storeTermVector) { return true; } } @@ -287,8 +479,8 @@ } public boolean hasNorms() { - for (int i = 0; i < size(); i++) { - if (!fieldInfo(i).omitNorms) { + for (FieldInfo fi : this) { + if (!fi.omitNorms) { return true; } } @@ -307,8 +499,7 @@ public void write(IndexOutput output) throws IOException { output.writeVInt(FORMAT_CURRENT); output.writeVInt(size()); - for (int i = 0; i < size(); i++) { - FieldInfo fi = fieldInfo(i); + for (FieldInfo fi : this) { byte bits = 0x0; if (fi.isIndexed) bits |= IS_INDEXED; if (fi.storeTermVector) bits |= STORE_TERMVECTOR; @@ -318,7 +509,8 @@ if (fi.storePayloads) bits |= STORE_PAYLOADS; if (fi.omitTermFreqAndPositions) bits |= OMIT_TERM_FREQ_AND_POSITIONS; output.writeString(fi.name); - output.writeInt(fi.codecId); + output.writeInt(fi.number); + output.writeInt(fi.getCodecId()); output.writeByte(bits); } } @@ -338,6 +530,7 @@ for (int i = 0; i < size; i++) { String name = StringHelper.intern(input.readString()); // if this is a previous format codec 0 will be preflex! + final int fieldNumber = format <= FORMAT_PER_FIELD_CODEC? input.readInt():i; final int codecId = format <= FORMAT_PER_FIELD_CODEC? input.readInt():0; byte bits = input.readByte(); boolean isIndexed = (bits & IS_INDEXED) != 0; @@ -347,8 +540,8 @@ boolean omitNorms = (bits & OMIT_NORMS) != 0; boolean storePayloads = (bits & STORE_PAYLOADS) != 0; boolean omitTermFreqAndPositions = (bits & OMIT_TERM_FREQ_AND_POSITIONS) != 0; - final FieldInfo addInternal = addInternal(name, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions); - addInternal.codecId = codecId; + final FieldInfo addInternal = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, omitTermFreqAndPositions); + addInternal.setCodecId(codecId); } if (input.getFilePointer() != input.length()) { Index: lucene/src/java/org/apache/lucene/index/FieldsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldsWriter.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/FieldsWriter.java (working copy) @@ -45,14 +45,12 @@ // If null - we were supplied with streams, if notnull - we manage them ourselves private Directory directory; private String segment; - private FieldInfos fieldInfos; private IndexOutput fieldsStream; private IndexOutput indexStream; - FieldsWriter(Directory directory, String segment, FieldInfos fn) throws IOException { + FieldsWriter(Directory directory, String segment) throws IOException { this.directory = directory; this.segment = segment; - fieldInfos = fn; boolean success = false; try { @@ -70,10 +68,9 @@ } } - FieldsWriter(IndexOutput fdx, IndexOutput fdt, FieldInfos fn) { + FieldsWriter(IndexOutput fdx, IndexOutput fdt) { directory = null; segment = null; - fieldInfos = fn; fieldsStream = fdt; indexStream = fdx; } @@ -166,7 +163,7 @@ assert fieldsStream.getFilePointer() == position; } - final void addDocument(Document doc) throws IOException { + final void addDocument(Document doc, FieldInfos fieldInfos) throws IOException { indexStream.writeLong(fieldsStream.getFilePointer()); int storedCount = 0; Index: lucene/src/java/org/apache/lucene/index/IndexFileNames.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexFileNames.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/IndexFileNames.java (working copy) @@ -78,6 +78,10 @@ /** Extension of separate norms */ public static final String SEPARATE_NORMS_EXTENSION = "s"; + + /** Extension of global field numbers */ + public static final String GLOBAL_FIELD_NUM_MAP_EXTENSION = "fnx"; + /** * This array contains all filename extensions used by @@ -98,6 +102,7 @@ GEN_EXTENSION, NORMS_EXTENSION, COMPOUND_FILE_STORE_EXTENSION, + GLOBAL_FIELD_NUM_MAP_EXTENSION, }; public static final String[] STORE_INDEX_EXTENSIONS = new String[] { Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -35,10 +35,10 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor; import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.search.Query; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.BufferedIndexInput; @@ -221,6 +221,7 @@ volatile long pendingCommitChangeCount; final SegmentInfos segmentInfos; // the segments + final FieldNumberBiMap globalFieldNumberMap; private DocumentsWriter docWriter; private IndexFileDeleter deleter; @@ -786,7 +787,10 @@ setRollbackSegmentInfos(segmentInfos); - docWriter = new DocumentsWriter(config, directory, this, getCurrentFieldInfos(), bufferedDeletesStream); + // start with previous field numbers, but new FieldInfos + globalFieldNumberMap = segmentInfos.getOrLoadGlobalFieldNumberMap(directory); + docWriter = new DocumentsWriter(config, directory, this, conf.getIndexingChain(), + globalFieldNumberMap.newFieldInfos(), bufferedDeletesStream); docWriter.setInfoStream(infoStream); // Default deleter (for backwards compatibility) is @@ -825,48 +829,7 @@ } } } - - private FieldInfos getFieldInfos(SegmentInfo info) throws IOException { - Directory cfsDir = null; - try { - if (info.getUseCompoundFile()) { - cfsDir = new CompoundFileReader(directory, IndexFileNames.segmentFileName(info.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION)); - } else { - cfsDir = directory; - } - return new FieldInfos(cfsDir, IndexFileNames.segmentFileName(info.name, "", IndexFileNames.FIELD_INFOS_EXTENSION)); - } finally { - if (info.getUseCompoundFile() && cfsDir != null) { - cfsDir.close(); - } - } - } - - private FieldInfos getCurrentFieldInfos() throws IOException { - final FieldInfos fieldInfos; - if (segmentInfos.size() > 0) { - if (segmentInfos.getFormat() > DefaultSegmentInfosWriter.FORMAT_HAS_VECTORS) { - // Pre-4.0 index. In this case we sweep all - // segments, merging their FieldInfos: - fieldInfos = new FieldInfos(); - for(SegmentInfo info : segmentInfos) { - final FieldInfos segFieldInfos = getFieldInfos(info); - final int fieldCount = segFieldInfos.size(); - for(int fieldNumber=0;fieldNumber BD final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments); @@ -3138,7 +3102,7 @@ SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge, codecs, payloadProcessorProvider, - ((FieldInfos) docWriter.getFieldInfos().clone())); + merge.info.getFieldInfos()); if (infoStream != null) { message("merging " + merge.segString(directory) + " mergeVectors=" + merger.fieldInfos().hasVectors()); @@ -3147,8 +3111,6 @@ merge.readers = new ArrayList(); merge.readerClones = new ArrayList(); - merge.info.setHasVectors(merger.fieldInfos().hasVectors()); - // This is try/finally to make sure merger's readers are // closed: boolean success = false; @@ -3190,6 +3152,8 @@ // Record which codec was used to write the segment merge.info.setSegmentCodecs(merger.getSegmentCodecs()); + // Record if we have merged vectors + merge.info.setHasVectors(merger.fieldInfos().hasVectors()); if (infoStream != null) { message("merge segmentCodecs=" + merger.getSegmentCodecs()); Index: lucene/src/java/org/apache/lucene/index/InvertedDocConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/InvertedDocConsumer.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/InvertedDocConsumer.java (working copy) @@ -35,10 +35,4 @@ /** Attempt to free RAM, returning true if any RAM was * freed */ abstract boolean freeRAM(); - - FieldInfos fieldInfos; - - void setFieldInfos(FieldInfos fieldInfos) { - this.fieldInfos = fieldInfos; } -} Index: lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/InvertedDocEndConsumer.java (working copy) @@ -25,5 +25,4 @@ abstract InvertedDocEndConsumerPerThread addThread(DocInverterPerThread docInverterPerThread); abstract void flush(Map> threadsAndFields, SegmentWriteState state) throws IOException; abstract void abort(); - abstract void setFieldInfos(FieldInfos fieldInfos); } Index: lucene/src/java/org/apache/lucene/index/NormsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/NormsWriter.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/NormsWriter.java (working copy) @@ -36,7 +36,6 @@ final class NormsWriter extends InvertedDocEndConsumer { - private FieldInfos fieldInfos; @Override public InvertedDocEndConsumerPerThread addThread(DocInverterPerThread docInverterPerThread) { return new NormsWriterPerThread(docInverterPerThread, this); @@ -48,11 +47,6 @@ // We only write the _X.nrm file at flush void files(Collection files) {} - @Override - void setFieldInfos(FieldInfos fieldInfos) { - this.fieldInfos = fieldInfos; - } - /** Produce _X.nrm if any document had a field with norms * not disabled */ @Override @@ -60,7 +54,7 @@ final Map> byField = new HashMap>(); - if (!fieldInfos.hasNorms()) { + if (!state.fieldInfos.hasNorms()) { return; } @@ -96,15 +90,10 @@ try { normsOut.writeBytes(SegmentMerger.NORMS_HEADER, 0, SegmentMerger.NORMS_HEADER.length); - final int numField = fieldInfos.size(); - int normCount = 0; - for(int fieldNumber=0;fieldNumber toMerge = byField.get(fieldInfo); + for (FieldInfo fi : state.fieldInfos) { + final List toMerge = byField.get(fi); int upto = 0; if (toMerge != null) { @@ -158,7 +147,7 @@ // Fill final hole with defaultNorm for(;upto producers = new HashMap(); boolean success = false; try { - for (int i = 0; i < fieldCount; i++) { - FieldInfo fi = fieldInfos.fieldInfo(i); + for (FieldInfo fi : fieldInfos) { if (fi.isIndexed) { // TODO this does not work for non-indexed fields fields.add(fi.name); - Codec codec = segmentCodecs.codecs[fi.codecId]; + Codec codec = segmentCodecs.codecs[fi.getCodecId()]; if (!producers.containsKey(codec)) { producers.put(codec, codec.fieldsProducer(new SegmentReadState(dir, - si, fieldInfos, readBufferSize, indexDivisor, ""+fi.codecId))); + si, fieldInfos, readBufferSize, indexDivisor, ""+fi.getCodecId()))); } codecs.put(fi.name, producers.get(codec)); } Index: lucene/src/java/org/apache/lucene/index/SegmentCodecs.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentCodecs.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/SegmentCodecs.java (working copy) @@ -74,22 +74,20 @@ } static SegmentCodecs build(FieldInfos infos, CodecProvider provider) { - final int size = infos.size(); final Map codecRegistry = new IdentityHashMap(); final ArrayList codecs = new ArrayList(); - for (int i = 0; i < size; i++) { - final FieldInfo info = infos.fieldInfo(i); - if (info.isIndexed) { + for (FieldInfo fi : infos) { + if (fi.isIndexed) { final Codec fieldCodec = provider.lookup(provider - .getFieldCodec(info.name)); + .getFieldCodec(fi.name)); Integer ord = codecRegistry.get(fieldCodec); if (ord == null) { ord = Integer.valueOf(codecs.size()); codecRegistry.put(fieldCodec, ord); codecs.add(fieldCodec); } - info.codecId = ord.intValue(); + fi.setCodecId(ord.intValue()); } } return new SegmentCodecs(provider, codecs.toArray(Codec.EMPTY)); Index: lucene/src/java/org/apache/lucene/index/SegmentInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfo.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/SegmentInfo.java (working copy) @@ -17,22 +17,23 @@ * limitations under the License. */ -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.Constants; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import java.io.IOException; -import java.util.Arrays; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; -import java.util.HashSet; -import java.util.HashMap; -import java.util.ArrayList; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.Constants; + /** * Information about a segment such as it's name, directory, and files related * to the segment. @@ -62,7 +63,7 @@ * - NO says this field has no separate norms * >= YES says this field has separate norms with the specified generation */ - private long[] normGen; + private Map normGen; private boolean isCompoundFile; @@ -83,6 +84,8 @@ private boolean hasProx; // True if this segment has any fields with omitTermFreqAndPositions==false private boolean hasVectors; // True if this segment wrote term vectors + + private FieldInfos fieldInfos; private SegmentCodecs segmentCodecs; @@ -100,7 +103,7 @@ private long bufferedDeletesGen; public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, - boolean hasProx, SegmentCodecs segmentCodecs, boolean hasVectors) { + boolean hasProx, SegmentCodecs segmentCodecs, boolean hasVectors, FieldInfos fieldInfos) { this.name = name; this.docCount = docCount; this.dir = dir; @@ -113,13 +116,14 @@ this.hasVectors = hasVectors; delCount = 0; version = Constants.LUCENE_MAIN_VERSION; + this.fieldInfos = fieldInfos; } /** * Copy everything from src SegmentInfo into our instance. */ void reset(SegmentInfo src) { - clearFiles(); + clearFilesCache(); version = src.version; name = src.name; docCount = src.docCount; @@ -130,11 +134,14 @@ docStoreIsCompoundFile = src.docStoreIsCompoundFile; hasVectors = src.hasVectors; hasProx = src.hasProx; + fieldInfos = src.fieldInfos == null ? null : (FieldInfos) src.fieldInfos.clone(); if (src.normGen == null) { normGen = null; } else { - normGen = new long[src.normGen.length]; - System.arraycopy(src.normGen, 0, normGen, 0, src.normGen.length); + normGen = new HashMap(src.normGen.size()); + for (Entry entry : src.normGen.entrySet()) { + normGen.put(entry.getKey(), entry.getValue()); + } } isCompoundFile = src.isCompoundFile; delCount = src.delCount; @@ -184,9 +191,14 @@ if (numNormGen == NO) { normGen = null; } else { - normGen = new long[numNormGen]; + normGen = new HashMap(); for(int j=0;j(diagnostics); if (normGen != null) { - si.normGen = normGen.clone(); + si.normGen = new HashMap(); + for (Entry entry : normGen.entrySet()) { + si.normGen.put(entry.getKey(), entry.getValue()); + } } - si.hasVectors = hasVectors; si.version = version; return si; } @@ -339,7 +377,12 @@ * @param fieldNumber the field index to check */ public boolean hasSeparateNorms(int fieldNumber) { - return normGen != null && normGen[fieldNumber] != NO; + if (normGen == null) { + return false; + } + + Long gen = normGen.get(fieldNumber); + return gen != null && gen.longValue() != NO; } /** @@ -349,7 +392,7 @@ if (normGen == null) { return false; } else { - for (long fieldNormGen : normGen) { + for (long fieldNormGen : normGen.values()) { if (fieldNormGen >= YES) { return true; } @@ -359,10 +402,9 @@ return false; } - void initNormGen(int numFields) { + void initNormGen() { if (normGen == null) { // normGen is null if this segments file hasn't had any norms set against it yet - normGen = new long[numFields]; - Arrays.fill(normGen, NO); + normGen = new HashMap(); } } @@ -373,12 +415,13 @@ * @param fieldIndex field whose norm file will be rewritten */ void advanceNormGen(int fieldIndex) { - if (normGen[fieldIndex] == NO) { - normGen[fieldIndex] = YES; + Long gen = normGen.get(fieldIndex); + if (gen == null || gen.longValue() == NO) { + normGen.put(fieldIndex, new Long(YES)); } else { - normGen[fieldIndex]++; + normGen.put(fieldIndex, gen+1); } - clearFiles(); + clearFilesCache(); } /** @@ -388,7 +431,7 @@ */ public String getNormFileName(int number) { if (hasSeparateNorms(number)) { - return IndexFileNames.fileNameFromGeneration(name, "s" + number, normGen[number]); + return IndexFileNames.fileNameFromGeneration(name, "s" + number, normGen.get(number)); } else { // single file for all norms return IndexFileNames.fileNameFromGeneration(name, IndexFileNames.NORMS_EXTENSION, WITHOUT_GEN); @@ -403,7 +446,7 @@ */ void setUseCompoundFile(boolean isCompoundFile) { this.isCompoundFile = isCompoundFile; - clearFiles(); + clearFilesCache(); } /** @@ -433,7 +476,7 @@ void setDocStoreIsCompoundFile(boolean v) { docStoreIsCompoundFile = v; - clearFiles(); + clearFilesCache(); } public String getDocStoreSegment() { @@ -446,14 +489,14 @@ void setDocStoreOffset(int offset) { docStoreOffset = offset; - clearFiles(); + clearFilesCache(); } void setDocStore(int offset, String segment, boolean isCompoundFile) { docStoreOffset = offset; docStoreSegment = segment; docStoreIsCompoundFile = isCompoundFile; - clearFiles(); + clearFilesCache(); } /** Save this segment's info. */ @@ -474,9 +517,10 @@ if (normGen == null) { output.writeInt(NO); } else { - output.writeInt(normGen.length); - for (long fieldNormGen : normGen) { - output.writeLong(fieldNormGen); + output.writeInt(normGen.size()); + for (Entry entry : normGen.entrySet()) { + output.writeInt(entry.getKey()); + output.writeLong(entry.getValue()); } } @@ -490,7 +534,7 @@ void setHasProx(boolean hasProx) { this.hasProx = hasProx; - clearFiles(); + clearFilesCache(); } public boolean getHasProx() { @@ -572,11 +616,11 @@ } if (normGen != null) { - for (int i = 0; i < normGen.length; i++) { - long gen = normGen[i]; + for (Entry entry : normGen.entrySet()) { + long gen = entry.getValue(); if (gen >= YES) { // Definitely a separate norm file, with generation: - fileSet.add(IndexFileNames.fileNameFromGeneration(name, IndexFileNames.SEPARATE_NORMS_EXTENSION + i, gen)); + fileSet.add(IndexFileNames.fileNameFromGeneration(name, IndexFileNames.SEPARATE_NORMS_EXTENSION + entry.getKey(), gen)); } } } @@ -588,7 +632,7 @@ /* Called whenever any change is made that affects which * files this segment has. */ - private void clearFiles() { + private void clearFilesCache() { files = null; sizeInBytesNoStore = -1; sizeInBytesWithStore = -1; Index: lucene/src/java/org/apache/lucene/index/SegmentInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfos.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/SegmentInfos.java (working copy) @@ -17,26 +17,28 @@ * limitations under the License. */ -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.NoSuchDirectoryException; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.SegmentInfosReader; -import org.apache.lucene.index.codecs.SegmentInfosWriter; -import org.apache.lucene.util.ThreadInterruptedException; - import java.io.FileNotFoundException; import java.io.IOException; import java.io.PrintStream; -import java.util.Vector; import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; -import java.util.HashMap; import java.util.Map; +import java.util.Vector; +import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; +import org.apache.lucene.index.codecs.SegmentInfosReader; +import org.apache.lucene.index.codecs.SegmentInfosWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.NoSuchDirectoryException; +import org.apache.lucene.util.ThreadInterruptedException; + /** * A collection of segmentInfo objects with methods for operating on * those segments in relation to the file system. @@ -64,6 +66,11 @@ * starting with the current time in milliseconds forces to create unique version numbers. */ public long version = System.currentTimeMillis(); + + private long globalFieldMapVersion = 0; // version of the GFNM for the next commit + private long lastGlobalFieldMapVersion = 0; // version of the GFNM file we last successfully read or wrote + private long pendingMapVersion = -1; // version of the GFNM itself that we have last successfully written + // or -1 if we it was not written. This is set during prepareCommit private long generation = 0; // generation of the "segments_N" for the next commit private long lastGeneration = 0; // generation of the "segments_N" file we last successfully read @@ -75,6 +82,8 @@ private CodecProvider codecs; private int format; + + private FieldNumberBiMap globalFieldNumberMap; // this segments global field number map - lazy loaded on demand /** * If non-null, information about loading segments_N files @@ -171,6 +180,10 @@ "", lastGeneration); } + + private String getGlobalFieldNumberName(long version) { + return IndexFileNames.segmentFileName(""+ version, "", IndexFileNames.GLOBAL_FIELD_NUM_MAP_EXTENSION); + } /** * Parse the generation off the segments file name and @@ -261,6 +274,8 @@ return null; } }.run(); + // either we are on 4.0 or we don't have a lastGlobalFieldMapVersion i.e. its still set to 0 + assert DefaultSegmentInfosWriter.FORMAT_4_0 <= format || (DefaultSegmentInfosWriter.FORMAT_4_0 > format && lastGlobalFieldMapVersion == 0); } // Only non-null after prepareCommit has been called and @@ -270,15 +285,24 @@ private void write(Directory directory) throws IOException { String segmentFileName = getNextSegmentFileName(); - + final String globalFieldMapFile; + if (globalFieldNumberMap != null && globalFieldNumberMap.isDirty()) { + globalFieldMapFile = getGlobalFieldNumberName(++globalFieldMapVersion); + pendingMapVersion = writeGlobalFieldMap(globalFieldNumberMap, directory, globalFieldMapFile); + } else { + globalFieldMapFile = null; + } + + // Always advance the generation on write: if (generation == -1) { generation = 1; } else { generation++; } - + IndexOutput segnOutput = null; + boolean success = false; @@ -304,6 +328,16 @@ } catch (Throwable t) { // Suppress so we keep throwing the original exception } + if (globalFieldMapFile != null) { // delete if written here + try { + // Try not to leave global field map in + // the index: + directory.deleteFile(globalFieldMapFile); + } catch (Throwable t) { + // Suppress so we keep throwing the original exception + } + } + pendingMapVersion = -1; } } } @@ -719,6 +753,8 @@ void updateGeneration(SegmentInfos other) { lastGeneration = other.lastGeneration; generation = other.generation; + lastGlobalFieldMapVersion = other.lastGlobalFieldMapVersion; + globalFieldMapVersion = other.globalFieldMapVersion; } final void rollbackCommit(Directory dir) throws IOException { @@ -742,6 +778,16 @@ // in our caller } pendingSegnOutput = null; + if (pendingMapVersion != -1) { + try { + final String fieldMapName = getGlobalFieldNumberName(globalFieldMapVersion--); + dir.deleteFile(fieldMapName); + } catch (Throwable t) { + // Suppress so we keep throwing the original exception + // in our caller + } + pendingMapVersion = -1; + } } } @@ -760,6 +806,40 @@ throw new IllegalStateException("prepareCommit was already called"); write(dir); } + + private final long writeGlobalFieldMap(FieldNumberBiMap map, Directory dir, String name) throws IOException { + final IndexOutput output = dir.createOutput(name); + boolean success = false; + long version; + try { + version = map.write(output); + success = true; + } finally { + try { + output.close(); + } catch (Throwable t) { + // throw orig excp + } + if (!success) { + try { + dir.deleteFile(name); + } catch (Throwable t) { + // throw orig excp + } + } + } + return version; + } + + private void readGlobalFieldMap(FieldNumberBiMap map, Directory dir) throws IOException { + final String name = getGlobalFieldNumberName(lastGlobalFieldMapVersion); + final IndexInput input = dir.openInput(name); + try { + map.read(input); + } finally { + input.close(); + } + } /** Returns all file names referenced by SegmentInfo * instances matching the provided Directory (ie files @@ -769,7 +849,17 @@ public Collection files(Directory dir, boolean includeSegmentsFile) throws IOException { HashSet files = new HashSet(); if (includeSegmentsFile) { - files.add(getCurrentSegmentFileName()); + final String segmentFileName = getCurrentSegmentFileName(); + if (segmentFileName != null) { + /* + * TODO: if lastGen == -1 we get might get null here it seems wrong to + * add null to the files set + */ + files.add(segmentFileName); + } + if (lastGlobalFieldMapVersion > 0) { + files.add(getGlobalFieldNumberName(lastGlobalFieldMapVersion)); + } } final int size = size(); for(int i=0;i 0) { + // if we don't have a global map or this is a SI from a earlier version we just return the empty map; + readGlobalFieldMap(map, dir); + } + if (size() > 0) { + if (format > DefaultSegmentInfosWriter.FORMAT_4_0) { + assert lastGlobalFieldMapVersion == 0; + // build the map up if we open a pre 4.0 index + for (SegmentInfo info : this) { + final FieldInfos segFieldInfos = info.getFieldInfos(); + for (FieldInfo fi : segFieldInfos) { + map.addOrGet(fi.name, fi.number); + } + } + } + } + return globalFieldNumberMap = map; + } + + /** + * Called by {@link SegmentInfosReader} when reading the global field map version + */ + public void setGlobalFieldMapVersion(long version) { + lastGlobalFieldMapVersion = globalFieldMapVersion = version; + } + + public long getGlobalFieldMapVersion() { + return globalFieldMapVersion; + } + + // for testing + long getLastGlobalFieldMapVersion() { + return lastGlobalFieldMapVersion; + } } Index: lucene/src/java/org/apache/lucene/index/SegmentMerger.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentMerger.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/SegmentMerger.java (working copy) @@ -26,16 +26,16 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexReader.FieldOption; import org.apache.lucene.index.MergePolicy.MergeAbortedException; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.MergeState; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.MergeState; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.ReaderUtil; import org.apache.lucene.util.MultiBits; +import org.apache.lucene.util.ReaderUtil; /** * The SegmentMerger class combines two or more Segments, represented by an IndexReader ({@link #add}, @@ -75,8 +75,8 @@ this.payloadProcessorProvider = payloadProcessorProvider; directory = dir; this.codecs = codecs; - this.fieldInfos = fieldInfos; segment = name; + this.fieldInfos = fieldInfos; if (merge != null) { checkAbort = new MergeState.CheckAbort(merge, directory); } else { @@ -180,9 +180,8 @@ SegmentReader segmentReader = (SegmentReader) reader; boolean same = true; FieldInfos segmentFieldInfos = segmentReader.fieldInfos(); - int numFieldInfos = segmentFieldInfos.size(); - for (int j = 0; same && j < numFieldInfos; j++) { - same = fieldInfos.fieldName(j).equals(segmentFieldInfos.fieldName(j)); + for (FieldInfo fi : segmentFieldInfos) { + same = fieldInfos.fieldName(fi.number).equals(fi.name); } if (same) { matchingSegmentReaders[i] = segmentReader; @@ -208,9 +207,8 @@ if (reader instanceof SegmentReader) { SegmentReader segmentReader = (SegmentReader) reader; FieldInfos readerFieldInfos = segmentReader.fieldInfos(); - int numReaderFieldInfos = readerFieldInfos.size(); - for (int j = 0; j < numReaderFieldInfos; j++) { - fieldInfos.add(readerFieldInfos.fieldInfo(j)); + for (FieldInfo fi : readerFieldInfos) { + fieldInfos.add(fi); } } else { addIndexed(reader, fieldInfos, reader.getFieldNames(FieldOption.TERMVECTOR_WITH_POSITION_OFFSET), true, true, true, false, false); @@ -224,13 +222,13 @@ } } final SegmentCodecs codecInfo = SegmentCodecs.build(fieldInfos, this.codecs); - fieldInfos.write(directory, segment + ".fnm"); + fieldInfos.write(directory, segment + "." + IndexFileNames.FIELD_INFOS_EXTENSION); int docCount = 0; setMatchingSegmentReaders(); - final FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, fieldInfos); + final FieldsWriter fieldsWriter = new FieldsWriter(directory, segment); try { int idx = 0; @@ -312,7 +310,7 @@ // NOTE: it's very important to first assign to doc then pass it to // termVectorsWriter.addAllDocVectors; see LUCENE-1282 Document doc = reader.document(j); - fieldsWriter.addDocument(doc); + fieldsWriter.addDocument(doc, fieldInfos); docCount++; checkAbort.work(300); } @@ -339,7 +337,7 @@ // NOTE: it's very important to first assign to doc then pass it to // termVectorsWriter.addAllDocVectors; see LUCENE-1282 Document doc = reader.document(docCount); - fieldsWriter.addDocument(doc); + fieldsWriter.addDocument(doc, fieldInfos); checkAbort.work(300); } } @@ -579,8 +577,7 @@ private void mergeNorms() throws IOException { IndexOutput output = null; try { - for (int i = 0, numFieldInfos = fieldInfos.size(); i < numFieldInfos; i++) { - final FieldInfo fi = fieldInfos.fieldInfo(i); + for (FieldInfo fi : fieldInfos) { if (fi.isIndexed && !fi.omitNorms) { if (output == null) { output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION)); Index: lucene/src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentReader.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -22,23 +22,22 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; - import java.util.List; import java.util.Map; import java.util.Set; - import java.util.concurrent.atomic.AtomicInteger; + import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; +import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BitVector; import org.apache.lucene.util.Bits; -import org.apache.lucene.util.CloseableThreadLocal; -import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CloseableThreadLocal; /** * @lucene.experimental @@ -119,8 +118,8 @@ dir0 = cfsReader; } cfsDir = dir0; - - fieldInfos = new FieldInfos(cfsDir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELD_INFOS_EXTENSION)); + si.loadFieldInfos(cfsDir, false); // prevent opening the CFS to load fieldInfos + fieldInfos = si.getFieldInfos(); this.termsIndexDivisor = termsIndexDivisor; @@ -598,12 +597,12 @@ && (!si.hasDeletions() || this.si.getDelFileName().equals(si.getDelFileName())); boolean normsUpToDate = true; - boolean[] fieldNormsChanged = new boolean[core.fieldInfos.size()]; - final int fieldCount = core.fieldInfos.size(); - for (int i = 0; i < fieldCount; i++) { - if (!this.si.getNormFileName(i).equals(si.getNormFileName(i))) { + Set fieldNormsChanged = new HashSet(); + for (FieldInfo fi : core.fieldInfos) { + int fieldNumber = fi.number; + if (!this.si.getNormFileName(fieldNumber).equals(si.getNormFileName(fieldNumber))) { normsUpToDate = false; - fieldNormsChanged[i] = true; + fieldNormsChanged.add(fieldNumber); } } @@ -659,11 +658,10 @@ clone.norms = new HashMap(); // Clone norms - for (int i = 0; i < fieldNormsChanged.length; i++) { - + for (FieldInfo fi : core.fieldInfos) { // Clone unchanged norms to the cloned reader - if (doClone || !fieldNormsChanged[i]) { - final String curField = core.fieldInfos.fieldInfo(i).name; + if (doClone || !fieldNormsChanged.contains(fi.number)) { + final String curField = fi.name; Norm norm = this.norms.get(curField); if (norm != null) clone.norms.put(curField, (Norm) norm.clone()); @@ -735,7 +733,7 @@ } if (normsDirty) { // re-write norms - si.initNormGen(core.fieldInfos.size()); + si.initNormGen(); for (final Norm norm : norms.values()) { if (norm.dirty) { norm.reWrite(si); @@ -880,8 +878,7 @@ ensureOpen(); Set fieldSet = new HashSet(); - for (int i = 0; i < core.fieldInfos.size(); i++) { - FieldInfo fi = core.fieldInfos.fieldInfo(i); + for (FieldInfo fi : core.fieldInfos) { if (fieldOption == IndexReader.FieldOption.ALL) { fieldSet.add(fi.name); } @@ -954,8 +951,7 @@ private void openNorms(Directory cfsDir, int readBufferSize) throws IOException { long nextNormSeek = SegmentMerger.NORMS_HEADER.length; //skip header (header unused for now) int maxDoc = maxDoc(); - for (int i = 0; i < core.fieldInfos.size(); i++) { - FieldInfo fi = core.fieldInfos.fieldInfo(i); + for (FieldInfo fi : core.fieldInfos) { if (norms.containsKey(fi.name)) { // in case this SegmentReader is being re-opened, we might be able to // reuse some norm instances and skip loading them here Index: lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java (working copy) @@ -27,15 +27,13 @@ FieldsWriter fieldsWriter; final DocumentsWriter docWriter; - final FieldInfos fieldInfos; int lastDocID; PerDoc[] docFreeList = new PerDoc[1]; int freeCount; - public StoredFieldsWriter(DocumentsWriter docWriter, FieldInfos fieldInfos) { + public StoredFieldsWriter(DocumentsWriter docWriter) { this.docWriter = docWriter; - this.fieldInfos = fieldInfos; } public StoredFieldsWriterPerThread addThread(DocumentsWriter.DocState docState) throws IOException { @@ -62,7 +60,7 @@ private synchronized void initFieldsWriter() throws IOException { if (fieldsWriter == null) { - fieldsWriter = new FieldsWriter(docWriter.directory, docWriter.getSegment(), fieldInfos); + fieldsWriter = new FieldsWriter(docWriter.directory, docWriter.getSegment()); lastDocID = 0; } } Index: lucene/src/java/org/apache/lucene/index/StoredFieldsWriterPerThread.java =================================================================== --- lucene/src/java/org/apache/lucene/index/StoredFieldsWriterPerThread.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/StoredFieldsWriterPerThread.java (working copy) @@ -32,7 +32,7 @@ public StoredFieldsWriterPerThread(DocumentsWriter.DocState docState, StoredFieldsWriter storedFieldsWriter) throws IOException { this.storedFieldsWriter = storedFieldsWriter; this.docState = docState; - localFieldsWriter = new FieldsWriter((IndexOutput) null, (IndexOutput) null, storedFieldsWriter.fieldInfos); + localFieldsWriter = new FieldsWriter((IndexOutput) null, (IndexOutput) null); } public void startDocument() { Index: lucene/src/java/org/apache/lucene/index/TermsHash.java =================================================================== --- lucene/src/java/org/apache/lucene/index/TermsHash.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/TermsHash.java (working copy) @@ -57,12 +57,6 @@ } @Override - void setFieldInfos(FieldInfos fieldInfos) { - this.fieldInfos = fieldInfos; - consumer.setFieldInfos(fieldInfos); - } - - @Override public void abort() { consumer.abort(); if (nextTermsHash != null) Index: lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/TermsHashConsumer.java (working copy) @@ -25,10 +25,4 @@ abstract TermsHashConsumerPerThread addThread(TermsHashPerThread perThread); abstract void flush(Map> threadsAndFields, final SegmentWriteState state) throws IOException; abstract void abort(); - - FieldInfos fieldInfos; - - void setFieldInfos(FieldInfos fieldInfos) { - this.fieldInfos = fieldInfos; } -} Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (working copy) @@ -56,7 +56,9 @@ infos.version = input.readLong(); // read version infos.counter = input.readInt(); // read counter - + if (infos.getFormat() <= DefaultSegmentInfosWriter.FORMAT_4_0) { + infos.setGlobalFieldMapVersion(input.readLong()); + } for (int i = input.readInt(); i > 0; i--) { // read segmentInfos SegmentInfo si = new SegmentInfo(directory, format, input, codecs); if (si.getVersion() == null) { Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (working copy) @@ -59,6 +59,7 @@ out.writeInt(FORMAT_CURRENT); // write FORMAT out.writeLong(infos.version); out.writeInt(infos.counter); // write counter + out.writeLong(infos.getGlobalFieldMapVersion()); out.writeInt(infos.size()); // write infos for (SegmentInfo si : infos) { si.write(out); Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (revision 1083701) +++ lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (working copy) @@ -19,14 +19,15 @@ import java.io.IOException; import java.util.Collection; -import java.util.Iterator; -import java.util.TreeMap; +import java.util.Comparator; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; -import java.util.Comparator; +import java.util.TreeMap; -import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.CompoundFileReader; import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldsEnum; @@ -35,7 +36,6 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.CompoundFileReader; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -94,13 +94,11 @@ // so that if an index update removes them we'll still have them freqStream = dir.openInput(info.name + ".frq", readBufferSize); boolean anyProx = false; - final int numFields = fieldInfos.size(); - for(int i=0;i fields = doc.getFields(); + for (Fieldable field : fields) { + fieldInfos.add(field.name(), field.isIndexed(), field.isTermVectorStored(), field.isStorePositionWithTermVector(), + field.isStoreOffsetWithTermVector(), field.getOmitNorms(), false, field.getOmitTermFreqAndPositions()); + } + } } Index: lucene/src/test/org/apache/lucene/index/TestAddIndexes.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (revision 1083701) +++ lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (working copy) @@ -1045,8 +1045,8 @@ IndexWriter w3 = new IndexWriter(dir, conf); w3.addIndexes(readers); w3.close(); - - assertEquals("Only one compound segment should exist", 3, dir.listAll().length); + // we should now see segments_X, segments.gen,_Y.cfs, _Z.fnx + assertEquals("Only one compound segment should exist", 4, dir.listAll().length); } } Index: lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 1083701) +++ lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy) @@ -529,10 +529,9 @@ CompoundFileReader cfsReader = new CompoundFileReader(dir, "_0.cfs"); FieldInfos fieldInfos = new FieldInfos(cfsReader, "_0.fnm"); int contentFieldIndex = -1; - for(int i=0;i filesToDelete = merger.createCompoundFile(merged + ".cfs", info); Index: lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (revision 1083701) +++ lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (working copy) @@ -25,20 +25,20 @@ import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.document.Fieldable; import org.apache.lucene.store.Directory; import org.apache.lucene.util.AttributeSource; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; -import org.apache.lucene.util.BytesRef; public class TestDocumentWriter extends LuceneTestCase { private Directory dir; @@ -98,8 +98,7 @@ // test that the norms are not present in the segment if // omitNorms is true - for (int i = 0; i < reader.core.fieldInfos.size(); i++) { - FieldInfo fi = reader.core.fieldInfos.fieldInfo(i); + for (FieldInfo fi : reader.core.fieldInfos) { if (fi.isIndexed) { assertTrue(fi.omitNorms == !reader.hasNorms(fi.name)); } Index: lucene/src/test/org/apache/lucene/index/TestFieldInfos.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (revision 1083701) +++ lucene/src/test/org/apache/lucene/index/TestFieldInfos.java (working copy) @@ -18,6 +18,7 @@ */ import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; import org.apache.lucene.document.Document; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexOutput; @@ -40,7 +41,7 @@ //Positive test of FieldInfos assertTrue(testDoc != null); FieldInfos fieldInfos = new FieldInfos(); - fieldInfos.add(testDoc); + _TestUtil.add(testDoc, fieldInfos); //Since the complement is stored as well in the fields map assertTrue(fieldInfos.size() == DocHelper.all.size()); //this is all b/c we are using the no-arg constructor Directory dir = newDirectory(); Index: lucene/src/test/org/apache/lucene/index/TestFieldsReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (revision 1083701) +++ lucene/src/test/org/apache/lucene/index/TestFieldsReader.java (working copy) @@ -49,7 +49,7 @@ super.setUp(); fieldInfos = new FieldInfos(); DocHelper.setupDoc(testDoc); - fieldInfos.add(testDoc); + _TestUtil.add(testDoc, fieldInfos); dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newLogMergePolicy()); ((LogMergePolicy) conf.getMergePolicy()).setUseCompoundFile(false); Index: lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java (revision 0) +++ lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java (revision 0) @@ -0,0 +1,537 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map.Entry; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.Field.Index; +import org.apache.lucene.document.Field.Store; +import org.apache.lucene.document.Field.TermVector; +import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; +import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +public class TestGlobalFieldNumbers extends LuceneTestCase { + + public void testGlobalFieldNumberFiles() throws IOException { + for (int i = 0; i < 39; i++) { + Directory dir = newDirectory(); + { + IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer()); + IndexWriter writer = new IndexWriter(dir, config); + Document d = new Document(); + d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED, + TermVector.NO)); + writer.addDocument(d); + Collection files = writer.segmentInfos.files(dir, true); + for (String string : files) { + assertFalse(string.endsWith(".fnx")); + } + writer.commit(); + files = writer.segmentInfos.files(dir, true); + files.remove("1.fnx"); + for (String string : files) { + assertFalse(string.endsWith(".fnx")); + } + + assertFNXFiles(dir, "1.fnx"); + d = new Document(); + d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f3", new byte[] { 1, 2, 3 })); + writer.addDocument(d); + writer.commit(); + files = writer.segmentInfos.files(dir, true); + files.remove("2.fnx"); + for (String string : files) { + assertFalse(string.endsWith(".fnx")); + } + assertFNXFiles(dir, "2.fnx"); + writer.close(); + assertFNXFiles(dir, "2.fnx"); + } + + { + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer())); + Document d = new Document(); + d.add(new Field("f1", "d3 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f2", "d3 second field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f3", new byte[] { 1, 2, 3, 4, 5 })); + writer.addDocument(d); + writer.close(); + Collection files = writer.segmentInfos.files(dir, true); + files.remove("2.fnx"); + for (String string : files) { + assertFalse(string.endsWith(".fnx")); + } + + assertFNXFiles(dir, "2.fnx"); + } + + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer())); + writer.optimize(); + assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); + writer.close(); + assertFNXFiles(dir, "2.fnx"); + + dir.close(); + } + } + + public void testIndexReaderCommit() throws IOException { + for (int i = 0; i < 39; i++) { + Directory dir = newDirectory(); + { + IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer()); + IndexWriter writer = new IndexWriter(dir, config); + Document d = new Document(); + d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED, + TermVector.NO)); + writer.addDocument(d); + writer.commit(); + assertFNXFiles(dir, "1.fnx"); + d = new Document(); + d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f3", new byte[] { 1, 2, 3 })); + writer.addDocument(d); + writer.commit(); + assertFNXFiles(dir, "2.fnx"); + writer.close(); + assertFNXFiles(dir, "2.fnx"); + } + IndexReader reader = IndexReader.open(dir, false); + reader.deleteDocument(0); + reader.commit(); + reader.close(); + // make sure this reader can not modify the field map + assertFNXFiles(dir, "2.fnx"); + + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer())); + writer.optimize(); + assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); + writer.close(); + assertFNXFiles(dir, "2.fnx"); + + dir.close(); + } + } + + public void testGlobalFieldNumberFilesAcrossCommits() throws IOException { + for (int i = 0; i < 39; i++) { + Directory dir = newDirectory(); + { + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer()).setIndexDeletionPolicy( + new KeepAllDeletionPolicy())); + Document d = new Document(); + d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED, + TermVector.NO)); + writer.addDocument(d); + writer.commit(); + assertFNXFiles(dir, "1.fnx"); + d = new Document(); + d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f3", new byte[] { 1, 2, 3 })); + writer.addDocument(d); + writer.commit(); + writer.commit(); + writer.commit(); + assertFNXFiles(dir, "1.fnx", "2.fnx"); + writer.close(); + assertFNXFiles(dir, "1.fnx", "2.fnx"); + } + + { + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer())); + Document d = new Document(); + d.add(new Field("f1", "d3 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f2", "d3 second field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f3", new byte[] { 1, 2, 3, 4, 5 })); + writer.addDocument(d); + writer.close(); + assertFNXFiles(dir, "2.fnx"); + } + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer())); + writer.optimize(); + assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); + writer.close(); + assertFNXFiles(dir, "2.fnx"); + dir.close(); + } + } + + public void testGlobalFieldNumberOnOldCommit() throws IOException { + for (int i = 0; i < 39; i++) { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer()).setIndexDeletionPolicy( + new KeepAllDeletionPolicy())); + Document d = new Document(); + d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f2", "d1 second field", Store.YES, Index.ANALYZED, + TermVector.NO)); + writer.addDocument(d); + writer.commit(); + assertFNXFiles(dir, "1.fnx"); + d = new Document(); + d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f3", new byte[] { 1, 2, 3 })); + writer.addDocument(d); + assertFNXFiles(dir, "1.fnx"); + writer.close(); + assertFNXFiles(dir, "1.fnx", "2.fnx"); + // open first commit + List listCommits = IndexReader.listCommits(dir); + assertEquals(2, listCommits.size()); + writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer()).setIndexDeletionPolicy( + new KeepAllDeletionPolicy()).setIndexCommit(listCommits.get(0))); + + d = new Document(); + d.add(new Field("f1", "d2 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + d.add(new Field("f3", new byte[] { 1, 2, 3 })); + writer.addDocument(d); + writer.commit(); + // now we have 3 files since f3 is not present in the first commit + assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx"); + writer.close(); + assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx"); + + writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer())); + writer.commit(); + listCommits = IndexReader.listCommits(dir); + assertEquals(1, listCommits.size()); + assertFNXFiles(dir, "3.fnx"); + writer.close(); + assertFNXFiles(dir, "3.fnx"); + dir.close(); + } + } + + private final Directory buildRandomIndex(String[] fieldNames, int numDocs, + IndexWriterConfig conf) throws CorruptIndexException, + LockObtainFailedException, IOException { + Directory dir = newDirectory(); + IndexWriter writer = new IndexWriter(dir, conf); + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + final int numFields = 1 + random.nextInt(fieldNames.length); + for (int j = 0; j < numFields; j++) { + doc.add(newField(fieldNames[random.nextInt(fieldNames.length)], + _TestUtil.randomRealisticUnicodeString(random), + Index.toIndex(true, random.nextBoolean(), random.nextBoolean()))); + + } + writer.addDocument(doc); + if (random.nextInt(20) == 0) { + writer.commit(); + } + } + writer.close(); + return dir; + } + + public void testOptimize() throws IOException { + for (int i = 0; i < 2; i++) { + Set fieldNames = new HashSet(); + final int numFields = 2 + random.nextInt(200); + for (int j = 0; j < numFields; j++) { + fieldNames.add("field_" + j); + } + Directory base = buildRandomIndex(fieldNames.toArray(new String[0]), + 20 + random.nextInt(100), + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())); + IndexWriter writer = new IndexWriter(base, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer())); + FieldNumberBiMap globalFieldMap = writer.segmentInfos + .getOrLoadGlobalFieldNumberMap(base); + Set> entries = globalFieldMap.entries(); + writer.optimize(); + writer.commit(); + writer.close(); + Set> afterOptmize = globalFieldMap.entries(); + assertEquals(entries, afterOptmize); + base.close(); + } + } + + public void testAddIndexesStableFieldNumbers() throws IOException { + for (int i = 0; i < 2; i++) { + Set fieldNames = new HashSet(); + final int numFields = 2 + random.nextInt(50); + for (int j = 0; j < numFields; j++) { + fieldNames.add("field_" + j); + } + + Directory base = newDirectory(); + IndexWriter writer = new IndexWriter(base, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer())); + Document doc = new Document(); + for (String string : fieldNames) { + doc.add(newField(string, + _TestUtil.randomRealisticUnicodeString(random), + Index.toIndex(true, random.nextBoolean(), random.nextBoolean()))); + + } + writer.addDocument(doc); + writer.commit(); + FieldNumberBiMap globalFieldMap = writer.segmentInfos + .getOrLoadGlobalFieldNumberMap(base); + final Set> entries = globalFieldMap.entries(); + assertEquals(entries.size(), fieldNames.size()); + for (Entry entry : entries) { + // all fields are in this fieldMap + assertTrue(fieldNames.contains(entry.getKey())); + } + writer.close(); + + int numIndexes = 1 + random.nextInt(10); + for (int j = 0; j < numIndexes; j++) { + Directory toAdd = buildRandomIndex(fieldNames.toArray(new String[0]), + 1 + random.nextInt(50), + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())); + IndexWriter w = new IndexWriter(base, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer())); + if (random.nextBoolean()) { + IndexReader open = IndexReader.open(toAdd); + w.addIndexes(open); + open.close(); + } else { + w.addIndexes(toAdd); + } + + w.close(); + FieldNumberBiMap map = w.segmentInfos + .getOrLoadGlobalFieldNumberMap(toAdd); + assertEquals(entries, map.entries()); + toAdd.close(); + } + IndexWriter w = new IndexWriter(base, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy( + new LogByteSizeMergePolicy())); + w.optimize(); + w.close(); + SegmentInfos sis = new SegmentInfos(); + sis.read(base); + SegmentInfo segmentInfo = sis.get(sis.size() - 1);// last segment must + // have all fields with + // consistent numbers + FieldInfos fieldInfos = segmentInfo.getFieldInfos(); + assertEquals(fieldInfos.size(), entries.size()); + for (Entry entry : entries) { + assertEquals(entry.getValue(), + Integer.valueOf(fieldInfos.fieldNumber(entry.getKey()))); + assertEquals(entry.getKey(), fieldInfos.fieldName(entry.getValue())); + } + base.close(); + } + } + + final String[] oldNames = { "30.cfs", "30.nocfs", "31.cfs", "31.nocfs", }; + + public void testAddOldIndex() throws IOException { + int i = random.nextInt(oldNames.length); + File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]); + Directory dir = null; + try { + _TestUtil + .unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir); + dir = newFSDirectory(oldIndxeDir); + SegmentInfos infos = new SegmentInfos(); + infos.read(dir); + SortedMap sortedMap = new TreeMap(); + + FieldNumberBiMap biMap = new FieldNumberBiMap(); + int maxFieldNum = Integer.MIN_VALUE; + for (SegmentInfo segmentInfo : infos) { + for (FieldInfo fieldInfo : segmentInfo.getFieldInfos()) { + int globNumber = biMap.addOrGet(fieldInfo.name, fieldInfo.number); + maxFieldNum = Math.max(maxFieldNum, globNumber); + sortedMap.put(globNumber, fieldInfo.name); + } + } + Directory base = newDirectory(); + IndexWriter writer = new IndexWriter(base, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy( + NoMergePolicy.NO_COMPOUND_FILES)); + + SortedMap copySortedMap = new TreeMap( + sortedMap); + while (!sortedMap.isEmpty()) { // add every field at least once + Document doc = new Document(); + int nextField = random.nextInt(maxFieldNum + 1); + sortedMap.remove(nextField); + + String name = copySortedMap.get(nextField); + assertNotNull(name); + + doc.add(newField(name, _TestUtil.randomRealisticUnicodeString(random), + Index.toIndex(true, random.nextBoolean(), random.nextBoolean()))); + writer.addDocument(doc); + if (random.nextInt(10) == 0) { + writer.commit(); + } + } + Set> expectedEntries = writer.segmentInfos + .getOrLoadGlobalFieldNumberMap(base).entries(); + writer.addIndexes(dir); // add the old index + writer.close(); + + writer = new IndexWriter(base, newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer()).setMergePolicy(NoMergePolicy.NO_COMPOUND_FILES)); + writer.commit(); // make sure the old index is the latest segment + writer.close(); + + // we don't merge here since we use NoMergePolicy + SegmentInfos sis = new SegmentInfos(); + sis.read(base); + // check that the latest global field numbers are consistent and carried + // over from the 4.0 index + FieldNumberBiMap actualGlobalMap = sis + .getOrLoadGlobalFieldNumberMap(base); + assertEquals(expectedEntries, actualGlobalMap.entries()); + base.close(); + } finally { + if (dir != null) + dir.close(); + _TestUtil.rmDir(oldIndxeDir); + } + } + + public void testFilesOnOldIndex() throws IOException { + int i = random.nextInt(oldNames.length); + File oldIndxeDir = _TestUtil.getTempDir(oldNames[i]); + Directory dir = null; + + MergePolicy policy = random.nextBoolean() ? NoMergePolicy.COMPOUND_FILES : NoMergePolicy.NO_COMPOUND_FILES; + try { + _TestUtil + .unzip(getDataFile("index." + oldNames[i] + ".zip"), oldIndxeDir); + dir = newFSDirectory(oldIndxeDir); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(policy)); + SegmentInfos segmentInfos = writer.segmentInfos; + assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat()); + assertEquals(0, segmentInfos.getGlobalFieldMapVersion()); + Collection files = segmentInfos.files(dir, true); + for (String string : files) { + assertFalse(string.endsWith(".fnx")); + } + writer.commit(); + + assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat()); + assertEquals(0, segmentInfos.getGlobalFieldMapVersion()); + files = segmentInfos.files(dir, true); + for (String string : files) { + assertFalse(string.endsWith(".fnx")); + } + + Document d = new Document(); + d.add(new Field("f1", "d1 first field", Store.YES, Index.ANALYZED, + TermVector.NO)); + writer.addDocument(d); + writer.prepareCommit(); + // the fnx file should still be under control of the SIS + assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat()); + assertEquals(0, segmentInfos.getLastGlobalFieldMapVersion()); + assertEquals(1, segmentInfos.getGlobalFieldMapVersion()); + files = segmentInfos.files(dir, true); + for (String string : files) { + assertFalse(string.endsWith(".fnx")); + } + + writer.commit(); + + // now we should see the fnx file even if this is a 3.x segment + assertTrue(DefaultSegmentInfosWriter.FORMAT_4_0 < segmentInfos.getFormat()); + assertEquals(1, segmentInfos.getGlobalFieldMapVersion()); + assertEquals(1, segmentInfos.getLastGlobalFieldMapVersion()); + files = segmentInfos.files(dir, true); + assertTrue(files.remove("1.fnx")); + for (String string : files) { + assertFalse(string.endsWith(".fnx")); + } + writer.close(); + } finally { + if (dir != null) + dir.close(); + _TestUtil.rmDir(oldIndxeDir); + } + } + + class KeepAllDeletionPolicy implements IndexDeletionPolicy { + public void onInit(List commits) throws IOException { + } + + public void onCommit(List commits) + throws IOException { + } + } + + public static void assertFNXFiles(Directory dir, String... expectedFnxFiles) + throws IOException { + String[] listAll = dir.listAll(); + Set fnxFiles = new HashSet(); + for (String string : listAll) { + if (string.endsWith(".fnx")) { + fnxFiles.add(string); + } + } + assertEquals("" + fnxFiles, expectedFnxFiles.length, fnxFiles.size()); + for (String string : expectedFnxFiles) { + assertTrue(" missing fnx file: " + string, fnxFiles.contains(string)); + } + } + +} Property changes on: lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java ___________________________________________________________________ Added: svn:keywords + Date Author Id Revision HeadURL Added: svn:eol-style + native Index: lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (revision 1083701) +++ lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java (working copy) @@ -92,10 +92,9 @@ CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs"); FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm"); int contentFieldIndex = -1; - for(i=0;i