Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1061498) +++ lucene/CHANGES.txt (working copy) @@ -464,6 +464,9 @@ * LUCENE-2864: Add getMaxTermFrequency (maximum within-document TF) to FieldInvertState so that it can be used in Similarity.computeNorm. (Robert Muir) + +* LUCENE-2720: Segments now record the code version which wrote them. + (Shai Erera, Mike McCandless, Uwe Schindler) Optimizations Index: lucene/src/java/org/apache/lucene/index/FieldsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldsReader.java (revision 1061498) +++ lucene/src/java/org/apache/lucene/index/FieldsReader.java (working copy) @@ -75,6 +75,26 @@ ensureOpen(); return new FieldsReader(fieldInfos, numTotalDocs, size, format, formatSize, docStoreOffset, cloneableFieldsStream, cloneableIndexStream); } + + /** + * Detects the code version this segment was written with. Returns either + * "2.x" for all pre-3.0 segments, or "3.0" for 3.0 segments. This method + * should not be called for 3.1+ segments since they already record their code + * version. + */ + static String detectCodeVersion(Directory dir, String segment) throws IOException { + IndexInput idxStream = dir.openInput(IndexFileNames.segmentFileName(segment, IndexFileNames.FIELDS_INDEX_EXTENSION), 1024); + try { + int format = idxStream.readInt(); + if (format < FieldsWriter.FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS) { + return "2.x"; + } else { + return "3.0"; + } + } finally { + idxStream.close(); + } + } // Used only by clone private FieldsReader(FieldInfos fieldInfos, int numTotalDocs, int size, int format, int formatSize, Index: lucene/src/java/org/apache/lucene/index/SegmentInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfo.java (revision 1061498) +++ lucene/src/java/org/apache/lucene/index/SegmentInfo.java (working copy) @@ -21,6 +21,8 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.BitVector; +import org.apache.lucene.util.Constants; + import java.io.IOException; import java.util.HashSet; import java.util.List; @@ -95,6 +97,9 @@ private Map diagnostics; + // Tracks the Lucene version this segment was created with. Since 3.1 + private String version; + public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, boolean hasSingleNormFile, boolean hasProx, boolean hasVectors) { this.name = name; @@ -108,6 +113,7 @@ delCount = 0; this.hasProx = hasProx; this.hasVectors = hasVectors; + this.version = Constants.LUCENE_MAIN_VERSION; } /** @@ -115,6 +121,7 @@ */ void reset(SegmentInfo src) { clearFiles(); + version = src.version; name = src.name; docCount = src.docCount; dir = src.dir; @@ -153,6 +160,9 @@ */ SegmentInfo(Directory dir, int format, IndexInput input) throws IOException { this.dir = dir; + if (format <= SegmentInfos.FORMAT_3_1) { + version = input.readString(); + } name = input.readString(); docCount = input.readInt(); if (format <= SegmentInfos.FORMAT_LOCKLESS) { @@ -361,6 +371,7 @@ if (normGen != null) { si.normGen = normGen.clone(); } + si.version = version; return si; } @@ -572,6 +583,8 @@ void write(IndexOutput output) throws IOException { assert delCount <= docCount: "delCount=" + delCount + " docCount=" + docCount + " segment=" + name; + // Write the Lucene version that created this segment, since 3.1 + output.writeString(version); output.writeString(name); output.writeInt(docCount); output.writeLong(delGen); @@ -743,7 +756,7 @@ public String toString(Directory dir, int pendingDelCount) { StringBuilder s = new StringBuilder(); - s.append(name).append(':'); + s.append(name).append('(').append(version == null ? "?" : version).append(')').append(':'); char cfs; try { @@ -813,4 +826,24 @@ public int hashCode() { return dir.hashCode() + name.hashCode(); } + + /** + * Used by SegmentInfos to upgrade segments that do not record their code + * version (either "2.x" or "3.0"). + *

+ * NOTE: this method is used for internal purposes only - you should + * not modify the version of a SegmentInfo, or it may result in unexpected + * exceptions thrown when you attempt to open the index. + * + * @lucene.internal + */ + void setVersion(String version) { + this.version = version; + } + + /** Returns the version of the code which wrote the segment. */ + public String getVersion() { + return version; + } + } Index: lucene/src/java/org/apache/lucene/index/SegmentInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfos.java (revision 1061498) +++ lucene/src/java/org/apache/lucene/index/SegmentInfos.java (working copy) @@ -92,8 +92,11 @@ /** Each segment records whether it has term vectors */ public static final int FORMAT_HAS_VECTORS = -10; + /** Each segment records the Lucene version that created it. */ + public static final int FORMAT_3_1 = -11; + /* This must always point to the most recent file format. */ - public static final int CURRENT_FORMAT = FORMAT_HAS_VECTORS; + public static final int CURRENT_FORMAT = FORMAT_3_1; public int counter = 0; // used to name new segments /** @@ -267,7 +270,30 @@ } for (int i = input.readInt(); i > 0; i--) { // read segmentInfos - add(new SegmentInfo(directory, format, input)); + SegmentInfo si = new SegmentInfo(directory, format, input); + if (si.getVersion() == null) { + // It's a pre-3.1 segment, upgrade its version to either 3.0 or 2.x + Directory dir = directory; + if (si.getDocStoreOffset() != -1) { + if (si.getDocStoreIsCompoundFile()) { + dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName( + si.getDocStoreSegment(), + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), 1024); + } + } else if (si.getUseCompoundFile()) { + dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName( + si.name, IndexFileNames.COMPOUND_FILE_EXTENSION), 1024); + } + + try { + String store = si.getDocStoreOffset() != -1 ? si.getDocStoreSegment() : si.name; + si.setVersion(FieldsReader.detectCodeVersion(dir, store)); + } finally { + // If we opened the directory, close it + if (dir != directory) dir.close(); + } + } + add(si); } if(format >= 0){ // in old format the version number may be at the end of the file @@ -351,8 +377,8 @@ segnOutput.writeLong(version); segnOutput.writeInt(counter); // write counter segnOutput.writeInt(size()); // write infos - for (int i = 0; i < size(); i++) { - info(i).write(segnOutput); + for (SegmentInfo si : this) { + si.write(segnOutput); } segnOutput.writeStringStringMap(userData); segnOutput.prepareCommit(); Index: lucene/src/java/org/apache/lucene/util/Constants.java =================================================================== --- lucene/src/java/org/apache/lucene/util/Constants.java (revision 1061498) +++ lucene/src/java/org/apache/lucene/util/Constants.java (working copy) @@ -70,6 +70,9 @@ return s.toString(); } + // NOTE: we track per-segment version as a String with the "X.Y" format, e.g. + // "4.0", "3.1", "3.0". Therefore when we change this constant, we should keep + // the format. public static final String LUCENE_MAIN_VERSION = ident("3.1"); public static final String LUCENE_VERSION;