+ * If you want to reuse functionality of this codec in another codec, extend
+ * {@link FilterCodec}.
+ *
+ * @see org.apache.lucene.codecs.lucene74 package documentation for file format details.
+ *
+ * @lucene.experimental
+ */
+public class Lucene74Codec extends Codec {
+ private final TermVectorsFormat vectorsFormat = new Lucene50TermVectorsFormat();
+ private final FieldInfosFormat fieldInfosFormat = new Lucene60FieldInfosFormat();
+ private final SegmentInfoFormat segmentInfosFormat = new Lucene70SegmentInfoFormat();
+ private final LiveDocsFormat liveDocsFormat = new Lucene74LiveDocsFormat();
+ private final CompoundFormat compoundFormat = new Lucene50CompoundFormat();
+
+ private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() {
+ @Override
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return Lucene74Codec.this.getPostingsFormatForField(field);
+ }
+ };
+
+ private final DocValuesFormat docValuesFormat = new PerFieldDocValuesFormat() {
+ @Override
+ public DocValuesFormat getDocValuesFormatForField(String field) {
+ return Lucene74Codec.this.getDocValuesFormatForField(field);
+ }
+ };
+
+ private final StoredFieldsFormat storedFieldsFormat;
+
+ /**
+ * Instantiates a new codec.
+ */
+ public Lucene74Codec() {
+ this(Mode.BEST_SPEED);
+ }
+
+ /**
+ * Instantiates a new codec, specifying the stored fields compression
+ * mode to use.
+ * @param mode stored fields compression mode to use for newly
+ * flushed/merged segments.
+ */
+ public Lucene74Codec(Mode mode) {
+ super("Lucene74");
+ this.storedFieldsFormat = new Lucene50StoredFieldsFormat(Objects.requireNonNull(mode));
+ }
+
+ @Override
+ public final StoredFieldsFormat storedFieldsFormat() {
+ return storedFieldsFormat;
+ }
+
+ @Override
+ public final TermVectorsFormat termVectorsFormat() {
+ return vectorsFormat;
+ }
+
+ @Override
+ public final PostingsFormat postingsFormat() {
+ return postingsFormat;
+ }
+
+ @Override
+ public final FieldInfosFormat fieldInfosFormat() {
+ return fieldInfosFormat;
+ }
+
+ @Override
+ public final SegmentInfoFormat segmentInfoFormat() {
+ return segmentInfosFormat;
+ }
+
+ @Override
+ public final LiveDocsFormat liveDocsFormat() {
+ return liveDocsFormat;
+ }
+
+ @Override
+ public final CompoundFormat compoundFormat() {
+ return compoundFormat;
+ }
+
+ @Override
+ public final PointsFormat pointsFormat() {
+ return new Lucene60PointsFormat();
+ }
+
+ /** Returns the postings format that should be used for writing
+ * new segments of field.
+ *
+ * The default implementation always returns "Lucene50".
+ *
+ * WARNING: if you subclass, you are responsible for index
+ * backwards compatibility: future version of Lucene are only
+ * guaranteed to be able to read the default implementation.
+ */
+ public PostingsFormat getPostingsFormatForField(String field) {
+ return defaultFormat;
+ }
+
+ /** Returns the docvalues format that should be used for writing
+ * new segments of field.
+ *
+ * The default implementation always returns "Lucene70".
+ *
+ * WARNING: if you subclass, you are responsible for index + * backwards compatibility: future version of Lucene are only + * guaranteed to be able to read the default implementation. + */ + public DocValuesFormat getDocValuesFormatForField(String field) { + return defaultDVFormat; + } + + @Override + public final DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene50"); + private final DocValuesFormat defaultDVFormat = DocValuesFormat.forName("Lucene70"); + + private final NormsFormat normsFormat = new Lucene70NormsFormat(); + + @Override + public final NormsFormat normsFormat() { + return normsFormat; + } +} diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene74/Lucene74LiveDocsFormat.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene74/Lucene74LiveDocsFormat.java new file mode 100644 index 0000000..a9168d3 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene74/Lucene74LiveDocsFormat.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.codecs.lucene74; + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.codecs.CodecUtil; +import org.apache.lucene.codecs.LiveDocsFormat; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentCommitInfo; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.DataOutput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.MutableBits; + +/** + * Lucene 7.4 live docs format + *
The .liv file is optional, and only exists when a segment contains + * deletions. + *
Although per-segment, this file is maintained exterior to compound segment + * files. + *
Deletions (.liv) --> IndexHeader,Generation,Format,Bits + *
This document defines the index file formats used in this version of Lucene.
+ * If you are using a different version of Lucene, please consult the copy of
+ * docs/ that was distributed with
+ * the version you are using.
This document attempts to provide a high-level definition of the Apache + * Lucene file formats.
+ *The fundamental concepts in Lucene are index, document, field and term.
+ *An index contains a sequence of documents.
+ *The same sequence of bytes in two different fields is considered a different + * term. Thus terms are represented as a pair: the string naming the field, and the + * bytes within the field.
+ * + *The index stores statistics about terms in order to make term-based search + * more efficient. Lucene's index falls into the family of indexes known as an + * inverted index. This is because it can list, for a term, the documents + * that contain it. This is the inverse of the natural relationship, in which + * documents list terms.
+ * + *In Lucene, fields may be stored, in which case their text is stored + * in the index literally, in a non-inverted manner. Fields that are inverted are + * called indexed. A field may be both stored and indexed.
+ *The text of a field may be tokenized into terms to be indexed, or the + * text of a field may be used literally as a term to be indexed. Most fields are + * tokenized, but sometimes it is useful for certain identifier fields to be + * indexed literally.
+ *See the {@link org.apache.lucene.document.Field Field} + * java docs for more information on Fields.
+ * + *Lucene indexes may be composed of multiple sub-indexes, or segments. + * Each segment is a fully independent index, which could be searched separately. + * Indexes evolve by:
+ *Searches may involve multiple segments and/or multiple indexes, each index + * potentially composed of a set of segments.
+ * + *Internally, Lucene refers to documents by an integer document number. + * The first document added to an index is numbered zero, and each subsequent + * document added gets a number one greater than the previous.
+ *Note that a document's number may change, so caution should be taken when + * storing these numbers outside of Lucene. In particular, numbers may change in + * the following situations:
+ *The numbers stored in each segment are unique only within the segment, and + * must be converted before they can be used in a larger context. The standard + * technique is to allocate each segment a range of values, based on the range of + * numbers used in that segment. To convert a document number from a segment to an + * external value, the segment's base document number is added. To convert + * an external value back to a segment-specific value, the segment is identified + * by the range that the external value is in, and the segment's base value is + * subtracted. For example two five document segments might be combined, so that + * the first segment has a base value of zero, and the second of five. Document + * three from the second segment would have an external value of eight.
+ *When documents are deleted, gaps are created in the numbering. These are + * eventually removed as the index evolves through merging. Deleted documents are + * dropped when segments are merged. A freshly-merged segment thus has no gaps in + * its numbering.
+ *Each segment index maintains the following:
+ *Details on each of these are provided in their linked pages.
+ *All files belonging to a segment have the same name with varying extensions. + * The extensions correspond to the different file formats described below. When + * using the Compound File format (default for small segments) these files (except + * for the Segment info file, the Lock file, and Deleted documents file) are collapsed + * into a single .cfs file (see below for details)
+ *Typically, all segments in an index are stored in a single directory, + * although this is not required.
+ *File names are never re-used. That is, when any file is saved + * to the Directory it is given a never before used filename. This is achieved + * using a simple generations approach. For example, the first segments file is + * segments_1, then segments_2, etc. The generation is a sequential long integer + * represented in alpha-numeric (base 36) form.
+ *The following table summarizes the names and extensions of the files in + * Lucene:
+ *| Name | + *Extension | + *Brief Description | + *
|---|---|---|
| {@link org.apache.lucene.index.SegmentInfos Segments File} | + *segments_N | + *Stores information about a commit point | + *
| Lock File | + *write.lock | + *The Write lock prevents multiple IndexWriters from writing to the same + * file. | + *
| {@link org.apache.lucene.codecs.lucene70.Lucene70SegmentInfoFormat Segment Info} | + *.si | + *Stores metadata about a segment | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat Compound File} | + *.cfs, .cfe | + *An optional "virtual" file consisting of all the other index files for + * systems that frequently run out of file handles. | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50FieldInfosFormat Fields} | + *.fnm | + *Stores information about the fields | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat Field Index} | + *.fdx | + *Contains pointers to field data | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat Field Data} | + *.fdt | + *The stored fields for documents | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term Dictionary} | + *.tim | + *The term dictionary, stores term info | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Term Index} | + *.tip | + *The index into the Term Dictionary | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Frequencies} | + *.doc | + *Contains the list of docs which contain each term along with frequency | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Positions} | + *.pos | + *Stores position information about where a term occurs in the index | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50PostingsFormat Payloads} | + *.pay | + *Stores additional per-position metadata information such as character offsets and user payloads | + *
| {@link org.apache.lucene.codecs.lucene70.Lucene70NormsFormat Norms} | + *.nvd, .nvm | + *Encodes length and boost factors for docs and fields | + *
| {@link org.apache.lucene.codecs.lucene70.Lucene70DocValuesFormat Per-Document Values} | + *.dvd, .dvm | + *Encodes additional scoring factors or other per-document information. | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat Term Vector Index} | + *.tvx | + *Stores offset into the document data file | + *
| {@link org.apache.lucene.codecs.lucene50.Lucene50TermVectorsFormat Term Vector Data} | + *.tvd | + *Contains term vector data. | + *
| {@link org.apache.lucene.codecs.lucene74.Lucene74LiveDocsFormat Live Documents} | + *.liv | + *Info about what documents are live | + *
| {@link org.apache.lucene.codecs.lucene60.Lucene60PointsFormat Point values} | + *.dii, .dim | + *Holds indexed points, if any | + *
Compatibility notes are provided in this document, describing how file + * formats have changed from prior versions:
+ *Lucene uses a Java int to refer to
+ * document numbers, and the index file format uses an Int32
+ * on-disk to store document numbers. This is a limitation
+ * of both the index file format and the current implementation. Eventually these
+ * should be replaced with either UInt64 values, or
+ * better yet, {@link org.apache.lucene.store.DataOutput#writeVInt VInt} values which have no limit.