diff --git a/dev-tools/eclipse/dot.classpath b/dev-tools/eclipse/dot.classpath index ee6a742..5876841 100644 --- a/dev-tools/eclipse/dot.classpath +++ b/dev-tools/eclipse/dot.classpath @@ -1,9 +1,7 @@ - - - + @@ -15,7 +13,6 @@ - diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index bc21a6d..247a47c 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -215,7 +215,7 @@ Changes in Runtime Behavior * LUCENE-2881: FieldInfos is now tracked per segment. Before it was tracked per IndexWriter session, which resulted in FieldInfos that had the FieldInfo properties from all previous segments combined. Field numbers are now tracked - globally across IndexWriter sessions and persisted into a _X.fnx file on + globally across IndexWriter sessions and persisted into a X.fnx file on successful commit. The corresponding file format changes are backwards- compatible. (Michael Busch, Simon Willnauer) @@ -375,8 +375,9 @@ New features * LUCENE-1458, LUCENE-2111: With flexible indexing it is now possible for an application to create its own postings codec, to alter how fields, terms, docs and positions are encoded into the index. The - standard codec is the default codec. IndexWriter accepts a Codec - class to obtain codecs for newly written segments. + standard codec is the default codec. Both IndexWriter and + IndexReader accept a CodecProvider class to obtain codecs for newly + written segments as well as existing segments opened for reading. * LUCENE-1458, LUCENE-2111: Some experimental codecs have been added for flexible indexing, including pulsing codec (inlines @@ -402,7 +403,7 @@ New features * LUCENE-2489: Added PerFieldCodecWrapper (in oal.index.codecs) which lets you set the Codec per field (Mike McCandless) -* LUCENE-2373: Extend Codec to use SegmentInfosWriter and +* LUCENE-2373: Extend CodecProvider to use SegmentInfosWriter and SegmentInfosReader to allow customization of SegmentInfos data. (Andrzej Bialecki) @@ -444,10 +445,10 @@ New features (i.e. \* or "*") Custom QueryParser subclasses overriding getRangeQuery() will be passed null for any open endpoint. (Adriano Crestani, yonik) -* LUCENE-2742: Add native per-field postings format support. Codec lets you now - register a postings format for each field and which is in turn recorded - into the index. Postings formtas are maintained on a per-segment basis and be - resolved without knowing the actual postings format used for writing the segment. +* LUCENE-2742: Add native per-field codec support. CodecProvider lets you now + register a codec for each field and which is in turn recorded in the segment + and field information. Codecs are maintained on a per-segment basis and be + resolved without knowing the actual codec used for writing the segment. (Simon Willnauer) * LUCENE-2741: Add support for multiple codecs that use the same file diff --git a/lucene/build.xml b/lucene/build.xml index 32a3547..e6c904a 100644 --- a/lucene/build.xml +++ b/lucene/build.xml @@ -30,15 +30,15 @@ - - + + - + @@ -556,11 +556,11 @@ - + diff --git a/lucene/common-build.xml b/lucene/common-build.xml index f86dd69..b897056 100644 --- a/lucene/common-build.xml +++ b/lucene/common-build.xml @@ -83,7 +83,8 @@ - + + @@ -472,13 +473,8 @@ - - - - - - @@ -555,7 +551,9 @@ - + + + diff --git a/lucene/contrib/contrib-build.xml b/lucene/contrib/contrib-build.xml index e2eb4bd..30d0bc3 100644 --- a/lucene/contrib/contrib-build.xml +++ b/lucene/contrib/contrib-build.xml @@ -39,8 +39,8 @@ - + diff --git a/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java b/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java index 57a96e1..060778c 100644 --- a/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java +++ b/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java @@ -34,7 +34,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TopDocs; @@ -108,7 +107,7 @@ public class MemoryIndexTest extends BaseTokenStreamTestCase { Directory ramdir = newDirectory(); Analyzer analyzer = randomAnalyzer(); IndexWriter writer = new IndexWriter(ramdir, - new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); + new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); Document doc = new Document(); Field field1 = newField("foo", fooField.toString(), TextField.TYPE_UNSTORED); Field field2 = newField("term", termField.toString(), TextField.TYPE_UNSTORED); diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java index 4c10dfc..e1c9cde 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java @@ -27,6 +27,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.lucene.index.IndexWriter; // Required for javadocs +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.FSDirectory; /** @@ -53,6 +54,8 @@ import org.apache.lucene.store.FSDirectory; */ public class IndexSplitter { public SegmentInfos infos; + + private final CodecProvider codecs; FSDirectory fsDir; @@ -93,12 +96,17 @@ public class IndexSplitter { is.split(targetDir, segs.toArray(new String[0])); } } - + public IndexSplitter(File dir) throws IOException { + this(dir, CodecProvider.getDefault()); + } + + public IndexSplitter(File dir, CodecProvider codecs) throws IOException { this.dir = dir; + this.codecs = codecs; fsDir = FSDirectory.open(dir); - infos = new SegmentInfos(); - infos.read(fsDir); + infos = new SegmentInfos(codecs); + infos.read(fsDir, codecs); } public void listSegments() throws IOException { @@ -132,13 +140,13 @@ public class IndexSplitter { infos.remove(idx); } infos.changed(); - infos.commit(fsDir, infos.codecFormat()); + infos.commit(fsDir); } public void split(File destDir, String[] segs) throws IOException { destDir.mkdirs(); FSDirectory destFSDir = FSDirectory.open(destDir); - SegmentInfos destInfos = new SegmentInfos(); + SegmentInfos destInfos = new SegmentInfos(codecs); destInfos.counter = infos.counter; for (String n : segs) { SegmentInfo info = getInfo(n); @@ -152,7 +160,7 @@ public class IndexSplitter { } } destInfos.changed(); - destInfos.commit(destFSDir, infos.codecFormat()); + destInfos.commit(destFSDir); // System.out.println("destDir:"+destDir.getAbsolutePath()); } diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java similarity index 60% rename from lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java rename to lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java index e2dd314..3272cf6 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java @@ -20,37 +20,50 @@ package org.apache.lucene.index.codecs.appending; import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesProducer; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; +import org.apache.lucene.index.codecs.PerDocValues; +import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.standard.StandardPostingsReader; import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; import org.apache.lucene.index.codecs.BlockTermsReader; import org.apache.lucene.index.codecs.TermsIndexReaderBase; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; /** - * Appending postings impl + * This codec extends {@link StandardCodec} to work on append-only outputs, such + * as plain output streams and append-only filesystems. + * + *

Note: compound file format feature is not compatible with + * this codec. You must call both + * LogMergePolicy.setUseCompoundFile(false) and + * LogMergePolicy.setUseCompoundDocStore(false) to disable + * compound file format.

+ * @lucene.experimental */ -class AppendingPostingsFormat extends PostingsFormat { +public class AppendingCodec extends Codec { public static String CODEC_NAME = "Appending"; - public AppendingPostingsFormat() { + public AppendingCodec() { super(CODEC_NAME); } @Override public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase docsWriter = new Lucene40PostingsWriter(state); + PostingsWriterBase docsWriter = new StandardPostingsWriter(state); boolean success = false; AppendingTermsIndexWriter indexWriter = null; try { @@ -80,7 +93,7 @@ class AppendingPostingsFormat extends PostingsFormat { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase docsReader = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -90,7 +103,7 @@ class AppendingPostingsFormat extends PostingsFormat { state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.segmentSuffix, state.context); + state.codecId, state.context); success = true; } finally { if (!success) { @@ -103,8 +116,8 @@ class AppendingPostingsFormat extends PostingsFormat { state.dir, state.fieldInfos, state.segmentInfo.name, docsReader, state.context, - Lucene40PostingsFormat.TERMS_CACHE_SIZE, - state.segmentSuffix); + StandardCodec.TERMS_CACHE_SIZE, + state.codecId); success = true; return ret; } finally { @@ -119,10 +132,27 @@ class AppendingPostingsFormat extends PostingsFormat { } @Override - public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) + public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { - Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); - BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + StandardPostingsReader.files(dir, segmentInfo, codecId, files); + BlockTermsReader.files(dir, segmentInfo, codecId, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); + DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files); + } + + @Override + public void getExtensions(Set extensions) { + StandardCodec.getStandardExtensions(extensions); + DefaultDocValuesConsumer.getExtensions(extensions); + } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new DefaultDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new DefaultDocValuesProducer(state); } } diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java deleted file mode 100644 index 03544ea..0000000 diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java index dc49fad..1ecb3e2 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java @@ -20,6 +20,8 @@ package org.apache.lucene.index.codecs.appending; import java.io.IOException; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; public class AppendingSegmentInfosWriter extends DefaultSegmentInfosWriter { diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java index 742de0b..7f885ee 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java @@ -34,9 +34,9 @@ public class AppendingTermsDictReader extends BlockTermsReader { public AppendingTermsDictReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext context, - int termsCacheSize, String segmentSuffix) throws IOException { + int termsCacheSize, int codecId) throws IOException { super(indexReader, dir, fieldInfos, segment, postingsReader, context, - termsCacheSize, segmentSuffix); + termsCacheSize, codecId); } @Override diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java index ed393ad..205dc15 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java @@ -31,9 +31,9 @@ import org.apache.lucene.util.CodecUtil; public class AppendingTermsIndexReader extends FixedGapTermsIndexReader { public AppendingTermsIndexReader(Directory dir, FieldInfos fieldInfos, - String segment, int indexDivisor, Comparator termComp, String segmentSuffix, IOContext context) + String segment, int indexDivisor, Comparator termComp, int codecId, IOContext context) throws IOException { - super(dir, fieldInfos, segment, indexDivisor, termComp, segmentSuffix, context); + super(dir, fieldInfos, segment, indexDivisor, termComp, codecId, context); } @Override diff --git a/lucene/contrib/misc/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec b/lucene/contrib/misc/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec deleted file mode 100644 index c29837d..0000000 diff --git a/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java b/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java index c35d823..cb79651 100644 --- a/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java +++ b/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java @@ -27,9 +27,7 @@ import org.apache.lucene.search.SearcherWarmer; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NRTCachingDirectory; -import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; -@UseNoMemoryExpensiveCodec public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase { private final ThreadLocal lastGens = new ThreadLocal(); diff --git a/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java b/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java index 9398551..60463f7 100644 --- a/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java +++ b/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java @@ -34,6 +34,11 @@ import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.DefaultSegmentInfosReader; +import org.apache.lucene.index.codecs.SegmentInfosReader; +import org.apache.lucene.index.codecs.SegmentInfosWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -45,7 +50,30 @@ import org.apache.lucene.util.Version; public class TestAppendingCodec extends LuceneTestCase { - private static class AppendingIndexOutputWrapper extends IndexOutput { + static class AppendingCodecProvider extends CodecProvider { + Codec appending = new AppendingCodec(); + SegmentInfosWriter infosWriter = new AppendingSegmentInfosWriter(); + SegmentInfosReader infosReader = new DefaultSegmentInfosReader(); + public AppendingCodecProvider() { + setDefaultFieldCodec(appending.name); + } + @Override + public Codec lookup(String name) { + return appending; + } + + @Override + public SegmentInfosReader getSegmentInfosReader() { + return infosReader; + } + @Override + public SegmentInfosWriter getSegmentInfosWriter() { + return infosWriter; + } + + } + + private static class AppendingIndexOutputWrapper extends IndexOutput { IndexOutput wrapped; public AppendingIndexOutputWrapper(IndexOutput wrapped) { @@ -109,7 +137,7 @@ public class TestAppendingCodec extends LuceneTestCase { Directory dir = new AppendingRAMDirectory(random, new RAMDirectory()); IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(random)); - cfg.setCodec(new AppendingCodec()); + cfg.setCodecProvider(new AppendingCodecProvider()); ((TieredMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false); IndexWriter writer = new IndexWriter(dir, cfg); Document doc = new Document(); @@ -123,7 +151,7 @@ public class TestAppendingCodec extends LuceneTestCase { writer.addDocument(doc); writer.optimize(); writer.close(); - IndexReader reader = IndexReader.open(dir, null, true, 1); + IndexReader reader = IndexReader.open(dir, null, true, 1, new AppendingCodecProvider()); assertEquals(2, reader.numDocs()); Document doc2 = reader.document(0); assertEquals(text, doc2.get("f")); diff --git a/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java b/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java index f86a20d..c06d8f1 100644 --- a/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java +++ b/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java @@ -34,11 +34,9 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; import org.apache.lucene.util.NamedThreadFactory; import org.apache.lucene.util._TestUtil; -@UseNoMemoryExpensiveCodec public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase { boolean warmCalled; diff --git a/lucene/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/src/java/org/apache/lucene/index/CheckIndex.java index 6dd0909..9712f5c 100644 --- a/lucene/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/src/java/org/apache/lucene/index/CheckIndex.java @@ -25,7 +25,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.document.Document; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import java.io.File; import java.io.IOException; @@ -143,8 +143,8 @@ public class CheckIndex { /** Name of the segment. */ public String name; - /** Codec used to read this segment. */ - public Codec codec; + /** CodecInfo used to read this segment. */ + public SegmentCodecs codec; /** Document count (does not take deletions into account). */ public int docCount; @@ -322,6 +322,10 @@ public class CheckIndex { public Status checkIndex() throws IOException { return checkIndex(null); } + + public Status checkIndex(List onlySegments) throws IOException { + return checkIndex(onlySegments, CodecProvider.getDefault()); + } /** Returns a {@link Status} instance detailing * the state of the index. @@ -335,13 +339,13 @@ public class CheckIndex { *

WARNING: make sure * you only call this when the index is not opened by any * writer. */ - public Status checkIndex(List onlySegments) throws IOException { + public Status checkIndex(List onlySegments, CodecProvider codecs) throws IOException { NumberFormat nf = NumberFormat.getInstance(); - SegmentInfos sis = new SegmentInfos(); + SegmentInfos sis = new SegmentInfos(codecs); Status result = new Status(); result.dir = dir; try { - sis.read(dir); + sis.read(dir, codecs); } catch (Throwable t) { msg("ERROR: could not read any segments file in directory"); result.missingSegments = true; @@ -373,7 +377,6 @@ public class CheckIndex { final int numSegments = sis.size(); final String segmentsFileName = sis.getCurrentSegmentFileName(); - // note: we only read the format byte (required preamble) here! IndexInput input = null; try { input = dir.openInput(segmentsFileName, IOContext.DEFAULT); @@ -486,7 +489,7 @@ public class CheckIndex { SegmentReader reader = null; try { - final Codec codec = info.getCodec(); + final SegmentCodecs codec = info.getSegmentCodecs(); msg(" codec=" + codec); segInfoStat.codec = codec; msg(" compound=" + info.getUseCompoundFile()); @@ -1179,11 +1182,11 @@ public class CheckIndex { * *

WARNING: Make sure you only call this when the * index is not opened by any writer. */ - public void fixIndex(Status result, Codec codec) throws IOException { + public void fixIndex(Status result) throws IOException { if (result.partial) throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)"); result.newSegments.changed(); - result.newSegments.commit(result.dir, codec); + result.newSegments.commit(result.dir); } private static boolean assertsOn; @@ -1233,7 +1236,6 @@ public class CheckIndex { public static void main(String[] args) throws IOException, InterruptedException { boolean doFix = false; - Codec codec = Codec.getDefault(); // only used when fixing boolean verbose = false; List onlySegments = new ArrayList(); String indexPath = null; @@ -1242,13 +1244,6 @@ public class CheckIndex { if (args[i].equals("-fix")) { doFix = true; i++; - } else if (args[i].equals("-codec")) { - if (i == args.length-1) { - System.out.println("ERROR: missing name for -codec option"); - System.exit(1); - } - codec = Codec.forName(args[i+1]); - i+=2; } else if (args[i].equals("-verbose")) { verbose = true; i++; @@ -1274,7 +1269,6 @@ public class CheckIndex { System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + - " -codec X: when fixing, codec to write the new segments_N file with\n" + " -verbose: print additional details\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + @@ -1335,7 +1329,7 @@ public class CheckIndex { System.out.println(" " + (5-s) + "..."); } System.out.println("Writing..."); - checker.fixIndex(result, codec); + checker.fixIndex(result); System.out.println("OK"); System.out.println("Wrote new segments file \"" + result.newSegments.getCurrentSegmentFileName() + "\""); } diff --git a/lucene/src/java/org/apache/lucene/index/DirectoryReader.java b/lucene/src/java/org/apache/lucene/index/DirectoryReader.java index f15f077..190ebd8 100644 --- a/lucene/src/java/org/apache/lucene/index/DirectoryReader.java +++ b/lucene/src/java/org/apache/lucene/index/DirectoryReader.java @@ -33,6 +33,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -44,6 +45,8 @@ import org.apache.lucene.util.MapBackedSet; class DirectoryReader extends IndexReader implements Cloneable { protected Directory directory; protected boolean readOnly; + + protected CodecProvider codecs; IndexWriter writer; @@ -75,13 +78,15 @@ class DirectoryReader extends IndexReader implements Cloneable { // } static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, - final int termInfosIndexDivisor) throws CorruptIndexException, IOException { + final int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException { + final CodecProvider codecProvider = codecs == null ? CodecProvider.getDefault() + : codecs; return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - SegmentInfos infos = new SegmentInfos(); - infos.read(directory, segmentFileName); - return new DirectoryReader(directory, infos, deletionPolicy, readOnly, termInfosIndexDivisor); + SegmentInfos infos = new SegmentInfos(codecProvider); + infos.read(directory, segmentFileName, codecProvider); + return new DirectoryReader(directory, infos, deletionPolicy, readOnly, termInfosIndexDivisor, codecProvider); } }.run(commit); } @@ -92,12 +97,17 @@ class DirectoryReader extends IndexReader implements Cloneable { // } /** Construct reading the named set of readers. */ - DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException { + DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws IOException { this.directory = directory; this.readOnly = readOnly; this.segmentInfos = sis; this.deletionPolicy = deletionPolicy; this.termInfosIndexDivisor = termInfosIndexDivisor; + if (codecs == null) { + this.codecs = CodecProvider.getDefault(); + } else { + this.codecs = codecs; + } readerFinishedListeners = new MapBackedSet(new ConcurrentHashMap()); applyAllDeletes = false; @@ -131,12 +141,17 @@ class DirectoryReader extends IndexReader implements Cloneable { } // Used by near real-time search - DirectoryReader(IndexWriter writer, SegmentInfos infos, boolean applyAllDeletes) throws IOException { + DirectoryReader(IndexWriter writer, SegmentInfos infos, CodecProvider codecs, boolean applyAllDeletes) throws IOException { this.directory = writer.getDirectory(); this.readOnly = true; this.applyAllDeletes = applyAllDeletes; // saved for reopen this.termInfosIndexDivisor = writer.getConfig().getReaderTermsIndexDivisor(); + if (codecs == null) { + this.codecs = CodecProvider.getDefault(); + } else { + this.codecs = codecs; + } readerFinishedListeners = writer.getReaderFinishedListeners(); // IndexWriter synchronizes externally before calling @@ -185,7 +200,8 @@ class DirectoryReader extends IndexReader implements Cloneable { /** This constructor is only used for {@link #doOpenIfChanged()} */ DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, - boolean readOnly, boolean doClone, int termInfosIndexDivisor, Collection readerFinishedListeners) throws IOException { + boolean readOnly, boolean doClone, int termInfosIndexDivisor, CodecProvider codecs, + Collection readerFinishedListeners) throws IOException { this.directory = directory; this.readOnly = readOnly; this.segmentInfos = infos; @@ -193,6 +209,13 @@ class DirectoryReader extends IndexReader implements Cloneable { this.readerFinishedListeners = readerFinishedListeners; applyAllDeletes = false; + if (codecs == null) { + this.codecs = CodecProvider.getDefault(); + } else { + this.codecs = codecs; + } + + // we put the old SegmentReaders in a map, that allows us // to lookup a reader using its segment name Map segmentReaders = new HashMap(); @@ -324,7 +347,7 @@ class DirectoryReader extends IndexReader implements Cloneable { starts[subReaders.length] = maxDoc; if (!readOnly) { - maxIndexVersion = SegmentInfos.readCurrentVersion(directory); + maxIndexVersion = SegmentInfos.readCurrentVersion(directory, codecs); } } @@ -475,15 +498,15 @@ class DirectoryReader extends IndexReader implements Cloneable { return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - final SegmentInfos infos = new SegmentInfos(); - infos.read(directory, segmentFileName); + final SegmentInfos infos = new SegmentInfos(codecs); + infos.read(directory, segmentFileName, codecs); return doOpenIfChanged(infos, false, openReadOnly); } }.run(commit); } private synchronized DirectoryReader doOpenIfChanged(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException { - return new DirectoryReader(directory, infos, subReaders, openReadOnly, doClone, termInfosIndexDivisor, readerFinishedListeners); + return new DirectoryReader(directory, infos, subReaders, openReadOnly, doClone, termInfosIndexDivisor, codecs, readerFinishedListeners); } /** Version number when this IndexReader was opened. */ @@ -689,7 +712,7 @@ class DirectoryReader extends IndexReader implements Cloneable { // we have to check whether index has changed since this reader was opened. // if so, this reader is no longer valid for deletion - if (SegmentInfos.readCurrentVersion(directory) > maxIndexVersion) { + if (SegmentInfos.readCurrentVersion(directory, codecs) > maxIndexVersion) { stale = true; this.writeLock.release(); this.writeLock = null; @@ -720,7 +743,7 @@ class DirectoryReader extends IndexReader implements Cloneable { // KeepOnlyLastCommitDeleter: IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, - segmentInfos, null, null); + segmentInfos, null, codecs, null); segmentInfos.updateGeneration(deleter.getLastSegmentInfos()); segmentInfos.changed(); @@ -741,7 +764,7 @@ class DirectoryReader extends IndexReader implements Cloneable { // Sync all files we just wrote directory.sync(segmentInfos.files(directory, false)); - segmentInfos.commit(directory, segmentInfos.codecFormat()); + segmentInfos.commit(directory); success = true; } finally { @@ -819,7 +842,7 @@ class DirectoryReader extends IndexReader implements Cloneable { ensureOpen(); if (writer == null || writer.isClosed()) { // we loaded SegmentInfos from the directory - return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion(); + return SegmentInfos.readCurrentVersion(directory, codecs) == segmentInfos.getVersion(); } else { return writer.nrtIsCurrent(segmentInfos); } @@ -902,12 +925,17 @@ class DirectoryReader extends IndexReader implements Cloneable { /** @see org.apache.lucene.index.IndexReader#listCommits */ public static List listCommits(Directory dir) throws IOException { + return listCommits(dir, CodecProvider.getDefault()); + } + + /** @see org.apache.lucene.index.IndexReader#listCommits */ + public static List listCommits(Directory dir, CodecProvider codecs) throws IOException { final String[] files = dir.listAll(); List commits = new ArrayList(); - SegmentInfos latest = new SegmentInfos(); - latest.read(dir); + SegmentInfos latest = new SegmentInfos(codecs); + latest.read(dir, codecs); final long currentGen = latest.getGeneration(); commits.add(new ReaderCommit(latest, dir)); @@ -920,11 +948,11 @@ class DirectoryReader extends IndexReader implements Cloneable { !fileName.equals(IndexFileNames.SEGMENTS_GEN) && SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) { - SegmentInfos sis = new SegmentInfos(); + SegmentInfos sis = new SegmentInfos(codecs); try { // IOException allowed to throw there, in case // segments_N is corrupt - sis.read(dir, fileName); + sis.read(dir, fileName, codecs); } catch (FileNotFoundException fnfe) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java b/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java index f4512af..02a4a73 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java @@ -25,7 +25,7 @@ import java.util.HashSet; import java.util.Map; import org.apache.lucene.index.DocumentsWriterPerThread.DocState; -import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DocValuesConsumer; import org.apache.lucene.index.codecs.PerDocConsumer; import org.apache.lucene.index.values.PerDocFieldValues; @@ -320,13 +320,14 @@ final class DocFieldProcessor extends DocConsumer { docValuesConsumerAndDocID.docID = docState.docID; return docValuesConsumerAndDocID.docValuesConsumer; } - - PerDocConsumer perDocConsumer = perDocConsumers.get(0); + PerDocConsumer perDocConsumer = perDocConsumers.get(fieldInfo.getCodecId()); if (perDocConsumer == null) { - PerDocWriteState perDocWriteState = docState.docWriter.newPerDocWriteState(""); - DocValuesFormat dvFormat = docState.docWriter.codec.docValuesFormat(); - perDocConsumer = dvFormat.docsConsumer(perDocWriteState); - perDocConsumers.put(0, perDocConsumer); + PerDocWriteState perDocWriteState = docState.docWriter.newPerDocWriteState(fieldInfo.getCodecId()); + SegmentCodecs codecs = perDocWriteState.segmentCodecs; + assert codecs.codecs.length > fieldInfo.getCodecId(); + Codec codec = codecs.codecs[fieldInfo.getCodecId()]; + perDocConsumer = codec.docsConsumer(perDocWriteState); + perDocConsumers.put(Integer.valueOf(fieldInfo.getCodecId()), perDocConsumer); } boolean success = false; DocValuesConsumer docValuesConsumer = null; diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index 73931a4..e414fe1 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -31,7 +31,6 @@ import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; -import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.Query; import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.AlreadyClosedException; @@ -126,11 +125,8 @@ final class DocumentsWriter { final DocumentsWriterPerThreadPool perThreadPool; final FlushPolicy flushPolicy; final DocumentsWriterFlushControl flushControl; - - final Codec codec; - DocumentsWriter(Codec codec, IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers, + DocumentsWriter(IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers, BufferedDeletesStream bufferedDeletesStream) throws IOException { - this.codec = codec; this.directory = directory; this.indexWriter = writer; this.similarityProvider = config.getSimilarityProvider(); diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index 1a42d77..fae432f 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -26,7 +26,7 @@ import java.text.NumberFormat; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FlushInfo; @@ -152,7 +152,7 @@ public class DocumentsWriterPerThread { } private final static boolean INFO_VERBOSE = false; final DocumentsWriter parent; - final Codec codec; + final CodecProvider codecProvider; final IndexWriter writer; final Directory directory; final DocState docState; @@ -183,7 +183,7 @@ public class DocumentsWriterPerThread { this.fieldInfos = fieldInfos; this.writer = parent.indexWriter; this.infoStream = parent.infoStream; - this.codec = parent.codec; + this.codecProvider = this.writer.codecs; this.docState = new DocState(this); this.docState.similarityProvider = parent.indexWriter.getConfig() .getSimilarityProvider(); @@ -405,8 +405,8 @@ public class DocumentsWriterPerThread { return numDocsInRAM; } - Codec getCodec() { - return flushState.codec; + SegmentCodecs getCodec() { + return flushState.segmentCodecs; } /** Reset after a flush */ @@ -443,7 +443,7 @@ public class DocumentsWriterPerThread { assert deleteSlice == null : "all deletes must be applied in prepareFlush"; flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos, numDocsInRAM, writer.getConfig().getTermIndexInterval(), - codec, pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed()))); + fieldInfos.buildSegmentCodecs(true), pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed()))); final double startMBUsed = parent.flushControl.netBytes() / 1024. / 1024.; // Apply delete-by-docID now (delete-byDocID only // happens when an exception is hit processing that @@ -474,12 +474,12 @@ public class DocumentsWriterPerThread { try { consumer.flush(flushState); pendingDeletes.terms.clear(); - final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.codec, fieldInfos.asReadOnly()); + final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.segmentCodecs, fieldInfos.asReadOnly()); if (infoStream != null) { message("new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.liveDocs.count())) + " deleted docs"); message("new segment has " + (newSegment.getHasVectors() ? "vectors" : "no vectors")); message("flushedFiles=" + newSegment.files()); - message("flushed codec=" + newSegment.getCodec()); + message("flushed codecs=" + newSegment.getSegmentCodecs()); } flushedDocCount += flushState.numDocs; @@ -556,9 +556,9 @@ public class DocumentsWriterPerThread { bytesUsed.addAndGet(-(length *(INT_BLOCK_SIZE*RamUsageEstimator.NUM_BYTES_INT))); } - PerDocWriteState newPerDocWriteState(String segmentSuffix) { + PerDocWriteState newPerDocWriteState(int codecId) { assert segment != null; - return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, segmentSuffix, IOContext.DEFAULT); + return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, codecId, IOContext.DEFAULT); } void setInfoStream(PrintStream infoStream) { diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java index 8b959ca..eb6f209 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java @@ -20,6 +20,8 @@ import java.util.Iterator; import java.util.concurrent.locks.ReentrantLock; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; +import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.SetOnce; /** @@ -126,6 +128,7 @@ public abstract class DocumentsWriterPerThreadPool { private final ThreadState[] perThreads; private volatile int numThreadStatesActive; + private CodecProvider codecProvider; private FieldNumberBiMap globalFieldMap; private final SetOnce documentsWriter = new SetOnce(); @@ -145,9 +148,11 @@ public abstract class DocumentsWriterPerThreadPool { public void initialize(DocumentsWriter documentsWriter, FieldNumberBiMap globalFieldMap, IndexWriterConfig config) { this.documentsWriter.set(documentsWriter); // thread pool is bound to DW + final CodecProvider codecs = config.getCodecProvider(); + this.codecProvider = codecs; this.globalFieldMap = globalFieldMap; for (int i = 0; i < perThreads.length; i++) { - final FieldInfos infos = new FieldInfos(globalFieldMap); + final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)); perThreads[i] = new ThreadState(new DocumentsWriterPerThread(documentsWriter.directory, documentsWriter, infos, documentsWriter.chain)); } } @@ -235,7 +240,7 @@ public abstract class DocumentsWriterPerThreadPool { assert threadState.isHeldByCurrentThread(); final DocumentsWriterPerThread dwpt = threadState.perThread; if (!closed) { - final FieldInfos infos = new FieldInfos(globalFieldMap); + final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentCodecsBuilder.create(codecProvider)); final DocumentsWriterPerThread newDwpt = new DocumentsWriterPerThread(dwpt, infos); newDwpt.initialize(); threadState.resetWriter(newDwpt); diff --git a/lucene/src/java/org/apache/lucene/index/FieldInfo.java b/lucene/src/java/org/apache/lucene/index/FieldInfo.java index 2a6ec56..7507f69 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldInfo.java +++ b/lucene/src/java/org/apache/lucene/index/FieldInfo.java @@ -21,6 +21,7 @@ import org.apache.lucene.index.values.ValueType; /** @lucene.experimental */ public final class FieldInfo { + public static final int UNASSIGNED_CODEC_ID = -1; public final String name; public final int number; @@ -37,6 +38,7 @@ public final class FieldInfo { public IndexOptions indexOptions; public boolean storePayloads; // whether this field stores payloads together with term positions + private int codecId = UNASSIGNED_CODEC_ID; // set inside SegmentCodecs#build() during segment flush - this is used to identify the codec used to write this field /** * Controls how much information is stored in the postings lists. @@ -75,11 +77,21 @@ public final class FieldInfo { } assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !storePayloads; } + + void setCodecId(int codecId) { + assert this.codecId == UNASSIGNED_CODEC_ID : "CodecId can only be set once."; + this.codecId = codecId; + } + + public int getCodecId() { + return codecId; + } @Override public Object clone() { FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues); + clone.codecId = this.codecId; return clone; } diff --git a/lucene/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/src/java/org/apache/lucene/index/FieldInfos.java index 306230e..08f7d60 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/src/java/org/apache/lucene/index/FieldInfos.java @@ -29,6 +29,9 @@ import java.util.TreeMap; import java.util.Map.Entry; import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.SegmentCodecs; // Required for Java 1.5 javadocs +import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -143,6 +146,17 @@ public final class FieldInfos implements Iterable { } /** + * Returns a new {@link FieldInfos} instance with this as the global field + * map + * + * @return a new {@link FieldInfos} instance with this as the global field + * map + */ + public FieldInfos newFieldInfos(SegmentCodecsBuilder segmentCodecsBuilder) { + return new FieldInfos(this, segmentCodecsBuilder); + } + + /** * Returns true iff the last committed version differs from the * current version, otherwise false * @@ -184,6 +198,7 @@ public final class FieldInfos implements Iterable { private final SortedMap byNumber = new TreeMap(); private final HashMap byName = new HashMap(); private final FieldNumberBiMap globalFieldNumbers; + private final SegmentCodecsBuilder segmentCodecsBuilder; // First used in 2.9; prior to 2.9 there was no format header public static final int FORMAT_START = -2; @@ -215,15 +230,16 @@ public final class FieldInfos implements Iterable { /** * Creates a new {@link FieldInfos} instance with a private - * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} + * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} and a default {@link SegmentCodecsBuilder} + * initialized with {@link CodecProvider#getDefault()}. *

* Note: this ctor should not be used during indexing use * {@link FieldInfos#FieldInfos(FieldInfos)} or - * {@link FieldInfos#FieldInfos(FieldNumberBiMap)} + * {@link FieldInfos#FieldInfos(FieldNumberBiMap,org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder)} * instead. */ public FieldInfos() { - this(new FieldNumberBiMap()); + this(new FieldNumberBiMap(), SegmentCodecsBuilder.create(CodecProvider.getDefault())); } /** @@ -233,7 +249,7 @@ public final class FieldInfos implements Iterable { * @see #isReadOnly() */ FieldInfos(FieldInfos other) { - this(other.globalFieldNumbers); + this(other.globalFieldNumbers, other.segmentCodecsBuilder); } /** @@ -241,8 +257,9 @@ public final class FieldInfos implements Iterable { * If the {@link FieldNumberBiMap} is null this instance will be read-only. * @see #isReadOnly() */ - FieldInfos(FieldNumberBiMap globalFieldNumbers) { + FieldInfos(FieldNumberBiMap globalFieldNumbers, SegmentCodecsBuilder segmentCodecsBuilder) { this.globalFieldNumbers = globalFieldNumbers; + this.segmentCodecsBuilder = segmentCodecsBuilder; } /** @@ -256,7 +273,7 @@ public final class FieldInfos implements Iterable { * @throws IOException */ public FieldInfos(Directory d, String name) throws IOException { - this((FieldNumberBiMap)null); // use null here to make this FIs Read-Only + this((FieldNumberBiMap)null, null); // use null here to make this FIs Read-Only final IndexInput input = d.openInput(name, IOContext.READONCE); try { read(input, name); @@ -292,7 +309,7 @@ public final class FieldInfos implements Iterable { */ @Override synchronized public Object clone() { - FieldInfos fis = new FieldInfos(globalFieldNumbers); + FieldInfos fis = new FieldInfos(globalFieldNumbers, segmentCodecsBuilder); fis.format = format; fis.hasFreq = hasFreq; fis.hasProx = hasProx; @@ -451,6 +468,7 @@ public final class FieldInfos implements Iterable { if (globalFieldNumbers == null) { throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos"); } + assert segmentCodecsBuilder != null : "SegmentCodecsBuilder is set to null but FieldInfos is not read-only"; FieldInfo fi = fieldInfo(name); if (fi == null) { final int fieldNumber = nextFieldNumber(name, preferredFieldNumber); @@ -459,6 +477,9 @@ public final class FieldInfos implements Iterable { fi.update(isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions); fi.setDocValues(docValues); } + if ((fi.isIndexed || fi.hasDocValues()) && fi.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) { + segmentCodecsBuilder.tryAddAndSet(fi); + } version++; return fi; } @@ -548,6 +569,22 @@ public final class FieldInfos implements Iterable { } return false; } + + /** + * Builds the {@link SegmentCodecs} mapping for this {@link FieldInfos} instance. + * @param clearBuilder true iff the internal {@link SegmentCodecsBuilder} must be cleared otherwise false + */ + public SegmentCodecs buildSegmentCodecs(boolean clearBuilder) { + if (globalFieldNumbers == null) { + throw new IllegalStateException("FieldInfos are read-only no SegmentCodecs available"); + } + assert segmentCodecsBuilder != null; + final SegmentCodecs segmentCodecs = segmentCodecsBuilder.build(); + if (clearBuilder) { + segmentCodecsBuilder.clear(); + } + return segmentCodecs; + } public void write(Directory d, String name) throws IOException { IndexOutput output = d.createOutput(name, IOContext.READONCE); @@ -591,6 +628,7 @@ public final class FieldInfos implements Iterable { bits |= OMIT_POSITIONS; output.writeString(fi.name); output.writeInt(fi.number); + output.writeInt(fi.getCodecId()); output.writeByte(bits); final byte b; @@ -660,7 +698,9 @@ public final class FieldInfos implements Iterable { for (int i = 0; i < size; i++) { String name = input.readString(); + // if this is a previous format codec 0 will be preflex! final int fieldNumber = format <= FORMAT_FLEX? input.readInt():i; + final int codecId = format <= FORMAT_FLEX? input.readInt():0; byte bits = input.readByte(); boolean isIndexed = (bits & IS_INDEXED) != 0; boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0; @@ -741,7 +781,8 @@ public final class FieldInfos implements Iterable { throw new IllegalStateException("unhandled indexValues type " + b); } } - addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType); + final FieldInfo addInternal = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType); + addInternal.setCodecId(codecId); } if (input.getFilePointer() != input.length()) { @@ -763,7 +804,7 @@ public final class FieldInfos implements Iterable { if (isReadOnly()) { return this; } - final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null); + final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null, null); for (FieldInfo fieldInfo : this) { FieldInfo clone = (FieldInfo) (fieldInfo).clone(); roFis.putInternal(clone); @@ -773,14 +814,5 @@ public final class FieldInfos implements Iterable { } return roFis; } - - public boolean anyDocValuesFields() { - for (FieldInfo fi : this) { - if (fi.hasDocValues()) { - return true; - } - } - - return false; - } + } diff --git a/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java b/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java index 758123c..203580d 100644 --- a/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java @@ -57,7 +57,7 @@ final class FreqProxTermsWriter extends TermsHashConsumer { // Sort by field name CollectionUtil.quickSort(allFields); - final FieldsConsumer consumer = state.codec.postingsFormat().fieldsConsumer(state); + final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state); boolean success = false; diff --git a/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java index 2c3be7d..1d5587a 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java @@ -18,6 +18,7 @@ package org.apache.lucene.index; */ import java.io.FileNotFoundException; +import java.io.FilenameFilter; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; @@ -28,6 +29,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NoSuchDirectoryException; import org.apache.lucene.util.CollectionUtil; @@ -120,6 +122,8 @@ final class IndexFileDeleter { infoStream.println("IFD [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message); } + private final FilenameFilter indexFilenameFilter; + // called only from assert private boolean locked() { return writer == null || Thread.holdsLock(writer); @@ -134,7 +138,7 @@ final class IndexFileDeleter { * @throws IOException if there is a low-level IO error */ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, - PrintStream infoStream, IndexWriter writer) throws CorruptIndexException, IOException { + PrintStream infoStream, CodecProvider codecs, IndexWriter writer) throws CorruptIndexException, IOException { this.infoStream = infoStream; this.writer = writer; @@ -150,6 +154,7 @@ final class IndexFileDeleter { // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.getGeneration(); + indexFilenameFilter = new IndexFileNameFilter(codecs); CommitPoint currentCommitPoint = null; String[] files = null; @@ -162,7 +167,7 @@ final class IndexFileDeleter { for (String fileName : files) { - if ((IndexFileNameFilter.INSTANCE.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) { + if ((indexFilenameFilter.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: getRefCount(fileName); @@ -175,9 +180,9 @@ final class IndexFileDeleter { if (infoStream != null) { message("init: load commit \"" + fileName + "\""); } - SegmentInfos sis = new SegmentInfos(); + SegmentInfos sis = new SegmentInfos(codecs); try { - sis.read(directory, fileName); + sis.read(directory, fileName, codecs); } catch (FileNotFoundException e) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth @@ -248,9 +253,9 @@ final class IndexFileDeleter { // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: - SegmentInfos sis = new SegmentInfos(); + SegmentInfos sis = new SegmentInfos(codecs); try { - sis.read(directory, currentSegmentsFile); + sis.read(directory, currentSegmentsFile, codecs); } catch (IOException e) { throw new CorruptIndexException("failed to locate current segments_N file"); } @@ -368,7 +373,7 @@ final class IndexFileDeleter { for(int i=0;i extensions; + + public IndexFileNameFilter(CodecProvider codecs) { + extensions = new HashSet(); + for (String ext : IndexFileNames.INDEX_EXTENSIONS) { + extensions.add(ext); + } + if (codecs != null) { + for(String ext : codecs.getAllExtensions()) { + extensions.add(ext); + } + } + } + + /* (non-Javadoc) + * @see java.io.FilenameFilter#accept(java.io.File, java.lang.String) + */ + public boolean accept(File dir, String name) { + int i = name.lastIndexOf('.'); + if (i != -1) { + String extension = name.substring(1+i); + if (extensions.contains(extension)) { + return true; + } else if (extension.startsWith("f") && + extension.matches("f\\d+")) { + return true; + } else if (extension.startsWith("s") && + extension.matches("s\\d+")) { + return true; + } } else { - return super.postingsFormat(); + if (name.startsWith(IndexFileNames.SEGMENTS)) return true; } + return false; } } diff --git a/lucene/src/java/org/apache/lucene/index/IndexFileNames.java b/lucene/src/java/org/apache/lucene/index/IndexFileNames.java index 0b3719e..e8d4ef6 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexFileNames.java +++ b/lucene/src/java/org/apache/lucene/index/IndexFileNames.java @@ -19,9 +19,7 @@ package org.apache.lucene.index; import java.util.regex.Pattern; -import org.apache.lucene.index.codecs.PostingsFormat; // for javadocs - -// TODO: put all files under codec and remove all the static extensions here +import org.apache.lucene.index.codecs.Codec; // for javadocs /** * This class contains useful constants representing filenames and extensions @@ -33,7 +31,7 @@ import org.apache.lucene.index.codecs.PostingsFormat; // for javadocs * {@link #segmentFileName(String, String, String) segmentFileName}). * *

NOTE: extensions used by codecs are not - * listed here. You must interact with the {@link PostingsFormat} + * listed here. You must interact with the {@link Codec} * directly. * * @lucene.internal @@ -190,20 +188,20 @@ public final class IndexFileNames { * NOTE: .<ext> is added to the result file name only if * ext is not empty. *

- * NOTE: _<segmentSuffix> is added to the result file name only if - * it's not the empty string + * NOTE: _<name> is added to the result file name only if + * name is not empty. *

* NOTE: all custom files should be named using this method, or * otherwise some structures may fail to handle them properly (such as if they * are added to compound files). */ - public static String segmentFileName(String segmentName, String segmentSuffix, String ext) { - if (ext.length() > 0 || segmentSuffix.length() > 0) { + public static String segmentFileName(String segmentName, String name, String ext) { + if (ext.length() > 0 || name.length() > 0) { assert !ext.startsWith("."); - StringBuilder sb = new StringBuilder(segmentName.length() + 2 + segmentSuffix.length() + ext.length()); + StringBuilder sb = new StringBuilder(segmentName.length() + 2 + name.length() + ext.length()); sb.append(segmentName); - if (segmentSuffix.length() > 0) { - sb.append('_').append(segmentSuffix); + if (name.length() > 0) { + sb.append('_').append(name); } if (ext.length() > 0) { sb.append('.').append(ext); @@ -214,6 +212,11 @@ public final class IndexFileNames { } } + /** Sugar for passing "" + name instead */ + public static String segmentFileName(String segmentName, int name, String ext) { + return segmentFileName(segmentName, ""+name, ext); + } + /** * Returns true if the given filename ends with the given extension. One * should provide a pure extension, without '.'. diff --git a/lucene/src/java/org/apache/lucene/index/IndexReader.java b/lucene/src/java/org/apache/lucene/index/IndexReader.java index ddd82b5..0961853 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/IndexReader.java @@ -28,7 +28,8 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.document.Document; import org.apache.lucene.document.DocumentStoredFieldVisitor; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.values.IndexDocValues; import org.apache.lucene.search.FieldCache; // javadocs @@ -311,7 +312,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory) throws CorruptIndexException, IOException { - return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR); + return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Returns an IndexReader reading the index in the given @@ -325,9 +326,9 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); } - + /** * Open a near real time IndexReader from the {@link org.apache.lucene.index.IndexWriter}. * @@ -362,7 +363,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Expert: returns an IndexReader reading the index in @@ -380,7 +381,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Expert: returns an IndexReader reading the index in @@ -408,7 +409,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor); + return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, null); } /** Expert: returns an IndexReader reading the index in @@ -428,7 +429,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); } /** Expert: returns an IndexReader reading the index in @@ -461,11 +462,78 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor); + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, null); + } + + /** Expert: returns an IndexReader reading the index in + * the given Directory, with a custom {@link + * IndexDeletionPolicy}, and specified {@link CodecProvider}. + * You should pass readOnly=true, since it gives much + * better concurrent performance, unless you intend to do + * write operations (delete documents or change norms) + * with the reader. + * @param directory the index directory + * @param deletionPolicy a custom deletion policy (only used + * if you use this reader to perform deletes or to set + * norms); see {@link IndexWriter} for details. + * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader + * @param termInfosIndexDivisor Subsamples which indexed + * terms are loaded into RAM. This has the same effect as {@link + * IndexWriterConfig#setTermIndexInterval} except that setting + * must be done at indexing time while this setting can be + * set per reader. When set to N, then one in every + * N*termIndexInterval terms in the index is loaded into + * memory. By setting this to a value > 1 you can reduce + * memory usage, at the expense of higher latency when + * loading a TermInfo. The default value is 1. Set this + * to -1 to skip loading the terms index entirely. + * @param codecs CodecProvider to use when opening index + * @throws CorruptIndexException if the index is corrupt + * @throws IOException if there is a low-level IO error + */ + public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException { + return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, codecs); } - private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor); + /** Expert: returns an IndexReader reading the index in + * the given Directory, using a specific commit and with + * a custom {@link IndexDeletionPolicy} and specified + * {@link CodecProvider}. You should pass readOnly=true, since + * it gives much better concurrent performance, unless + * you intend to do write operations (delete documents or + * change norms) with the reader. + + * @param commit the specific {@link IndexCommit} to open; + * see {@link IndexReader#listCommits} to list all commits + * in a directory + * @param deletionPolicy a custom deletion policy (only used + * if you use this reader to perform deletes or to set + * norms); see {@link IndexWriter} for details. + * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader + * @param termInfosIndexDivisor Subsamples which indexed + * terms are loaded into RAM. This has the same effect as {@link + * IndexWriterConfig#setTermIndexInterval} except that setting + * must be done at indexing time while this setting can be + * set per reader. When set to N, then one in every + * N*termIndexInterval terms in the index is loaded into + * memory. By setting this to a value > 1 you can reduce + * memory usage, at the expense of higher latency when + * loading a TermInfo. The default value is 1. Set this + * to -1 to skip loading the terms index entirely. + * @param codecs CodecProvider to use when opening index + * @throws CorruptIndexException if the index is corrupt + * @throws IOException if there is a low-level IO error + */ + public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException { + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs); + } + + private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor, + CodecProvider codecs) throws CorruptIndexException, IOException { + if (codecs == null) { + codecs = CodecProvider.getDefault(); + } + return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs); } /** @@ -699,7 +767,7 @@ public abstract class IndexReader implements Cloneable,Closeable { } }.run()).longValue(); } - + /** * Reads version number from segments files. The version number is * initialized with a timestamp and then increased by one for each change of @@ -711,10 +779,25 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static long getCurrentVersion(Directory directory) throws CorruptIndexException, IOException { - return SegmentInfos.readCurrentVersion(directory); + return getCurrentVersion(directory, CodecProvider.getDefault()); } /** + * Reads version number from segments files. The version number is + * initialized with a timestamp and then increased by one for each change of + * the index. + * + * @param directory where the index resides. + * @param codecs the {@link CodecProvider} holding all {@link Codec}s required to open the index + * @return version number. + * @throws CorruptIndexException if the index is corrupt + * @throws IOException if there is a low-level IO error + */ + public static long getCurrentVersion(Directory directory, CodecProvider codecs) throws CorruptIndexException, IOException { + return SegmentInfos.readCurrentVersion(directory, codecs); + } + + /** * Reads commitUserData, previously passed to {@link * IndexWriter#commit(Map)}, from current index * segments file. This will return null if {@link @@ -728,8 +811,28 @@ public abstract class IndexReader implements Cloneable,Closeable { * * @see #getCommitUserData() */ - public static Map getCommitUserData(Directory directory) throws CorruptIndexException, IOException { - return SegmentInfos.readCurrentUserData(directory); + public static Map getCommitUserData(Directory directory) throws CorruptIndexException, IOException { + return getCommitUserData(directory, CodecProvider.getDefault()); + } + + + /** + * Reads commitUserData, previously passed to {@link + * IndexWriter#commit(Map)}, from current index + * segments file. This will return null if {@link + * IndexWriter#commit(Map)} has never been called for + * this index. + * + * @param directory where the index resides. + * @param codecs the {@link CodecProvider} provider holding all {@link Codec}s required to open the index + * @return commit userData. + * @throws CorruptIndexException if the index is corrupt + * @throws IOException if there is a low-level IO error + * + * @see #getCommitUserData() + */ + public static Map getCommitUserData(Directory directory, CodecProvider codecs) throws CorruptIndexException, IOException { + return SegmentInfos.readCurrentUserData(directory, codecs); } /** @@ -882,6 +985,22 @@ public abstract class IndexReader implements Cloneable,Closeable { } } + /** + * Returns true if an index exists at the specified directory. + * @param directory the directory to check for an index + * @param codecProvider provides a CodecProvider in case the index uses non-core codecs + * @return true if an index exists; false otherwise + * @throws IOException if there is a problem with accessing the index + */ + public static boolean indexExists(Directory directory, CodecProvider codecProvider) throws IOException { + try { + new SegmentInfos().read(directory, codecProvider); + return true; + } catch (IOException ioe) { + return false; + } + } + /** Returns the number of documents in this index. */ public abstract int numDocs(); diff --git a/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java b/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java index 34d5348..87a656a 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java +++ b/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java @@ -114,12 +114,12 @@ public final class IndexUpgrader { } public void upgrade() throws IOException { - if (!IndexReader.indexExists(dir)) { + if (!IndexReader.indexExists(dir, iwc.getCodecProvider())) { throw new IndexNotFoundException(dir.toString()); } if (!deletePriorCommits) { - final Collection commits = DirectoryReader.listCommits(dir); + final Collection commits = DirectoryReader.listCommits(dir, iwc.getCodecProvider()); if (commits.size() > 1) { throw new IllegalArgumentException("This tool was invoked to not delete prior commit points, but the following commits were found: " + commits); } diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index 56c4571..0f5b284 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -39,7 +39,8 @@ import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.Query; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.CompoundFileDirectory; @@ -374,7 +375,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // just like we do when loading segments_N synchronized(this) { maybeApplyDeletes(applyAllDeletes); - r = new DirectoryReader(this, segmentInfos, applyAllDeletes); + r = new DirectoryReader(this, segmentInfos, codecs, applyAllDeletes); if (infoStream != null) { message("return reader version=" + r.getVersion() + " reader=" + r); } @@ -801,7 +802,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { infoStream.println("IW " + messageID + " [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message); } - final Codec codec; // for writing new segments + CodecProvider codecs; /** * Constructs a new IndexWriter per the settings given in conf. @@ -836,7 +837,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { mergePolicy = conf.getMergePolicy(); mergePolicy.setIndexWriter(this); mergeScheduler = conf.getMergeScheduler(); - codec = conf.getCodec(); + codecs = conf.getCodecProvider(); bufferedDeletesStream = new BufferedDeletesStream(messageID); bufferedDeletesStream.setInfoStream(infoStream); @@ -861,7 +862,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // If index is too old, reading the segments will throw // IndexFormatTooOldException. - segmentInfos = new SegmentInfos(); + segmentInfos = new SegmentInfos(codecs); try { if (create) { // Try to read first. This is to allow create @@ -869,7 +870,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // searching. In this case we write the next // segments_N file with no segments: try { - segmentInfos.read(directory); + segmentInfos.read(directory, codecs); segmentInfos.clear(); } catch (IOException e) { // Likely this means it's a fresh directory @@ -880,7 +881,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { changeCount++; segmentInfos.changed(); } else { - segmentInfos.read(directory); + segmentInfos.read(directory, codecs); IndexCommit commit = conf.getIndexCommit(); if (commit != null) { @@ -891,8 +892,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // points. if (commit.getDirectory() != directory) throw new IllegalArgumentException("IndexCommit's directory doesn't match my directory"); - SegmentInfos oldInfos = new SegmentInfos(); - oldInfos.read(directory, commit.getSegmentsFileName()); + SegmentInfos oldInfos = new SegmentInfos(codecs); + oldInfos.read(directory, commit.getSegmentsFileName(), codecs); segmentInfos.replace(oldInfos); changeCount++; segmentInfos.changed(); @@ -905,7 +906,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // start with previous field numbers, but new FieldInfos globalFieldNumberMap = segmentInfos.getOrLoadGlobalFieldNumberMap(directory); - docWriter = new DocumentsWriter(codec, config, directory, this, globalFieldNumberMap, bufferedDeletesStream); + docWriter = new DocumentsWriter(config, directory, this, globalFieldNumberMap, bufferedDeletesStream); docWriter.setInfoStream(infoStream); // Default deleter (for backwards compatibility) is @@ -913,7 +914,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { synchronized(this) { deleter = new IndexFileDeleter(directory, conf.getIndexDeletionPolicy(), - segmentInfos, infoStream, this); + segmentInfos, infoStream, + codecs, this); } if (deleter.startingCommitDeleted) { @@ -2147,7 +2149,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { */ public synchronized void deleteAll() throws IOException { ensureOpen(); - boolean success = false; try { // Abort any running merges @@ -2169,11 +2170,10 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Mark that the index has changed ++changeCount; segmentInfos.changed(); - success = true; } catch (OutOfMemoryError oom) { handleOOM(oom, "deleteAll"); } finally { - if (!success && infoStream != null) { + if (infoStream != null) { message("hit exception during deleteAll"); } } @@ -2476,8 +2476,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { if (infoStream != null) { message("addIndexes: process directory " + dir); } - SegmentInfos sis = new SegmentInfos(); // read infos from dir - sis.read(dir); + SegmentInfos sis = new SegmentInfos(codecs); // read infos from dir + sis.read(dir, codecs); final Set dsFilesCopied = new HashSet(); final Map dsNames = new HashMap(); for (SegmentInfo info : sis) { @@ -2567,7 +2567,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // abortable so that IW.close(false) is able to stop it SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, null, payloadProcessorProvider, - new FieldInfos(globalFieldNumberMap), codec, context); + globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)), context); for (IndexReader reader : readers) // add new indexes merger.add(reader); @@ -2575,7 +2575,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { final FieldInfos fieldInfos = merger.fieldInfos(); SegmentInfo info = new SegmentInfo(mergedName, docCount, directory, - false, merger.getCodec(), + false, merger.getSegmentCodecs(), fieldInfos); setDiagnostics(info, "addIndexes(IndexReader...)"); @@ -2591,7 +2591,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Now create the compound file if needed if (useCompoundFile) { - merger.createCompoundFile(IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_EXTENSION), info, context); + merger.createCompoundFile(mergedName + ".cfs", info, context); // delete new non cfs files directly: they were never // registered with IFD @@ -2916,7 +2916,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { try { if (infoStream != null) message("commit: pendingCommit != null"); - pendingCommit.finishCommit(directory, codec); + pendingCommit.finishCommit(directory); if (infoStream != null) message("commit: wrote segments file \"" + pendingCommit.getCurrentSegmentFileName() + "\""); lastCommitChangeCount = pendingCommitChangeCount; @@ -3459,7 +3459,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Bind a new segment name here so even with // ConcurrentMergePolicy we keep deterministic segment // names. - merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, new FieldInfos(globalFieldNumberMap)); + merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs))); // Lock order: IW -> BD final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments); @@ -3633,7 +3633,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { IOContext context = new IOContext(merge.getMergeInfo()); SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge, - payloadProcessorProvider, merge.info.getFieldInfos(), codec, context); + payloadProcessorProvider, merge.info.getFieldInfos(), context); if (infoStream != null) { message("merging " + merge.segString(directory) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors()); @@ -3679,10 +3679,10 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { mergedDocCount = merge.info.docCount = merger.merge(); // Record which codec was used to write the segment - merge.info.setCodec(merger.getCodec()); + merge.info.setSegmentCodecs(merger.getSegmentCodecs()); if (infoStream != null) { - message("merge codecs=" + merger.getCodec()); + message("merge segmentCodecs=" + merger.getSegmentCodecs()); message("merge store matchedCount=" + merger.getMatchedSubReaderCount() + " vs " + merge.readers.size()); } anyNonBulkMerges |= merger.getAnyNonBulkMerges(); @@ -3975,7 +3975,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Exception here means nothing is prepared // (this method unwinds everything it did on // an exception) - toSync.prepareCommit(directory, codec); + toSync.prepareCommit(directory); pendingCommitSet = true; pendingCommit = toSync; diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java b/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java index af56141..326a1d8 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java @@ -20,7 +20,7 @@ package org.apache.lucene.index; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.util.Version; @@ -121,7 +121,7 @@ public final class IndexWriterConfig implements Cloneable { private volatile int maxBufferedDocs; private volatile IndexingChain indexingChain; private volatile IndexReaderWarmer mergedSegmentWarmer; - private volatile Codec codec; + private volatile CodecProvider codecProvider; private volatile MergePolicy mergePolicy; private volatile DocumentsWriterPerThreadPool indexerThreadPool; private volatile boolean readerPooling; @@ -158,7 +158,7 @@ public final class IndexWriterConfig implements Cloneable { maxBufferedDocs = DEFAULT_MAX_BUFFERED_DOCS; indexingChain = DocumentsWriterPerThread.defaultIndexingChain; mergedSegmentWarmer = null; - codec = Codec.getDefault(); + codecProvider = CodecProvider.getDefault(); if (matchVersion.onOrAfter(Version.LUCENE_32)) { mergePolicy = new TieredMergePolicy(); } else { @@ -521,17 +521,17 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** Set the Codec. See {@link Codec}. + /** Set the CodecProvider. See {@link CodecProvider}. * *

Only takes effect when IndexWriter is first created. */ - public IndexWriterConfig setCodec(Codec codec) { - this.codec = codec; + public IndexWriterConfig setCodecProvider(CodecProvider codecProvider) { + this.codecProvider = codecProvider; return this; } - /** Returns the current Codec. See {@link Codec}. */ - public Codec getCodec() { - return codec; + /** Returns the current merged segment warmer. See {@link IndexReaderWarmer}. */ + public CodecProvider getCodecProvider() { + return codecProvider; } @@ -694,7 +694,7 @@ public final class IndexWriterConfig implements Cloneable { sb.append("ramBufferSizeMB=").append(ramBufferSizeMB).append("\n"); sb.append("maxBufferedDocs=").append(maxBufferedDocs).append("\n"); sb.append("mergedSegmentWarmer=").append(mergedSegmentWarmer).append("\n"); - sb.append("codec=").append(codec).append("\n"); + sb.append("codecProvider=").append(codecProvider).append("\n"); sb.append("mergePolicy=").append(mergePolicy).append("\n"); sb.append("indexerThreadPool=").append(indexerThreadPool).append("\n"); sb.append("readerPooling=").append(readerPooling).append("\n"); diff --git a/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java b/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java index 8821bee..d5c1abb 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java +++ b/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java @@ -19,6 +19,7 @@ package org.apache.lucene.index; import org.apache.lucene.index.FieldInfo.IndexOptions; +/** @lucene.experimental */ public interface IndexableFieldType { /** True if this field should be indexed (inverted) */ diff --git a/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java b/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java index cfa1ab0..84574e6 100644 --- a/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java +++ b/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java @@ -34,37 +34,41 @@ public class PerDocWriteState { public final String segmentName; public final FieldInfos fieldInfos; public final Counter bytesUsed; - public final String segmentSuffix; + public final SegmentCodecs segmentCodecs; + public final int codecId; public final IOContext context; - public PerDocWriteState(PrintStream infoStream, Directory directory, + PerDocWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos, Counter bytesUsed, - String segmentSuffix, IOContext context) { + int codecId, IOContext context) { this.infoStream = infoStream; this.directory = directory; this.segmentName = segmentName; this.fieldInfos = fieldInfos; - this.segmentSuffix = segmentSuffix; + this.segmentCodecs = fieldInfos.buildSegmentCodecs(false); + this.codecId = codecId; this.bytesUsed = bytesUsed; this.context = context; } - public PerDocWriteState(SegmentWriteState state) { + PerDocWriteState(SegmentWriteState state) { infoStream = state.infoStream; directory = state.directory; + segmentCodecs = state.segmentCodecs; segmentName = state.segmentName; fieldInfos = state.fieldInfos; - segmentSuffix = state.segmentSuffix; + codecId = state.codecId; bytesUsed = Counter.newCounter(); context = state.context; } - public PerDocWriteState(PerDocWriteState state, String segmentSuffix) { + PerDocWriteState(PerDocWriteState state, int codecId) { this.infoStream = state.infoStream; this.directory = state.directory; this.segmentName = state.segmentName; this.fieldInfos = state.fieldInfos; - this.segmentSuffix = segmentSuffix; + this.segmentCodecs = state.segmentCodecs; + this.codecId = codecId; this.bytesUsed = state.bytesUsed; this.context = state.context; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java b/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java similarity index 26% rename from lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java rename to lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java index 98bb641..9e950d1 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java +++ b/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.perfield; +package org.apache.lucene.index; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,51 +17,43 @@ package org.apache.lucene.index.codecs.perfield; * limitations under the License. */ -import java.io.Closeable; -import java.io.FileNotFoundException; import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; -import java.util.IdentityHashMap; import java.util.Iterator; import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldsEnum; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.TermsConsumer; +import org.apache.lucene.index.codecs.DocValuesConsumer; +import org.apache.lucene.index.values.IndexDocValues; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; /** - * Enables per field format support. + * Enables native per field codec support. This class selects the codec used to + * write a field depending on the provided {@link SegmentCodecs}. For each field + * seen it resolves the codec based on the {@link FieldInfo#codecId} which is + * only valid during a segment merge. See {@link SegmentCodecs} javadoc for + * details. * - * @lucene.experimental + * @lucene.internal */ +final class PerFieldCodecWrapper extends Codec { + private final SegmentCodecs segmentCodecs; -public abstract class PerFieldPostingsFormat extends PostingsFormat { - - public static final String PER_FIELD_EXTENSION = "per"; - public static final String PER_FIELD_NAME = "PerField40"; - - public static final int VERSION_START = 0; - public static final int VERSION_LATEST = VERSION_START; - - public PerFieldPostingsFormat() { - super(PER_FIELD_NAME); + PerFieldCodecWrapper(SegmentCodecs segmentCodecs) { + super("PerField"); + this.segmentCodecs = segmentCodecs; } @Override @@ -70,146 +62,79 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { return new FieldsWriter(state); } - // NOTE: not private to avoid $accessN at runtime!! - static class FieldsConsumerAndID implements Closeable { - final FieldsConsumer fieldsConsumer; - final String segmentSuffix; - - public FieldsConsumerAndID(FieldsConsumer fieldsConsumer, String segmentSuffix) { - this.fieldsConsumer = fieldsConsumer; - this.segmentSuffix = segmentSuffix; - } - - @Override - public void close() throws IOException { - fieldsConsumer.close(); - } - }; - private class FieldsWriter extends FieldsConsumer { - - private final Map formats = new IdentityHashMap(); - - /** Records all fields we wrote. */ - private final Map fieldToFormat = new HashMap(); - - private final SegmentWriteState segmentWriteState; + private final ArrayList consumers = new ArrayList(); public FieldsWriter(SegmentWriteState state) throws IOException { - segmentWriteState = state; + assert segmentCodecs == state.segmentCodecs; + final Codec[] codecs = segmentCodecs.codecs; + for (int i = 0; i < codecs.length; i++) { + boolean success = false; + try { + consumers.add(codecs[i].fieldsConsumer(new SegmentWriteState(state, i))); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(consumers); + } + } + } } @Override public TermsConsumer addField(FieldInfo field) throws IOException { - final PostingsFormat format = getPostingsFormatForField(field.name); - if (format == null) { - throw new IllegalStateException("invalid null PostingsFormat for field=\"" + field.name + "\""); - } - - assert !fieldToFormat.containsKey(field.name); - fieldToFormat.put(field.name, format); - - FieldsConsumerAndID consumerAndId = formats.get(format); - if (consumerAndId == null) { - // First time we are seeing this format; assign - // next id and init it: - final String segmentSuffix = getFullSegmentSuffix(field.name, - segmentWriteState.segmentSuffix, - ""+formats.size()); - consumerAndId = new FieldsConsumerAndID(format.fieldsConsumer(new SegmentWriteState(segmentWriteState, segmentSuffix)), - segmentSuffix); - formats.put(format, consumerAndId); - } - - return consumerAndId.fieldsConsumer.addField(field); + assert field.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID; + final FieldsConsumer fields = consumers.get(field.getCodecId()); + return fields.addField(field); } @Override public void close() throws IOException { - - // Close all subs - IOUtils.close(formats.values()); - - // Write _X.per: maps field name -> format name and - // format name -> format id - final String mapFileName = IndexFileNames.segmentFileName(segmentWriteState.segmentName, segmentWriteState.segmentSuffix, PER_FIELD_EXTENSION); - final IndexOutput out = segmentWriteState.directory.createOutput(mapFileName, segmentWriteState.context); - boolean success = false; - try { - CodecUtil.writeHeader(out, PER_FIELD_NAME, VERSION_LATEST); - - // format name -> int id - out.writeVInt(formats.size()); - for(Map.Entry ent : formats.entrySet()) { - out.writeString(ent.getValue().segmentSuffix); - out.writeString(ent.getKey().getName()); - } - - // field name -> format name - out.writeVInt(fieldToFormat.size()); - for(Map.Entry ent : fieldToFormat.entrySet()) { - out.writeString(ent.getKey()); - out.writeString(ent.getValue().getName()); - } - - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(out); - } else { - IOUtils.close(out); - } - } - } - } - - static String getFullSegmentSuffix(String fieldName, String outerSegmentSuffix, String segmentSuffix) { - if (outerSegmentSuffix.length() == 0) { - return segmentSuffix; - } else { - // TODO: support embedding; I think it should work but - // we need a test confirm to confirm - // return outerSegmentSuffix + "_" + segmentSuffix; - throw new IllegalStateException("cannot embed PerFieldPostingsFormat inside itself (field \"" + fieldName + "\" returned PerFieldPostingsFormat)"); + IOUtils.close(consumers); } } private class FieldsReader extends FieldsProducer { - private final Map fields = new TreeMap(); - private final Map formats = new IdentityHashMap(); + private final Set fields = new TreeSet(); + private final Map codecs = new HashMap(); - public FieldsReader(final SegmentReadState readState) throws IOException { + public FieldsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo si, + IOContext context, int indexDivisor) throws IOException { - // Read _X.per and init each format: + final Map producers = new HashMap(); boolean success = false; try { - new VisitPerFieldFile(readState.dir, readState.segmentInfo.name, readState.segmentSuffix) { - @Override - protected void visitOneFormat(String segmentSuffix, PostingsFormat postingsFormat) throws IOException { - formats.put(postingsFormat, postingsFormat.fieldsProducer(new SegmentReadState(readState, segmentSuffix))); + for (FieldInfo fi : fieldInfos) { + if (fi.isIndexed) { + fields.add(fi.name); + assert fi.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID; + Codec codec = segmentCodecs.codecs[fi.getCodecId()]; + if (!producers.containsKey(codec)) { + producers.put(codec, codec.fieldsProducer(new SegmentReadState(dir, + si, fieldInfos, context, indexDivisor, fi.getCodecId()))); + } + codecs.put(fi.name, producers.get(codec)); } - - @Override - protected void visitOneField(String fieldName, PostingsFormat postingsFormat) throws IOException { - assert formats.containsKey(postingsFormat); - fields.put(fieldName, formats.get(postingsFormat)); - } - }; + } success = true; } finally { if (!success) { - IOUtils.closeWhileHandlingException(formats.values()); + // If we hit exception (eg, IOE because writer was + // committing, or, for any other reason) we must + // go back and close all FieldsProducers we opened: + IOUtils.closeWhileHandlingException(producers.values()); } } } + private final class FieldsIterator extends FieldsEnum { private final Iterator it; private String current; public FieldsIterator() { - it = fields.keySet().iterator(); + it = fields.iterator(); } @Override @@ -225,7 +150,7 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { @Override public TermsEnum terms() throws IOException { - final Terms terms = fields.get(current).terms(current); + Terms terms = codecs.get(current).terms(current); if (terms != null) { return terms.iterator(); } else { @@ -241,101 +166,121 @@ public abstract class PerFieldPostingsFormat extends PostingsFormat { @Override public Terms terms(String field) throws IOException { - FieldsProducer fieldsProducer = fields.get(field); - return fieldsProducer == null ? null : fieldsProducer.terms(field); + FieldsProducer fields = codecs.get(field); + return fields == null ? null : fields.terms(field); } @Override public void close() throws IOException { - IOUtils.close(formats.values()); + IOUtils.close(codecs.values()); } } @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - return new FieldsReader(state); + return new FieldsReader(state.dir, state.fieldInfos, state.segmentInfo, + state.context, state.termsIndexDivisor); } - private abstract class VisitPerFieldFile { - public VisitPerFieldFile(Directory dir, String segmentName, String outerSegmentSuffix) throws IOException { - final String mapFileName = IndexFileNames.segmentFileName(segmentName, outerSegmentSuffix, PER_FIELD_EXTENSION); - final IndexInput in = dir.openInput(mapFileName, IOContext.READONCE); - boolean success = false; - try { - CodecUtil.checkHeader(in, PER_FIELD_NAME, VERSION_START, VERSION_LATEST); + @Override + public void files(Directory dir, SegmentInfo info, int codecId, Set files) + throws IOException { + // ignore codecid since segmentCodec will assign it per codec + segmentCodecs.files(dir, info, files); + } - // Read format name -> format id - final int formatCount = in.readVInt(); - for(int formatIDX=0;formatIDX " + postingsFormat); - if (postingsFormat == null) { - throw new IllegalStateException("unable to lookup PostingsFormat for name=\"" + formatName + "\": got null"); - } + @Override + public void getExtensions(Set extensions) { + for (Codec codec : segmentCodecs.codecs) { + codec.getExtensions(extensions); + } + } - // Better be defined, because it was defined - // during indexing: - visitOneFormat(segmentSuffix, postingsFormat); - } + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new PerDocConsumers(state); + } - // Read field name -> format name - final int fieldCount = in.readVInt(); - for(int fieldIDX=0;fieldIDX codecs = new TreeMap(); + public PerDocProducers(Directory dir, FieldInfos fieldInfos, SegmentInfo si, + IOContext context, int indexDivisor) throws IOException { + final Map producers = new HashMap(); + boolean success = false; + try { + for (FieldInfo fi : fieldInfos) { + if (fi.hasDocValues()) { + assert fi.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID; + Codec codec = segmentCodecs.codecs[fi.getCodecId()]; + if (!producers.containsKey(codec)) { + producers.put(codec, codec.docsProducer(new SegmentReadState(dir, + si, fieldInfos, context, indexDivisor, fi.getCodecId()))); + } + codecs.put(fi.name, producers.get(codec)); + } + } success = true; } finally { if (!success) { - IOUtils.closeWhileHandlingException(in); - } else { - IOUtils.close(in); + IOUtils.closeWhileHandlingException(producers.values()); } } } - - // This is called first, for all formats: - protected abstract void visitOneFormat(String segmentSuffix, PostingsFormat format) throws IOException; - - // ... then this is called, for all fields: - protected abstract void visitOneField(String fieldName, PostingsFormat format) throws IOException; + + @Override + public Collection fields() { + return codecs.keySet(); + } + @Override + public IndexDocValues docValues(String field) throws IOException { + final PerDocValues perDocProducer = codecs.get(field); + if (perDocProducer == null) { + return null; + } + return perDocProducer.docValues(field); + } + + public void close() throws IOException { + IOUtils.close(codecs.values()); + } } + + private final class PerDocConsumers extends PerDocConsumer { + private final PerDocConsumer[] consumers; + private final Codec[] codecs; + private final PerDocWriteState state; + + public PerDocConsumers(PerDocWriteState state) throws IOException { + assert segmentCodecs == state.segmentCodecs; + this.state = state; + codecs = segmentCodecs.codecs; + consumers = new PerDocConsumer[codecs.length]; + } - @Override - public void files(final Directory dir, final SegmentInfo info, String segmentSuffix, final Set files) - throws IOException { - - final String mapFileName = IndexFileNames.segmentFileName(info.name, segmentSuffix, PER_FIELD_EXTENSION); - files.add(mapFileName); - - try { - new VisitPerFieldFile(dir, info.name, segmentSuffix) { - @Override - protected void visitOneFormat(String segmentSuffix, PostingsFormat format) throws IOException { - format.files(dir, info, segmentSuffix, files); - } + public void close() throws IOException { + IOUtils.close(consumers); + } - @Override - protected void visitOneField(String field, PostingsFormat format) { - } - }; - } catch (FileNotFoundException fnfe) { - // TODO: this is somewhat shady... if we can't open - // the .per file then most likely someone is calling - // .files() after this segment was deleted, so, they - // wouldn't be able to do anything with the files even - // if we could return them, so we don't add any files - // in this case. + @Override + public DocValuesConsumer addValuesField(FieldInfo field) throws IOException { + final int codecId = field.getCodecId(); + assert codecId != FieldInfo.UNASSIGNED_CODEC_ID; + PerDocConsumer perDoc = consumers[codecId]; + if (perDoc == null) { + perDoc = codecs[codecId].docsConsumer(new PerDocWriteState(state, codecId)); + assert perDoc != null; + consumers[codecId] = perDoc; + } + return perDoc.addValuesField(field); } + } - - // NOTE: only called during writing; for reading we read - // all we need from the index (ie we save the field -> - // format mapping) - public abstract PostingsFormat getPostingsFormatForField(String field); } diff --git a/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java b/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java new file mode 100644 index 0000000..68c4aa3 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java @@ -0,0 +1,167 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.IdentityHashMap; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.preflex.PreFlexCodec; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; + +/** + * SegmentCodecs maintains an ordered list of distinct codecs used within a + * segment. Within a segment on codec is used to write multiple fields while + * each field could be written by a different codec. To enable codecs per field + * within a single segment we need to record the distinct codecs and map them to + * each field present in the segment. SegmentCodecs is created together with + * {@link SegmentWriteState} for each flush and is maintained in the + * corresponding {@link SegmentInfo} until it is committed. + *

+ * During indexing {@link FieldInfos} uses {@link SegmentCodecsBuilder} to incrementally + * build the {@link SegmentCodecs} mapping. Once a segment is flushed + * DocumentsWriter creates a {@link SegmentCodecs} instance from + * {@link FieldInfos#buildSegmentCodecs(boolean)} The {@link FieldInfo#codecId} + * assigned by {@link SegmentCodecsBuilder} refers to the codecs ordinal + * maintained inside {@link SegmentCodecs}. This ord is later used to get the + * right codec when the segment is opened in a reader.The {@link Codec} returned + * from {@link SegmentCodecs#codec()} in turn uses {@link SegmentCodecs} + * internal structure to select and initialize the right codec for a fields when + * it is written. + *

+ * Once a flush succeeded the {@link SegmentCodecs} is maintained inside the + * {@link SegmentInfo} for the flushed segment it was created for. + * {@link SegmentInfo} writes the name of each codec in {@link SegmentCodecs} + * for each segment and maintains the order. Later if a segment is opened by a + * reader this mapping is deserialized and used to create the codec per field. + * + * + * @lucene.internal + */ +final class SegmentCodecs implements Cloneable { + /** + * internal structure to map codecs to fields - don't modify this from outside + * of this class! + */ + final Codec[] codecs; + final CodecProvider provider; + private final Codec codec; + + SegmentCodecs(CodecProvider provider, IndexInput input) throws IOException { + this(provider, read(input, provider)); + } + + SegmentCodecs(CodecProvider provider, Codec... codecs) { + this.provider = provider; + this.codecs = codecs; + if (codecs.length == 1 && codecs[0] instanceof PreFlexCodec) { + this.codec = codecs[0]; // hack for backwards break... don't wrap the codec in preflex + } else { + this.codec = new PerFieldCodecWrapper(this); + } + } + + Codec codec() { + return codec; + } + + void write(IndexOutput out) throws IOException { + out.writeVInt(codecs.length); + for (Codec codec : codecs) { + out.writeString(codec.name); + } + } + + private static Codec[] read(IndexInput in, CodecProvider provider) throws IOException { + final int size = in.readVInt(); + final ArrayList list = new ArrayList(); + for (int i = 0; i < size; i++) { + final String codecName = in.readString(); + final Codec lookup = provider.lookup(codecName); + list.add(i, lookup); + } + return list.toArray(Codec.EMPTY); + } + + void files(Directory dir, SegmentInfo info, Set files) + throws IOException { + final Codec[] codecArray = codecs; + for (int i = 0; i < codecArray.length; i++) { + codecArray[i].files(dir, info, i, files); + } + + } + + @Override + public String toString() { + return "SegmentCodecs [codecs=" + Arrays.toString(codecs) + ", provider=" + provider + "]"; + } + + /** + * Used in {@link FieldInfos} to incrementally build the codec ID mapping for + * {@link FieldInfo} instances. + *

+ * Note: this class is not thread-safe + *

+ * @see FieldInfo#getCodecId() + */ + final static class SegmentCodecsBuilder { + private final Map codecRegistry = new IdentityHashMap(); + private final ArrayList codecs = new ArrayList(); + private final CodecProvider provider; + + private SegmentCodecsBuilder(CodecProvider provider) { + this.provider = provider; + } + + static SegmentCodecsBuilder create(CodecProvider provider) { + return new SegmentCodecsBuilder(provider); + } + + SegmentCodecsBuilder tryAddAndSet(FieldInfo fi) { + if (fi.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) { + final Codec fieldCodec = provider.lookup(provider + .getFieldCodec(fi.name)); + Integer ord = codecRegistry.get(fieldCodec); + if (ord == null) { + ord = Integer.valueOf(codecs.size()); + codecRegistry.put(fieldCodec, ord); + codecs.add(fieldCodec); + } + fi.setCodecId(ord.intValue()); + } + return this; + } + + SegmentCodecs build() { + return new SegmentCodecs(provider, codecs.toArray(Codec.EMPTY)); + } + + SegmentCodecsBuilder clear() { + codecRegistry.clear(); + codecs.clear(); + return this; + } + } +} \ No newline at end of file diff --git a/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java b/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java index f5f77c6..3e6e307 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java @@ -21,7 +21,6 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FieldsReader; import org.apache.lucene.index.codecs.PerDocValues; @@ -69,7 +68,7 @@ final class SegmentCoreReaders { } segment = si.name; - final Codec codec = si.getCodec(); + final SegmentCodecs segmentCodecs = si.getSegmentCodecs(); this.context = context; this.dir = dir; @@ -86,12 +85,12 @@ final class SegmentCoreReaders { fieldInfos = si.getFieldInfos(); this.termsIndexDivisor = termsIndexDivisor; - final PostingsFormat format = codec.postingsFormat(); + final Codec codec = segmentCodecs.codec(); final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, context, termsIndexDivisor); // Ask codec for its Fields - fields = format.fieldsProducer(segmentReadState); + fields = codec.fieldsProducer(segmentReadState); assert fields != null; - perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState); + perDocProducer = codec.docsProducer(segmentReadState); success = true; } finally { if (!success) { @@ -166,7 +165,7 @@ final class SegmentCoreReaders { } final String storesSegment = si.getDocStoreSegment(); - fieldsReaderOrig = si.getCodec().fieldsFormat().fieldsReader(storeDir, storesSegment, fieldInfos, context, + fieldsReaderOrig = si.getSegmentCodecs().provider.fieldsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount); // Verify two sources of "maxDoc" agree: diff --git a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java index e4bf7fc..005672e 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java @@ -28,6 +28,7 @@ import java.util.Map.Entry; import java.util.Set; import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; @@ -96,7 +97,7 @@ public final class SegmentInfo implements Cloneable { private FieldInfos fieldInfos; - private Codec codec; + private SegmentCodecs segmentCodecs; private Map diagnostics; @@ -115,7 +116,7 @@ public final class SegmentInfo implements Cloneable { private long fieldInfosVersion; public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, - Codec codec, FieldInfos fieldInfos) { + SegmentCodecs segmentCodecs, FieldInfos fieldInfos) { this.name = name; this.docCount = docCount; this.dir = dir; @@ -123,7 +124,7 @@ public final class SegmentInfo implements Cloneable { this.isCompoundFile = isCompoundFile; this.docStoreOffset = -1; this.docStoreSegment = name; - this.codec = codec; + this.segmentCodecs = segmentCodecs; delCount = 0; version = Constants.LUCENE_MAIN_VERSION; this.fieldInfos = fieldInfos; @@ -155,7 +156,7 @@ public final class SegmentInfo implements Cloneable { } isCompoundFile = src.isCompoundFile; delCount = src.delCount; - codec = src.codec; + segmentCodecs = src.segmentCodecs; } void setDiagnostics(Map diagnostics) { @@ -176,7 +177,7 @@ public final class SegmentInfo implements Cloneable { * @param format format of the segments info file * @param input input handle to read segment info from */ - public SegmentInfo(Directory dir, int format, IndexInput input) throws IOException { + public SegmentInfo(Directory dir, int format, IndexInput input, CodecProvider codecs) throws IOException { this.dir = dir; if (format <= DefaultSegmentInfosWriter.FORMAT_3_1) { version = input.readString(); @@ -220,13 +221,13 @@ public final class SegmentInfo implements Cloneable { hasProx = input.readByte(); - // System.out.println(Thread.currentThread().getName() + ": si.read hasProx=" + hasProx + " seg=" + name); - // note: if the codec is not available: Codec.forName will throw an exception. if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { - codec = Codec.forName(input.readString()); + segmentCodecs = new SegmentCodecs(codecs, input); } else { - codec = Codec.forName("Lucene3x"); + // codec ID on FieldInfo is 0 so it will simply use the first codec available + // TODO what todo if preflex is not available in the provider? register it or fail? + segmentCodecs = new SegmentCodecs(codecs, new Codec[] { codecs.lookup("PreFlex")}); } diagnostics = input.readStringStringMap(); @@ -349,7 +350,7 @@ public final class SegmentInfo implements Cloneable { @Override public Object clone() { - final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, codec, + final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, segmentCodecs, fieldInfos == null ? null : (FieldInfos) fieldInfos.clone()); si.docStoreOffset = docStoreOffset; si.docStoreSegment = docStoreSegment; @@ -572,7 +573,7 @@ public final class SegmentInfo implements Cloneable { output.writeByte((byte) (isCompoundFile ? YES : NO)); output.writeInt(delCount); output.writeByte((byte) (hasProx)); - output.writeString(codec.getName()); + segmentCodecs.write(output); output.writeStringStringMap(diagnostics); output.writeByte((byte) (hasVectors)); } @@ -582,16 +583,16 @@ public final class SegmentInfo implements Cloneable { } /** Can only be called once. */ - public void setCodec(Codec codec) { - assert this.codec == null; - if (codec == null) { + public void setSegmentCodecs(SegmentCodecs segmentCodecs) { + assert this.segmentCodecs == null; + if (segmentCodecs == null) { throw new IllegalArgumentException("segmentCodecs must be non-null"); } - this.codec = codec; + this.segmentCodecs = segmentCodecs; } - Codec getCodec() { - return codec; + SegmentCodecs getSegmentCodecs() { + return segmentCodecs; } private void addIfExists(Set files, String fileName) throws IOException { @@ -627,7 +628,7 @@ public final class SegmentInfo implements Cloneable { for(String ext : IndexFileNames.NON_STORE_INDEX_EXTENSIONS) { addIfExists(fileSet, IndexFileNames.segmentFileName(name, "", ext)); } - codec.files(dir, this, fileSet); + segmentCodecs.files(dir, this, fileSet); } if (docStoreOffset != -1) { diff --git a/lucene/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/src/java/org/apache/lucene/index/SegmentInfos.java index 1040674..f9867b8 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentInfos.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentInfos.java @@ -32,11 +32,10 @@ import java.util.Map; import java.util.Set; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.index.codecs.SegmentInfosReader; import org.apache.lucene.index.codecs.SegmentInfosWriter; -import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -84,6 +83,8 @@ public final class SegmentInfos implements Cloneable, Iterable { // there was an IOException that had interrupted a commit public Map userData = Collections.emptyMap(); // Opaque Map that user can specify during IndexWriter.commit + + private CodecProvider codecs; private int format; @@ -94,14 +95,20 @@ public final class SegmentInfos implements Cloneable, Iterable { private transient List cachedUnmodifiableList; private transient Set cachedUnmodifiableSet; - private Codec codecFormat; - /** * If non-null, information about loading segments_N files * will be printed here. @see #setInfoStream. */ private static PrintStream infoStream = null; + public SegmentInfos() { + this(CodecProvider.getDefault()); + } + + public SegmentInfos(CodecProvider codecs) { + this.codecs = codecs; + } + public void setFormat(int format) { this.format = format; } @@ -190,7 +197,7 @@ public final class SegmentInfos implements Cloneable, Iterable { * since this file might belong to more than one segment (global map) and * could otherwise easily be confused with a per-segment file. */ - return IndexFileNames.segmentFileName("_"+ version, "", IndexFileNames.GLOBAL_FIELD_NUM_MAP_EXTENSION); + return IndexFileNames.segmentFileName(""+ version, "", IndexFileNames.GLOBAL_FIELD_NUM_MAP_EXTENSION); } /** @@ -234,7 +241,9 @@ public final class SegmentInfos implements Cloneable, Iterable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public final void read(Directory directory, String segmentFileName) throws CorruptIndexException, IOException { + public final void read(Directory directory, String segmentFileName, + CodecProvider codecs) throws CorruptIndexException, IOException { + this.codecs = codecs; boolean success = false; // Clear any previous segments: @@ -244,40 +253,12 @@ public final class SegmentInfos implements Cloneable, Iterable { lastGeneration = generation; - // TODO: scary to have default impl reopen the file... but to make it a bit more flexible, - // maybe we could use a plain indexinput here... could default impl rewind/wrap with checksumII, - // and any checksumming is then up to implementation? - ChecksumIndexInput input = null; try { - input = new ChecksumIndexInput(directory.openInput(segmentFileName, IOContext.READ)); - final int format = input.readInt(); - setFormat(format); - - // check that it is a format we can understand - if (format > DefaultSegmentInfosWriter.FORMAT_MINIMUM) - throw new IndexFormatTooOldException(segmentFileName, format, - DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); - if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT) - throw new IndexFormatTooNewException(segmentFileName, format, - DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); - - if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { - codecFormat = Codec.forName(input.readString()); - } else { - codecFormat = Codec.forName("Lucene3x"); - } - SegmentInfosReader infosReader = codecFormat.segmentInfosFormat().getSegmentInfosReader(); - infosReader.read(directory, segmentFileName, input, this, IOContext.READ); - final long checksumNow = input.getChecksum(); - final long checksumThen = input.readLong(); - if (checksumNow != checksumThen) - throw new CorruptIndexException("checksum mismatch in segments file"); + SegmentInfosReader infosReader = codecs.getSegmentInfosReader(); + infosReader.read(directory, segmentFileName, codecs, this, IOContext.READ); success = true; } finally { - if (input != null) { - input.close(); - } if (!success) { // Clear any segment infos we had loaded so we // have a clean slate on retry: @@ -286,14 +267,25 @@ public final class SegmentInfos implements Cloneable, Iterable { } } + /** + * This version of read uses the retry logic (for lock-less + * commits) to find the right segments file to load. + * @throws CorruptIndexException if the index is corrupt + * @throws IOException if there is a low-level IO error + */ public final void read(Directory directory) throws CorruptIndexException, IOException { + read(directory, CodecProvider.getDefault()); + } + + public final void read(Directory directory, final CodecProvider codecs) throws CorruptIndexException, IOException { generation = lastGeneration = -1; + this.codecs = codecs; new FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - read(directory, segmentFileName); + read(directory, segmentFileName, codecs); return null; } }.run(); @@ -305,7 +297,7 @@ public final class SegmentInfos implements Cloneable, Iterable { // before finishCommit is called IndexOutput pendingSegnOutput; - private void write(Directory directory, Codec codec) throws IOException { + private void write(Directory directory) throws IOException { String segmentFileName = getNextSegmentFileName(); final String globalFieldMapFile; @@ -330,8 +322,8 @@ public final class SegmentInfos implements Cloneable, Iterable { boolean success = false; try { - SegmentInfosWriter infosWriter = codec.segmentInfosFormat().getSegmentInfosWriter(); - segnOutput = infosWriter.writeInfos(directory, segmentFileName, codec.getName(), this, IOContext.DEFAULT); + SegmentInfosWriter infosWriter = codecs.getSegmentInfosWriter(); + segnOutput = infosWriter.writeInfos(directory, segmentFileName, this, IOContext.DEFAULT); infosWriter.prepareCommit(segnOutput); pendingSegnOutput = segnOutput; success = true; @@ -388,7 +380,7 @@ public final class SegmentInfos implements Cloneable, Iterable { sis.cachedUnmodifiableList = null; sis.cachedUnmodifiableSet = null; for(final SegmentInfo info : this) { - assert info.getCodec() != null; + assert info.getSegmentCodecs() != null; // dont directly access segments, use add method!!! sis.add((SegmentInfo) info.clone()); } @@ -417,7 +409,7 @@ public final class SegmentInfos implements Cloneable, Iterable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static long readCurrentVersion(Directory directory) + public static long readCurrentVersion(Directory directory, final CodecProvider codecs) throws CorruptIndexException, IOException { // Fully read the segments file: this ensures that it's @@ -425,8 +417,8 @@ public final class SegmentInfos implements Cloneable, Iterable { // IndexWriter.prepareCommit has been called (but not // yet commit), then the reader will still see itself as // current: - SegmentInfos sis = new SegmentInfos(); - sis.read(directory); + SegmentInfos sis = new SegmentInfos(codecs); + sis.read(directory, codecs); return sis.version; } @@ -435,10 +427,10 @@ public final class SegmentInfos implements Cloneable, Iterable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static Map readCurrentUserData(Directory directory) + public static Map readCurrentUserData(Directory directory, CodecProvider codecs) throws CorruptIndexException, IOException { - SegmentInfos sis = new SegmentInfos(); - sis.read(directory); + SegmentInfos sis = new SegmentInfos(codecs); + sis.read(directory, codecs); return sis.getUserData(); } @@ -816,10 +808,10 @@ public final class SegmentInfos implements Cloneable, Iterable { * method if changes have been made to this {@link SegmentInfos} instance *

**/ - final void prepareCommit(Directory dir, Codec codec) throws IOException { + final void prepareCommit(Directory dir) throws IOException { if (pendingSegnOutput != null) throw new IllegalStateException("prepareCommit was already called"); - write(dir, codec); + write(dir); } private final long writeGlobalFieldMap(FieldNumberBiMap map, Directory dir, String name) throws IOException { @@ -890,12 +882,12 @@ public final class SegmentInfos implements Cloneable, Iterable { return files; } - final void finishCommit(Directory dir, Codec codec) throws IOException { + final void finishCommit(Directory dir) throws IOException { if (pendingSegnOutput == null) throw new IllegalStateException("prepareCommit was not called"); boolean success = false; try { - SegmentInfosWriter infosWriter = codec.segmentInfosFormat().getSegmentInfosWriter(); + SegmentInfosWriter infosWriter = codecs.getSegmentInfosWriter(); infosWriter.finishCommit(pendingSegnOutput); pendingSegnOutput = null; success = true; @@ -966,9 +958,9 @@ public final class SegmentInfos implements Cloneable, Iterable { * method if changes have been made to this {@link SegmentInfos} instance *

**/ - final void commit(Directory dir, Codec codec) throws IOException { - prepareCommit(dir, codec); - finishCommit(dir, codec); + final void commit(Directory dir) throws IOException { + prepareCommit(dir); + finishCommit(dir); } public String toString(Directory directory) { @@ -1114,7 +1106,7 @@ public final class SegmentInfos implements Cloneable, Iterable { if (cloneChildren) { final List list = new ArrayList(size()); for(final SegmentInfo info : this) { - assert info.getCodec() != null; + assert info.getSegmentCodecs() != null; list.add((SegmentInfo) info.clone()); } return list; @@ -1128,14 +1120,6 @@ public final class SegmentInfos implements Cloneable, Iterable { this.addAll(infos); } - /** - * Returns the codec used to decode this SegmentInfos from disk - * @lucene.internal - */ - Codec codecFormat() { - return codecFormat; - } - /** Returns an unmodifiable {@link Iterator} of contained segments in order. */ // @Override (comment out until Java 6) public Iterator iterator() { diff --git a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java index 2e775b1..a53d4b3 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java @@ -73,7 +73,7 @@ final class SegmentMerger { private IOContext context; - SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, Codec codec, IOContext context) { + SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, IOContext context) { this.payloadProcessorProvider = payloadProcessorProvider; directory = dir; segment = name; @@ -89,7 +89,6 @@ final class SegmentMerger { }; } this.termIndexInterval = termIndexInterval; - this.codec = codec; this.context = context; } @@ -255,11 +254,12 @@ final class SegmentMerger { fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.DOC_VALUES), false); } } + final SegmentCodecs codecInfo = fieldInfos.buildSegmentCodecs(false); int docCount = 0; setMatchingSegmentReaders(); - final FieldsWriter fieldsWriter = codec.fieldsFormat().fieldsWriter(directory, segment, context); + final FieldsWriter fieldsWriter = codecInfo.provider.fieldsWriter(directory, segment, context); try { int idx = 0; for (MergeState.IndexReaderAndLiveDocs reader : readers) { @@ -293,7 +293,7 @@ final class SegmentMerger { // entering the index. See LUCENE-1282 for // details. throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption"); - segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codec, null, context); + segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo, null, context); return docCount; } @@ -493,9 +493,9 @@ final class SegmentMerger { } } - Codec getCodec() { + SegmentCodecs getSegmentCodecs() { assert segmentWriteState != null; - return segmentWriteState.codec; + return segmentWriteState.segmentCodecs; } private final void mergeTerms() throws CorruptIndexException, IOException { @@ -566,8 +566,8 @@ final class SegmentMerger { mergeState.dirPayloadProcessor[i] = payloadProcessorProvider.getDirProcessor(reader.reader.directory()); } } - - final FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(segmentWriteState); + codec = segmentWriteState.segmentCodecs.codec(); + final FieldsConsumer consumer = codec.fieldsConsumer(segmentWriteState); boolean success = false; try { consumer.merge(mergeState, @@ -584,7 +584,7 @@ final class SegmentMerger { } private void mergePerDoc() throws IOException { - final PerDocConsumer docsConsumer = codec.docValuesFormat() + final PerDocConsumer docsConsumer = codec .docsConsumer(new PerDocWriteState(segmentWriteState)); // TODO: remove this check when 3.x indexes are no longer supported // (3.x indexes don't have docvalues) diff --git a/lucene/src/java/org/apache/lucene/index/SegmentReadState.java b/lucene/src/java/org/apache/lucene/index/SegmentReadState.java index b59d572..e7e717f 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentReadState.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentReadState.java @@ -35,11 +35,11 @@ public class SegmentReadState { // that must do so), then it should negate this value to // get the app's terms divisor: public int termsIndexDivisor; - public final String segmentSuffix; + public final int codecId; public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, IOContext context, int termsIndexDivisor) { - this(dir, info, fieldInfos, context, termsIndexDivisor, ""); + this(dir, info, fieldInfos, context, termsIndexDivisor, -1); } public SegmentReadState(Directory dir, @@ -47,22 +47,12 @@ public class SegmentReadState { FieldInfos fieldInfos, IOContext context, int termsIndexDivisor, - String segmentSuffix) { + int codecId) { this.dir = dir; this.segmentInfo = info; this.fieldInfos = fieldInfos; this.context = context; this.termsIndexDivisor = termsIndexDivisor; - this.segmentSuffix = segmentSuffix; + this.codecId = codecId; } - - public SegmentReadState(SegmentReadState other, - String newSegmentSuffix) { - this.dir = other.dir; - this.segmentInfo = other.segmentInfo; - this.fieldInfos = other.fieldInfos; - this.context = other.context; - this.termsIndexDivisor = other.termsIndexDivisor; - this.segmentSuffix = newSegmentSuffix; - } -} +} \ No newline at end of file diff --git a/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java b/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java index b092fca..270c084 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java @@ -19,7 +19,6 @@ package org.apache.lucene.index; import java.io.PrintStream; -import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.BitVector; @@ -44,8 +43,8 @@ public class SegmentWriteState { // Lazily created: public BitVector liveDocs; - public final Codec codec; - public final String segmentSuffix; + final SegmentCodecs segmentCodecs; + public final int codecId; /** Expert: The fraction of terms in the "dictionary" which should be stored * in RAM. Smaller values use more memory, but make searching slightly @@ -57,7 +56,7 @@ public class SegmentWriteState { public final IOContext context; public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos, - int numDocs, int termIndexInterval, Codec codec, BufferedDeletes segDeletes, IOContext context) { + int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, BufferedDeletes segDeletes, IOContext context) { this.infoStream = infoStream; this.segDeletes = segDeletes; this.directory = directory; @@ -65,24 +64,24 @@ public class SegmentWriteState { this.fieldInfos = fieldInfos; this.numDocs = numDocs; this.termIndexInterval = termIndexInterval; - this.codec = codec; - segmentSuffix = ""; + this.segmentCodecs = segmentCodecs; + codecId = -1; this.context = context; } /** - * Create a shallow {@link SegmentWriteState} copy final a format ID + * Create a shallow {@link SegmentWriteState} copy final a codec ID */ - public SegmentWriteState(SegmentWriteState state, String segmentSuffix) { + SegmentWriteState(SegmentWriteState state, int codecId) { infoStream = state.infoStream; directory = state.directory; segmentName = state.segmentName; fieldInfos = state.fieldInfos; numDocs = state.numDocs; termIndexInterval = state.termIndexInterval; + segmentCodecs = state.segmentCodecs; context = state.context; - codec = state.codec; - this.segmentSuffix = segmentSuffix; + this.codecId = codecId; segDeletes = state.segDeletes; } } diff --git a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java index 2dd2b79..08a3414 100644 --- a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java @@ -19,7 +19,7 @@ package org.apache.lucene.index; import java.io.IOException; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.FieldsWriter; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.ArrayUtil; @@ -35,12 +35,12 @@ final class StoredFieldsWriter { int freeCount; final DocumentsWriterPerThread.DocState docState; - final Codec codec; + final CodecProvider codecProvider; public StoredFieldsWriter(DocumentsWriterPerThread docWriter) { this.docWriter = docWriter; this.docState = docWriter.docState; - this.codec = docWriter.codec; + this.codecProvider = docWriter.codecProvider; } private int numStoredFields; @@ -81,7 +81,7 @@ final class StoredFieldsWriter { private synchronized void initFieldsWriter(IOContext context) throws IOException { if (fieldsWriter == null) { - fieldsWriter = codec.fieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegment(), context); + fieldsWriter = codecProvider.fieldsWriter(docWriter.directory, docWriter.getSegment(), context); lastDocID = 0; } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java index 5497c69..6534109 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java @@ -109,14 +109,14 @@ public class BlockTermsReader extends FieldsProducer { // private String segment; public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext context, - int termsCacheSize, String segmentSuffix) + int termsCacheSize, int codecId) throws IOException { this.postingsReader = postingsReader; termsCache = new DoubleBarrelLRUCache(termsCacheSize); // this.segment = segment; - in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTermsWriter.TERMS_EXTENSION), + in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTermsWriter.TERMS_EXTENSION), context); boolean success = false; @@ -194,8 +194,12 @@ public class BlockTermsReader extends FieldsProducer { } } - public static void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Collection files) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTermsWriter.TERMS_EXTENSION)); + public static void files(Directory dir, SegmentInfo segmentInfo, int id, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, BlockTermsWriter.TERMS_EXTENSION)); + } + + public static void getExtensions(Collection extensions) { + extensions.add(BlockTermsWriter.TERMS_EXTENSION); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java index af93d73..a07b251 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java @@ -71,7 +71,7 @@ public class BlockTermsWriter extends FieldsConsumer { public BlockTermsWriter(TermsIndexWriterBase termsIndexWriter, SegmentWriteState state, PostingsWriterBase postingsWriter) throws IOException { - final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_EXTENSION); + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); this.termsIndexWriter = termsIndexWriter; out = state.directory.createOutput(termsFileName, state.context); boolean success = false; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java index b409246..aa04ae2 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java @@ -110,13 +110,13 @@ public class BlockTreeTermsReader extends FieldsProducer { public BlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext ioContext, - String segmentSuffix, int indexDivisor) + int codecId, int indexDivisor) throws IOException { this.postingsReader = postingsReader; this.segment = segment; - in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTreeTermsWriter.TERMS_EXTENSION), + in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTreeTermsWriter.TERMS_EXTENSION), ioContext); boolean success = false; @@ -125,7 +125,7 @@ public class BlockTreeTermsReader extends FieldsProducer { try { readHeader(in); if (indexDivisor != -1) { - indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION), + indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION), ioContext); readIndexHeader(indexIn); } @@ -206,9 +206,14 @@ public class BlockTreeTermsReader extends FieldsProducer { } } - public static void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Collection files) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTreeTermsWriter.TERMS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION)); + public static void files(Directory dir, SegmentInfo segmentInfo, int codecID, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, BlockTreeTermsWriter.TERMS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION)); + } + + public static void getExtensions(Collection extensions) { + extensions.add(BlockTreeTermsWriter.TERMS_EXTENSION); + extensions.add(BlockTreeTermsWriter.TERMS_INDEX_EXTENSION); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java index 59d96d9..bee12eb 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java @@ -145,7 +145,7 @@ public class BlockTreeTermsWriter extends FieldsConsumer { throw new IllegalArgumentException("maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock); } - final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_EXTENSION); + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); out = state.directory.createOutput(termsFileName, state.context); boolean success = false; IndexOutput indexOut = null; @@ -157,7 +157,7 @@ public class BlockTreeTermsWriter extends FieldsConsumer { //DEBUG = state.segmentName.equals("_4a"); - final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_INDEX_EXTENSION); + final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); indexOut = state.directory.createOutput(termsIndexFileName, state.context); writeIndexHeader(indexOut); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/Codec.java similarity index 66% rename from lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java rename to lucene/src/java/org/apache/lucene/index/codecs/Codec.java index 407092d..a5ab3cb 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/Codec.java @@ -20,65 +20,61 @@ package org.apache.lucene.index.codecs; import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.util.NamedSPILoader; - import org.apache.lucene.store.Directory; /** @lucene.experimental */ -public abstract class PostingsFormat implements NamedSPILoader.NamedSPI { - - private static final NamedSPILoader loader = - new NamedSPILoader(PostingsFormat.class); - - public static final PostingsFormat[] EMPTY = new PostingsFormat[0]; - /** Unique name that's used to retrieve this format when - * reading the index. - */ - private final String name; +public abstract class Codec { + public static final Codec[] EMPTY = new Codec[0]; + /** Unique name that's used to retrieve this codec when + * reading the index */ + public final String name; - protected PostingsFormat(String name) { + protected Codec(String name) { this.name = name; } - @Override - public String getName() { - return name; - } - /** Writes a new segment */ public abstract FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException; + public static void debug(String s, String desc) { + if (desc != null) { + System.out.println(Thread.currentThread().getName()+ " [" + desc + "]:" + s); + } else { + System.out.println(Thread.currentThread().getName() + ": " + s); + } + } + public static void debug(String s) { + debug(s, null); + } + /** Reads a segment. NOTE: by the time this call * returns, it must hold open any files it will need to * use; else, those files may be deleted. */ public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException; + + public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException; + + public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException; /** * Gathers files associated with this segment * * @param dir the {@link Directory} this segment was written to * @param segmentInfo the {@link SegmentInfo} for this segment - * @param segmentSuffix the format's suffix within this segment + * @param id the codec id within this segment * @param files the of files to add the codec files to. */ - public abstract void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException; + public abstract void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException; + /** Records all file extensions this codec uses */ + public abstract void getExtensions(Set extensions); + @Override public String toString() { - return "PostingsFormat(name=" + name + ")"; - } - - /** looks up a format by name */ - public static PostingsFormat forName(String name) { - return loader.lookup(name); - } - - /** returns a list of all available format names */ - public static Set availablePostingsFormats() { - return loader.availableServices(); + return name; } - } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java b/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java new file mode 100644 index 0000000..086fd7b --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java @@ -0,0 +1,199 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +/** Holds a set of codecs, keyed by name. You subclass + * this, instantiate it, and register your codecs, then + * pass this instance to IndexReader/IndexWriter (via + * package private APIs) to use different codecs when + * reading & writing segments. + * + * @lucene.experimental */ + +public class CodecProvider { + private SegmentInfosWriter infosWriter = new DefaultSegmentInfosWriter(); + private SegmentInfosReader infosReader = new DefaultSegmentInfosReader(); + private String defaultFieldCodec = "Standard"; + private final Map perFieldMap = new HashMap(); + + + private final HashMap codecs = new HashMap(); + + private final Set knownExtensions = new HashSet(); + + + public final static String[] CORE_CODECS = new String[] {"Standard", "Pulsing", "PreFlex", "SimpleText", "Memory"}; + + public synchronized void register(Codec codec) { + if (codec.name == null) { + throw new IllegalArgumentException("code.name is null"); + } + if (!codecs.containsKey(codec.name)) { + codecs.put(codec.name, codec); + codec.getExtensions(knownExtensions); + } else if (codecs.get(codec.name) != codec) { + throw new IllegalArgumentException("codec '" + codec.name + "' is already registered as a different codec instance"); + } + } + + /** @lucene.internal */ + public synchronized void unregister(Codec codec) { + if (codec.name == null) { + throw new IllegalArgumentException("code.name is null"); + } + if (codecs.containsKey(codec.name)) { + Codec c = codecs.get(codec.name); + if (codec == c) { + codecs.remove(codec.name); + } else { + throw new IllegalArgumentException("codec '" + codec.name + "' is being impersonated by a different codec instance!!!"); + } + } + } + + /** @lucene.internal */ + public synchronized Set listAll() { + return codecs.keySet(); + } + + public Collection getAllExtensions() { + return knownExtensions; + } + + public synchronized Codec lookup(String name) { + final Codec codec = codecs.get(name); + if (codec == null) { + throw new IllegalArgumentException("required codec '" + name + "' not found; known codecs: " + codecs.keySet()); + } + return codec; + } + + /** + * Returns true iff a codec with the given name is registered + * @param name codec name + * @return true iff a codec with the given name is registered, otherwise false. + */ + public synchronized boolean isCodecRegistered(String name) { + return codecs.containsKey(name); + } + + public SegmentInfosWriter getSegmentInfosWriter() { + return infosWriter; + } + + public SegmentInfosReader getSegmentInfosReader() { + return infosReader; + } + + /** expert */ + public FieldsReader fieldsReader(Directory directory, String segment, FieldInfos fn, IOContext context, int docStoreOffset, int size) throws IOException { + return new DefaultFieldsReader(directory, segment, fn, context, docStoreOffset, size); + } + + /** expert */ + public FieldsWriter fieldsWriter(Directory directory, String segment, IOContext context) throws IOException { + return new DefaultFieldsWriter(directory, segment, context); + } + + static private CodecProvider defaultCodecs = new CoreCodecProvider(); + + public static CodecProvider getDefault() { + return defaultCodecs; + } + + /** For testing only + * @lucene.internal */ + public static void setDefault(CodecProvider cp) { + defaultCodecs = cp; + } + + /** + * Sets the {@link Codec} for a given field. Not that setting a field's codec is + * write-once. If the field's codec is already set this method will throw an + * {@link IllegalArgumentException}. + * + * @param field + * the name of the field + * @param codec + * the name of the codec + * @throws IllegalArgumentException + * if the codec for the given field is already set + * + */ + public synchronized void setFieldCodec(String field, String codec) { + if (perFieldMap.containsKey(field)) + throw new IllegalArgumentException("codec for field: " + field + + " already set to " + perFieldMap.get(field)); + perFieldMap.put(field, codec); + } + + /** + * Returns the {@link Codec} name for the given field or the default codec if + * not set. + * + * @param name + * the fields name + * @return the {@link Codec} name for the given field or the default codec if + * not set. + */ + public synchronized String getFieldCodec(String name) { + final String codec; + if ((codec = perFieldMap.get(name)) == null) { + return defaultFieldCodec; + } + return codec; + } + + /** + * Returns true if this provider has a Codec registered for this + * field. + */ + public synchronized boolean hasFieldCodec(String name) { + return perFieldMap.containsKey(name); + } + + /** + * Returns the default {@link Codec} for this {@link CodecProvider} + * + * @return the default {@link Codec} for this {@link CodecProvider} + */ + public synchronized String getDefaultFieldCodec() { + return defaultFieldCodec; + } + + /** + * Sets the default {@link Codec} for this {@link CodecProvider} + * + * @param codec + * the codecs name + */ + public synchronized void setDefaultFieldCodec(String codec) { + defaultFieldCodec = codec; + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java similarity index 43% rename from lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosFormat.java rename to lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java index 96bdbc1..74d547b 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosFormat.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java @@ -17,20 +17,35 @@ package org.apache.lucene.index.codecs; * limitations under the License. */ +import org.apache.lucene.index.codecs.memory.MemoryCodec; +import org.apache.lucene.index.codecs.preflex.PreFlexCodec; +import org.apache.lucene.index.codecs.pulsing.PulsingCodec; +import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; +import org.apache.lucene.index.codecs.standard.StandardCodec; + /** - * @lucene.experimental + * A CodecProvider that registers all core codecs that ship + * with Lucene. This will not register any user codecs, but + * you can easily instantiate this class and register them + * yourself and specify per-field codecs: + * + * 
+ *   CodecProvider cp = new CoreCodecProvider();
+ *   cp.register(new MyFastCodec());
+ *   cp.setDefaultFieldCodec("Standard");
+ *   cp.setFieldCodec("id", "Pulsing");
+ *   cp.setFieldCodec("body", "MyFastCodec");
+ *   IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
+ *   iwc.setCodecProvider(cp);
+ *
*/ -public class DefaultSegmentInfosFormat extends SegmentInfosFormat { - private final SegmentInfosReader reader = new DefaultSegmentInfosReader(); - private final SegmentInfosWriter writer = new DefaultSegmentInfosWriter(); - - @Override - public SegmentInfosReader getSegmentInfosReader() { - return reader; - } - @Override - public SegmentInfosWriter getSegmentInfosWriter() { - return writer; +public class CoreCodecProvider extends CodecProvider { + public CoreCodecProvider() { + register(new StandardCodec()); + register(new PreFlexCodec()); + register(new PulsingCodec()); + register(new SimpleTextCodec()); + register(new MemoryCodec()); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java similarity index 33% rename from lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java rename to lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java index aaaa169..114b029 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java @@ -20,29 +20,56 @@ package org.apache.lucene.index.codecs; import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.codecs.DocValuesWriterBase; +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; - -/** @lucene.experimental */ -public class DefaultFieldsFormat extends FieldsFormat { +/** + * Default PerDocConsumer implementation that uses compound file. + * @lucene.experimental + */ +public class DefaultDocValuesConsumer extends DocValuesWriterBase { + private final Directory directory; + + public DefaultDocValuesConsumer(PerDocWriteState state) throws IOException { + super(state); + //TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files? + this.directory = new CompoundFileDirectory(state.directory, + IndexFileNames.segmentFileName(state.segmentName, state.codecId, + IndexFileNames.COMPOUND_FILE_EXTENSION), state.context, true); + } + @Override - public FieldsReader fieldsReader(Directory directory, String segment, - FieldInfos fn, IOContext context, int docStoreOffset, int size) - throws IOException { - return new DefaultFieldsReader(directory, segment, fn, context, docStoreOffset, size); + protected Directory getDirectory() { + return directory; } @Override - public FieldsWriter fieldsWriter(Directory directory, String segment, - IOContext context) throws IOException { - return new DefaultFieldsWriter(directory, segment, context); + public void close() throws IOException { + this.directory.close(); } - @Override - public void files(Directory dir, SegmentInfo info, Set files) throws IOException { - // TODO! + @SuppressWarnings("fallthrough") + public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { + FieldInfos fieldInfos = segmentInfo.getFieldInfos(); + for (FieldInfo fieldInfo : fieldInfos) { + if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); + return; + } + } + } + + public static void getExtensions(Set extensions) { + extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION); + extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java similarity index 40% rename from lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java rename to lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java index e35563a..6a903a0 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java @@ -17,28 +17,49 @@ package org.apache.lucene.index.codecs; * limitations under the License. */ +import java.io.Closeable; import java.io.IOException; -import java.util.Set; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Map; +import java.util.TreeMap; -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.DocValuesReaderBase; +import org.apache.lucene.index.values.IndexDocValues; +import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.IOUtils; -public class DefaultDocValuesFormat extends DocValuesFormat { +/** + * Default PerDocValues implementation that uses compound file. + * @lucene.experimental + */ +public class DefaultDocValuesProducer extends DocValuesReaderBase { + protected final TreeMap docValues; + private final Directory cfs; - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); + /** + * Creates a new {@link DefaultDocValuesProducer} instance and loads all + * {@link IndexDocValues} instances for this segment and codec. + */ + public DefaultDocValuesProducer(SegmentReadState state) throws IOException { + cfs = new CompoundFileDirectory(state.dir, + IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), + state.context, false); + docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.codecId, state.context); } - + @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); + protected Map docValues() { + return docValues; } @Override - public void files(Directory dir, SegmentInfo info, Set files) throws IOException { - DefaultDocValuesConsumer.files(dir, info, files); + protected void closeInternal(Collection closeables) throws IOException { + final ArrayList list = new ArrayList(closeables); + list.add(cfs); + IOUtils.close(list); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java index eda8b0b..cf0bbb2 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java @@ -82,7 +82,7 @@ public final class DefaultFieldsReader extends FieldsReader implements Cloneable /** Verifies that the code version which wrote the segment is supported. */ public static void checkCodeVersion(Directory dir, String segment) throws IOException { - final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_INDEX_EXTENSION); + final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); IndexInput idxStream = dir.openInput(indexStreamFN, IOContext.DEFAULT); try { @@ -121,8 +121,8 @@ public final class DefaultFieldsReader extends FieldsReader implements Cloneable try { fieldInfos = fn; - cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_EXTENSION), context); - final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_INDEX_EXTENSION); + cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); + final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); cloneableIndexStream = d.openInput(indexStreamFN, context); format = cloneableIndexStream.readInt(); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java index ec0fa6b..7d4503b 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java @@ -61,10 +61,6 @@ public final class DefaultFieldsWriter extends FieldsWriter { // when removing support for old versions, leave the last supported version here static final int FORMAT_MINIMUM = FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS; - // TODO: remove from IndexFileNames - public static final String FIELDS_EXTENSION = IndexFileNames.FIELDS_EXTENSION; - public static final String FIELDS_INDEX_EXTENSION = IndexFileNames.FIELDS_INDEX_EXTENSION; - // If null - we were supplied with streams, if notnull - we manage them ourselves private Directory directory; private String segment; @@ -77,8 +73,8 @@ public final class DefaultFieldsWriter extends FieldsWriter { boolean success = false; try { - fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context); - indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION), context); + fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); + indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION), context); fieldsStream.writeInt(FORMAT_CURRENT); indexStream.writeInt(FORMAT_CURRENT); @@ -133,11 +129,11 @@ public final class DefaultFieldsWriter extends FieldsWriter { } catch (IOException ignored) { } try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION)); + directory.deleteFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION)); } catch (IOException ignored) { } try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION)); + directory.deleteFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); } catch (IOException ignored) { } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java dissimilarity index 62% index 6518f62..b8f1bee 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java @@ -?,? +1,126 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.index.IndexFormatTooNewException; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.CompoundFileDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; + +/** + * Default implementation of {@link SegmentInfosReader}. + * @lucene.experimental + */ +public class DefaultSegmentInfosReader extends SegmentInfosReader { + + @Override + public void read(Directory directory, String segmentsFileName, CodecProvider codecs, + SegmentInfos infos, IOContext context) throws IOException { + IndexInput input = null; + try { + input = openInput(directory, segmentsFileName, context); + final int format = input.readInt(); + infos.setFormat(format); + + // check that it is a format we can understand + if (format > DefaultSegmentInfosWriter.FORMAT_MINIMUM) + throw new IndexFormatTooOldException(segmentsFileName, format, + DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); + if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT) + throw new IndexFormatTooNewException(segmentsFileName, format, + DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); + + infos.version = input.readLong(); // read version + infos.counter = input.readInt(); // read counter + if (infos.getFormat() <= DefaultSegmentInfosWriter.FORMAT_4_0) { + infos.setGlobalFieldMapVersion(input.readLong()); + } + for (int i = input.readInt(); i > 0; i--) { // read segmentInfos + SegmentInfo si = new SegmentInfo(directory, format, input, codecs); + if (si.getVersion() == null) { + // Could be a 3.0 - try to open the doc stores - if it fails, it's a + // 2.x segment, and an IndexFormatTooOldException will be thrown, + // which is what we want. + Directory dir = directory; + if (si.getDocStoreOffset() != -1) { + if (si.getDocStoreIsCompoundFile()) { + dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName( + si.getDocStoreSegment(), "", + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false); + } + } else if (si.getUseCompoundFile()) { + dir = new CompoundFileDirectory(dir,IndexFileNames.segmentFileName( + si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); + } + + try { + DefaultFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment()); + } finally { + // If we opened the directory, close it + if (dir != directory) dir.close(); + } + + // Above call succeeded, so it's a 3.0 segment. Upgrade it so the next + // time the segment is read, its version won't be null and we won't + // need to open FieldsReader every time for each such segment. + si.setVersion("3.0"); + } else if (si.getVersion().equals("2.x")) { + // If it's a 3x index touched by 3.1+ code, then segments record their + // version, whether they are 2.x ones or not. We detect that and throw + // appropriate exception. + throw new IndexFormatTooOldException(si.name, si.getVersion()); + } + infos.add(si); + } + + infos.userData = input.readStringStringMap(); + finalizeInput(input); + + } finally { + if (input != null) { + input.close(); + } + } + + } + + public IndexInput openInput(Directory dir, String segmentsFileName, IOContext context) throws IOException { + IndexInput in = dir.openInput(segmentsFileName, context); + return new ChecksumIndexInput(in); + + } + + public void finalizeInput(IndexInput input) throws IOException, CorruptIndexException { + ChecksumIndexInput cksumInput = (ChecksumIndexInput)input; + final long checksumNow = cksumInput.getChecksum(); + final long checksumThen = cksumInput.readLong(); + if (checksumNow != checksumThen) + throw new CorruptIndexException("checksum mismatch in segments file"); + + } + +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java index cf69f56..e029262 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java @@ -56,13 +56,12 @@ public class DefaultSegmentInfosWriter extends SegmentInfosWriter { public static final int FORMAT_MINIMUM = FORMAT_DIAGNOSTICS; @Override - public IndexOutput writeInfos(Directory dir, String segmentFileName, String codecID, SegmentInfos infos, IOContext context) + public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos, IOContext context) throws IOException { IndexOutput out = createOutput(dir, segmentFileName, new IOContext(new FlushInfo(infos.size(), infos.totalDocCount()))); boolean success = false; try { out.writeInt(FORMAT_CURRENT); // write FORMAT - out.writeString(codecID); // write codecID out.writeLong(infos.version); out.writeInt(infos.counter); // write counter out.writeLong(infos.getGlobalFieldMapVersion()); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java deleted file mode 100644 index 2bfc466..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java index e545d4a..568ee4e 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java @@ -65,19 +65,19 @@ public abstract class DocValuesReaderBase extends PerDocValues { // Only opens files... doesn't actually load any values protected TreeMap load(FieldInfos fieldInfos, - String segment, int docCount, Directory dir, IOContext context) + String segment, int docCount, Directory dir, int codecId, IOContext context) throws IOException { TreeMap values = new TreeMap(); boolean success = false; try { for (FieldInfo fieldInfo : fieldInfos) { - if (fieldInfo.hasDocValues()) { + if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) { final String field = fieldInfo.name; // TODO can we have a compound file per segment and codec for // docvalues? final String id = DefaultDocValuesConsumer.docValuesId(segment, - fieldInfo.number); + codecId, fieldInfo.number); values.put(field, loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), context)); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java index 619b124..e328704 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java @@ -33,19 +33,19 @@ import org.apache.lucene.util.Counter; * @lucene.experimental */ public abstract class DocValuesWriterBase extends PerDocConsumer { - protected final String segmentName; - protected final String segmentSuffix; + private final String segmentName; + private final int codecId; private final Counter bytesUsed; - protected final IOContext context; + private final IOContext context; protected DocValuesWriterBase(PerDocWriteState state) { this.segmentName = state.segmentName; - this.segmentSuffix = state.segmentSuffix; + this.codecId = state.codecId; this.bytesUsed = state.bytesUsed; this.context = state.context; } - protected abstract Directory getDirectory() throws IOException; + protected abstract Directory getDirectory(); @Override public void close() throws IOException { @@ -54,12 +54,12 @@ public abstract class DocValuesWriterBase extends PerDocConsumer { @Override public DocValuesConsumer addValuesField(FieldInfo field) throws IOException { return Writer.create(field.getDocValues(), - docValuesId(segmentName, field.number), + docValuesId(segmentName, codecId, field.number), getDirectory(), getComparator(), bytesUsed, context); } - public static String docValuesId(String segmentsName, int fieldId) { - return segmentsName + "_" + fieldId; + public static String docValuesId(String segmentsName, int codecID, int fieldId) { + return segmentsName + "_" + codecID + "-" + fieldId; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java deleted file mode 100644 index ed4f413..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java b/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java index 60f4bf5..acb08ca 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java @@ -34,4 +34,23 @@ import org.apache.lucene.index.Terms; public abstract class FieldsProducer extends Fields implements Closeable { public abstract void close() throws IOException; + + public static final FieldsProducer EMPTY = new FieldsProducer() { + + @Override + public Terms terms(String field) throws IOException { + return null; + } + + @Override + public FieldsEnum iterator() throws IOException { + return FieldsEnum.EMPTY; + } + + @Override + public void close() throws IOException { + + } + }; + } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java index 062179b..6ce2eeb 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java @@ -68,14 +68,14 @@ public class FixedGapTermsIndexReader extends TermsIndexReaderBase { // start of the field info data protected long dirOffset; - public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, String segmentSuffix, IOContext context) + public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, int codecId, IOContext context) throws IOException { this.termComp = termComp; assert indexDivisor == -1 || indexDivisor > 0; - in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); + in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); boolean success = false; @@ -387,8 +387,17 @@ public class FixedGapTermsIndexReader extends TermsIndexReaderBase { } } - public static void files(Directory dir, SegmentInfo info, String segmentSuffix, Collection files) { - files.add(IndexFileNames.segmentFileName(info.name, segmentSuffix, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + public static void files(Directory dir, SegmentInfo info, int id, Collection files) { + files.add(IndexFileNames.segmentFileName(info.name, id, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + } + + public static void getIndexExtensions(Collection extensions) { + extensions.add(FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION); + } + + @Override + public void getExtensions(Collection extensions) { + getIndexExtensions(extensions); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java index 58ae9e8..4d424a6 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java @@ -56,7 +56,7 @@ public class FixedGapTermsIndexWriter extends TermsIndexWriterBase { private final FieldInfos fieldInfos; // unread public FixedGapTermsIndexWriter(SegmentWriteState state) throws IOException { - final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_INDEX_EXTENSION); + final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); termIndexInterval = state.termIndexInterval; out = state.directory.createOutput(indexFileName, state.context); boolean success = false; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java b/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java index 11de4d4..869a975 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java @@ -29,7 +29,7 @@ import org.apache.lucene.index.values.ValueType; * this convert field values into a Codec specific format during indexing. *

* The {@link PerDocConsumer} API is accessible through the - * {@link PostingsFormat} - API providing per field consumers and producers for inverted + * {@link Codec} - API providing per field consumers and producers for inverted * data (terms, postings) as well as per-document data. * * @lucene.experimental diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java b/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java index 3420c70..5c2b721 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java @@ -28,7 +28,7 @@ import org.apache.lucene.index.values.IndexDocValues; * {@link PerDocConsumer} counterpart. *

* The {@link PerDocValues} API is accessible through the - * {@link PostingsFormat} - API providing per field consumers and producers for inverted + * {@link Codec} - API providing per field consumers and producers for inverted * data (terms, postings) as well as per-document data. * * @lucene.experimental diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PostingsBaseFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/PostingsBaseFormat.java deleted file mode 100644 index fabb8dc..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java b/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java index 54253b3..05aec94 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java @@ -36,9 +36,6 @@ import org.apache.lucene.util.Bits; * time. * @lucene.experimental */ -// TODO: find a better name; this defines the API that the -// terms dict impls use to talk to a postings impl. -// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer public abstract class PostingsReaderBase implements Closeable { public abstract void init(IndexInput termsIn) throws IOException; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java b/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java index b456615..5e6ea14 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java @@ -27,9 +27,6 @@ import org.apache.lucene.index.FieldInfo; * @lucene.experimental */ -// TODO: find a better name; this defines the API that the -// terms dict impls use to talk to a postings impl. -// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer public abstract class PostingsWriterBase extends PostingsConsumer implements Closeable { public abstract void start(IndexOutput termsOut) throws IOException; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosFormat.java deleted file mode 100644 index 1632ce1..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java index e3f8a7c..69b6804 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java @@ -20,7 +20,6 @@ package org.apache.lucene.index.codecs; import java.io.IOException; import org.apache.lucene.index.SegmentInfos; -import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -34,9 +33,9 @@ public abstract class SegmentInfosReader { * Read {@link SegmentInfos} data from a directory. * @param directory directory to read from * @param segmentsFileName name of the "segments_N" file - * @param header input of "segments_N" file after reading preamble + * @param codecs current codecs * @param infos empty instance to be populated with data * @throws IOException */ - public abstract void read(Directory directory, String segmentsFileName, ChecksumIndexInput header, SegmentInfos infos, IOContext context) throws IOException; + public abstract void read(Directory directory, String segmentsFileName, CodecProvider codecs, SegmentInfos infos, IOContext context) throws IOException; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java index d392b87..9c79edf 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java @@ -42,13 +42,13 @@ public abstract class SegmentInfosWriter { * phase commit" operations as described above. * @throws IOException */ - public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, String codecID, SegmentInfos infos, IOContext context) throws IOException; + public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, SegmentInfos infos, IOContext context) throws IOException; /** * First phase of the two-phase commit - ensure that all output can be * successfully written out. * @param out an instance of {@link IndexOutput} returned from a previous - * call to {@link #writeInfos(Directory, String, String, SegmentInfos, IOContext)}. + * call to {@link #writeInfos(Directory, String, SegmentInfos, IOContext)}. * @throws IOException */ public abstract void prepareCommit(IndexOutput out) throws IOException; @@ -57,7 +57,7 @@ public abstract class SegmentInfosWriter { * Second phase of the two-phase commit. In this step the output should be * finalized and closed. * @param out an instance of {@link IndexOutput} returned from a previous - * call to {@link #writeInfos(Directory, String, String, SegmentInfos, IOContext)}. + * call to {@link #writeInfos(Directory, String, SegmentInfos, IOContext)}. * @throws IOException */ public abstract void finishCommit(IndexOutput out) throws IOException; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexReaderBase.java b/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexReaderBase.java index d66999a..9123ff3 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexReaderBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexReaderBase.java @@ -45,6 +45,8 @@ public abstract class TermsIndexReaderBase implements Closeable { public abstract void close() throws IOException; + public abstract void getExtensions(Collection extensions); + public abstract boolean supportsOrd(); public abstract int getDivisor(); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java index de80e6d..f9d1552 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java @@ -57,9 +57,9 @@ public class VariableGapTermsIndexReader extends TermsIndexReaderBase { protected long dirOffset; final String segment; - public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, String segmentSuffix, IOContext context) + public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, int codecId, IOContext context) throws IOException { - in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true)); + in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true)); this.segment = segment; boolean success = false; assert indexDivisor == -1 || indexDivisor > 0; @@ -215,8 +215,17 @@ public class VariableGapTermsIndexReader extends TermsIndexReaderBase { } } - public static void files(Directory dir, SegmentInfo info, String segmentSuffix, Collection files) { - files.add(IndexFileNames.segmentFileName(info.name, segmentSuffix, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + public static void files(Directory dir, SegmentInfo info, int id, Collection files) { + files.add(IndexFileNames.segmentFileName(info.name, id, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + } + + public static void getIndexExtensions(Collection extensions) { + extensions.add(VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION); + } + + @Override + public void getExtensions(Collection extensions) { + getIndexExtensions(extensions); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java index 9964609..221451f 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java @@ -158,7 +158,7 @@ public class VariableGapTermsIndexWriter extends TermsIndexWriterBase { // in the extremes. public VariableGapTermsIndexWriter(SegmentWriteState state, IndexTermSelector policy) throws IOException { - final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_INDEX_EXTENSION); + final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); out = state.directory.createOutput(indexFileName, state.context); boolean success = false; try { diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java deleted file mode 100644 index 668ffce..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsBaseFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsBaseFormat.java deleted file mode 100644 index edd4a53..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java similarity index 95% rename from lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java rename to lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java index 0b6c070..3eaa8ee 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java @@ -31,14 +31,19 @@ import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; +import org.apache.lucene.index.codecs.DefaultDocValuesProducer; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsConsumer; import org.apache.lucene.index.codecs.TermStats; import org.apache.lucene.index.codecs.TermsConsumer; @@ -76,12 +81,9 @@ import org.apache.lucene.util.fst.FST; * * @lucene.experimental */ -// TODO: Maybe name this 'Cached' or something to reflect -// the reality that it is actually written to disk, but -// loads itself in ram? -public class MemoryPostingsFormat extends PostingsFormat { +public class MemoryCodec extends Codec { - public MemoryPostingsFormat() { + public MemoryCodec() { super("Memory"); } @@ -243,7 +245,7 @@ public class MemoryPostingsFormat extends PostingsFormat { @Override public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - final String fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, EXTENSION); + final String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, EXTENSION); final IndexOutput out = state.directory.createOutput(fileName, state.context); return new FieldsConsumer() { @@ -735,7 +737,7 @@ public class MemoryPostingsFormat extends PostingsFormat { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION); + final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, EXTENSION); final IndexInput in = state.dir.openInput(fileName, IOContext.READONCE); final SortedMap fields = new TreeMap(); @@ -790,7 +792,24 @@ public class MemoryPostingsFormat extends PostingsFormat { } @Override - public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION)); + public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, EXTENSION)); + DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); + } + + @Override + public void getExtensions(Set extensions) { + extensions.add(EXTENSION); + DefaultDocValuesConsumer.getExtensions(extensions); + } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new DefaultDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new DefaultDocValuesProducer(state); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xPostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java similarity index 65% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xPostingsFormat.java rename to lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java index 23da306..257c3ea 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xPostingsFormat.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -21,12 +21,15 @@ import java.util.Set; import java.io.IOException; import org.apache.lucene.store.Directory; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; /** Codec that reads the pre-flex-indexing postings * format. It does not provide a writer because newly @@ -37,7 +40,7 @@ import org.apache.lucene.index.codecs.FieldsProducer; * @lucene.experimental */ @Deprecated -public class Lucene3xPostingsFormat extends PostingsFormat { +public class PreFlexCodec extends Codec { /** Extension of terms file */ public static final String TERMS_EXTENSION = "tis"; @@ -51,8 +54,8 @@ public class Lucene3xPostingsFormat extends PostingsFormat { /** Extension of prox postings file */ public static final String PROX_EXTENSION = "prx"; - public Lucene3xPostingsFormat() { - super("Lucene3x"); + public PreFlexCodec() { + super("PreFlex"); } @Override @@ -62,12 +65,30 @@ public class Lucene3xPostingsFormat extends PostingsFormat { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - return new Lucene3xFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor); + return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor); } @Override - public void files(Directory dir, SegmentInfo info, String segmentSuffix, Set files) throws IOException { - // preflex fields have no segmentSuffix - we ignore it here - Lucene3xFields.files(dir, info, files); + public void files(Directory dir, SegmentInfo info, int id, Set files) throws IOException { + // preflex fields have no codec ID - we ignore it here + PreFlexFields.files(dir, info, files); + } + + @Override + public void getExtensions(Set extensions) { + extensions.add(FREQ_EXTENSION); + extensions.add(PROX_EXTENSION); + extensions.add(TERMS_EXTENSION); + extensions.add(TERMS_INDEX_EXTENSION); + } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return null; + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return null; } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java similarity index 97% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java rename to lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java index 194eb3c..e68555e 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -51,7 +51,7 @@ import org.apache.lucene.util.UnicodeUtil; * @deprecated (4.0) */ @Deprecated -public class Lucene3xFields extends FieldsProducer { +public class PreFlexFields extends FieldsProducer { private static final boolean DEBUG_SURROGATES = false; @@ -68,7 +68,7 @@ public class Lucene3xFields extends FieldsProducer { private final IOContext context; private Directory cfsReader; - public Lucene3xFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, IOContext context, int indexDivisor) + public PreFlexFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, IOContext context, int indexDivisor) throws IOException { si = info; @@ -95,7 +95,7 @@ public class Lucene3xFields extends FieldsProducer { // make sure that all index files have been read or are kept open // so that if an index update removes them we'll still have them - freqStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.FREQ_EXTENSION), context); + freqStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.FREQ_EXTENSION), context); boolean anyProx = false; for (FieldInfo fi : fieldInfos) { if (fi.isIndexed) { @@ -108,7 +108,7 @@ public class Lucene3xFields extends FieldsProducer { } if (anyProx) { - proxStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.PROX_EXTENSION), context); + proxStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.PROX_EXTENSION), context); } else { proxStream = null; } @@ -136,16 +136,16 @@ public class Lucene3xFields extends FieldsProducer { } static void files(Directory dir, SegmentInfo info, Collection files) throws IOException { - files.add(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.TERMS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION)); - files.add(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.FREQ_EXTENSION)); + files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.TERMS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.TERMS_INDEX_EXTENSION)); + files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.FREQ_EXTENSION)); if (info.getHasProx()) { // LUCENE-1739: for certain versions of 2.9-dev, // hasProx would be incorrectly computed during // indexing as true, and then stored into the segments // file, when it should have been false. So we do the // extra check, here: - final String prx = IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.PROX_EXTENSION); + final String prx = IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.PROX_EXTENSION); if (dir.fileExists(prx)) { files.add(prx); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java similarity index 98% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java rename to lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java index 5177b7d..b140f61 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -23,7 +23,7 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.lucene40.DefaultSkipListReader; +import org.apache.lucene.index.codecs.standard.DefaultSkipListReader; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Bits; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermEnum.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java similarity index 99% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermEnum.java rename to lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java index 1bb854f..179e946 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermEnum.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermPositions.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java similarity index 99% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermPositions.java rename to lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java index 5f6921c..882e784 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermPositions.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermBuffer.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java similarity index 98% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermBuffer.java rename to lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java index 24c2089..9a51206 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermBuffer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfo.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java similarity index 97% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfo.java rename to lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java index 23009cb..8f91569 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfo.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java similarity index 97% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReader.java rename to lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java index ad6e2a0..c5192d7 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -109,7 +109,7 @@ public final class TermInfosReader { segment = seg; fieldInfos = fis; - origEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), + origEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_EXTENSION), context), fieldInfos, false); size = origEnum.size; @@ -118,7 +118,7 @@ public final class TermInfosReader { // Load terms index totalIndexInterval = origEnum.indexInterval * indexDivisor; - final String indexFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); + final String indexFileName = IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION); final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(indexFileName, context), fieldInfos, true); @@ -228,8 +228,8 @@ public final class TermInfosReader { // optimize sequential access: first try scanning cached enum w/o seeking if (enumerator.term() != null // term is at or past current - && ((enumerator.prev() != null && compareAsUTF16(term, enumerator.prev())> 0) - || compareAsUTF16(term, enumerator.term()) >= 0)) { + && ((enumerator.prev() != null && compareAsUTF16(term, enumerator.prev())> 0) + || compareAsUTF16(term, enumerator.term()) >= 0)) { int enumOffset = (int)(enumerator.position/totalIndexInterval)+1; if (indexLength == enumOffset // but before end of block || index.compareTo(term, enumOffset) < 0) { diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java similarity index 99% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java rename to lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java index 6dd0398..d384ff9 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/package.html b/lucene/src/java/org/apache/lucene/index/codecs/preflex/package.html similarity index 100% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene3x/package.html rename to lucene/src/java/org/apache/lucene/index/codecs/preflex/package.html diff --git a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/Pulsing40PostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/Pulsing40PostingsFormat.java deleted file mode 100644 index 6d20af0..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java similarity index 57% rename from lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java rename to lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java index 0c83f20..04d1a14 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java @@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.pulsing; import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; @@ -27,48 +28,61 @@ import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.BlockTreeTermsReader; import org.apache.lucene.index.codecs.BlockTreeTermsWriter; -import org.apache.lucene.index.codecs.PostingsBaseFormat; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; +import org.apache.lucene.index.codecs.DefaultDocValuesProducer; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; +import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.index.codecs.standard.StandardPostingsReader; +import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; import org.apache.lucene.store.Directory; -/** This postings format "inlines" the postings for terms that have - * low docFreq. It wraps another postings format, which is used for +/** This codec "inlines" the postings for terms that have + * low docFreq. It wraps another codec, which is used for * writing the non-inlined terms. * + * Currently in only inlines docFreq=1 terms, and + * otherwise uses the normal "standard" codec. * @lucene.experimental */ -public abstract class PulsingPostingsFormat extends PostingsFormat { +public class PulsingCodec extends Codec { private final int freqCutoff; private final int minBlockSize; private final int maxBlockSize; - private final PostingsBaseFormat wrappedPostingsBaseFormat; + + public PulsingCodec() { + this(1); + } - public PulsingPostingsFormat(String name, PostingsBaseFormat wrappedPostingsBaseFormat, int freqCutoff) { - this(name, wrappedPostingsBaseFormat, freqCutoff, BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); + public PulsingCodec(int freqCutoff) { + this(freqCutoff, BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); } /** Terms with freq <= freqCutoff are inlined into terms * dict. */ - public PulsingPostingsFormat(String name, PostingsBaseFormat wrappedPostingsBaseFormat, int freqCutoff, int minBlockSize, int maxBlockSize) { - super(name); + public PulsingCodec(int freqCutoff, int minBlockSize, int maxBlockSize) { + super("Pulsing"); this.freqCutoff = freqCutoff; this.minBlockSize = minBlockSize; assert minBlockSize > 1; this.maxBlockSize = maxBlockSize; - this.wrappedPostingsBaseFormat = wrappedPostingsBaseFormat; } @Override public String toString() { - return getName() + "(freqCutoff=" + freqCutoff + " minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")"; + return name + "(freqCutoff=" + freqCutoff + " minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")"; } @Override public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase docsWriter = wrappedPostingsBaseFormat.postingsWriterBase(state); + // We wrap StandardPostingsWriter, but any PostingsWriterBase + // will work: + + PostingsWriterBase docsWriter = new StandardPostingsWriter(state); // Terms that have <= freqCutoff number of docs are // "pulsed" (inlined): @@ -90,7 +104,9 @@ public abstract class PulsingPostingsFormat extends PostingsFormat { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state); + // We wrap StandardPostingsReader, but any StandardPostingsReader + // will work: + PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); PostingsReaderBase pulsingReader = new PulsingPostingsReader(docsReader); boolean success = false; @@ -99,7 +115,7 @@ public abstract class PulsingPostingsFormat extends PostingsFormat { state.dir, state.fieldInfos, state.segmentInfo.name, pulsingReader, state.context, - state.segmentSuffix, + state.codecId, state.termsIndexDivisor); success = true; return ret; @@ -115,8 +131,25 @@ public abstract class PulsingPostingsFormat extends PostingsFormat { } @Override - public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { - wrappedPostingsBaseFormat.files(dir, segmentInfo, segmentSuffix, files); - BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); + public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set files) throws IOException { + StandardPostingsReader.files(dir, segmentInfo, codecID, files); + BlockTreeTermsReader.files(dir, segmentInfo, codecID, files); + DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files); + } + + @Override + public void getExtensions(Set extensions) { + StandardCodec.getStandardExtensions(extensions); + DefaultDocValuesConsumer.getExtensions(extensions); + } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new DefaultDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new DefaultDocValuesProducer(state); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java index 8a387a2..6083d29 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java @@ -47,12 +47,12 @@ public class SepDocValuesConsumer extends DocValuesWriterBase { } @SuppressWarnings("fallthrough") - public static void files(Directory dir, SegmentInfo segmentInfo, + public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { FieldInfos fieldInfos = segmentInfo.getFieldInfos(); for (FieldInfo fieldInfo : fieldInfos) { - if (fieldInfo.hasDocValues()) { - String filename = docValuesId(segmentInfo.name, fieldInfo.number); + if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) { + String filename = docValuesId(segmentInfo.name, codecId, fieldInfo.number); switch (fieldInfo.getDocValues()) { case BYTES_FIXED_DEREF: case BYTES_VAR_DEREF: @@ -83,4 +83,9 @@ public class SepDocValuesConsumer extends DocValuesWriterBase { } } } + + public static void getExtensions(Set extensions) { + extensions.add(Writer.DATA_EXTENSION); + extensions.add(Writer.INDEX_EXTENSION); + } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java index 88d9f1f..71f42a7 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java @@ -39,7 +39,7 @@ public class SepDocValuesProducer extends DocValuesReaderBase { * {@link IndexDocValues} instances for this segment and codec. */ public SepDocValuesProducer(SegmentReadState state) throws IOException { - docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, state.dir, state.context); + docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, state.dir, state.codecId, state.context); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java index 678d3c8..50f87f2 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java @@ -60,23 +60,23 @@ public class SepPostingsReader extends PostingsReaderBase { int maxSkipLevels; int skipMinimum; - public SepPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext context, IntStreamFactory intFactory, String segmentSuffix) throws IOException { + public SepPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext context, IntStreamFactory intFactory, int codecId) throws IOException { boolean success = false; try { - final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.DOC_EXTENSION); + final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.DOC_EXTENSION); docIn = intFactory.openInput(dir, docFileName, context); - skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.SKIP_EXTENSION), context); + skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.SKIP_EXTENSION), context); if (segmentInfo.getFieldInfos().hasFreq()) { - freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.FREQ_EXTENSION), context); + freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.FREQ_EXTENSION), context); } else { freqIn = null; } if (segmentInfo.getHasProx()) { - posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.POS_EXTENSION), context); - payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.PAYLOAD_EXTENSION), context); + posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.POS_EXTENSION), context); + payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.PAYLOAD_EXTENSION), context); } else { posIn = null; payloadIn = null; @@ -89,17 +89,17 @@ public class SepPostingsReader extends PostingsReaderBase { } } - public static void files(SegmentInfo segmentInfo, String segmentSuffix, Collection files) throws IOException { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.DOC_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.SKIP_EXTENSION)); + public static void files(SegmentInfo segmentInfo, int codecId, Collection files) throws IOException { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.DOC_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.SKIP_EXTENSION)); if (segmentInfo.getFieldInfos().hasFreq()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.FREQ_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.FREQ_EXTENSION)); } if (segmentInfo.getHasProx()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.POS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.PAYLOAD_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.POS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.PAYLOAD_EXTENSION)); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java index 7031e5c..05a5639 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java @@ -116,27 +116,27 @@ public final class SepPostingsWriter extends PostingsWriterBase { try { this.skipInterval = skipInterval; this.skipMinimum = skipInterval; /* set to the same for now */ - final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, DOC_EXTENSION); + final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, DOC_EXTENSION); docOut = factory.createOutput(state.directory, docFileName, state.context); docIndex = docOut.index(); if (state.fieldInfos.hasFreq()) { - final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, FREQ_EXTENSION); + final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, FREQ_EXTENSION); freqOut = factory.createOutput(state.directory, frqFileName, state.context); freqIndex = freqOut.index(); } if (state.fieldInfos.hasProx()) { - final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, POS_EXTENSION); + final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, POS_EXTENSION); posOut = factory.createOutput(state.directory, posFileName, state.context); posIndex = posOut.index(); // TODO: -- only if at least one field stores payloads? - final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, PAYLOAD_EXTENSION); + final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, PAYLOAD_EXTENSION); payloadOut = state.directory.createOutput(payloadFileName, state.context); } - final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, SKIP_EXTENSION); + final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SKIP_EXTENSION); skipOut = state.directory.createOutput(skipFileName, state.context); totalNumDocs = state.numDocs; @@ -391,4 +391,12 @@ public final class SepPostingsWriter extends PostingsWriterBase { public void close() throws IOException { IOUtils.close(docOut, skipOut, freqOut, posOut, payloadOut); } + + public static void getExtensions(Set extensions) { + extensions.add(DOC_EXTENSION); + extensions.add(FREQ_EXTENSION); + extensions.add(SKIP_EXTENSION); + extensions.add(POS_EXTENSION); + extensions.add(PAYLOAD_EXTENSION); + } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java similarity index 60% rename from lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java rename to lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java index 6b47bf3..93fac42 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java @@ -20,13 +20,18 @@ package org.apache.lucene.index.codecs.simpletext; import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesProducer; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; +import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.store.Directory; /** For debugging, curiosity, transparency only!! Do not @@ -37,12 +42,13 @@ import org.apache.lucene.store.Directory; * any text editor, and even edit it to alter your index. * * @lucene.experimental */ -public class SimpleTextPostingsFormat extends PostingsFormat { +public class SimpleTextCodec extends Codec { - public SimpleTextPostingsFormat() { + public SimpleTextCodec() { super("SimpleText"); } + @Override public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { return new SimpleTextFieldsWriter(state); @@ -56,12 +62,30 @@ public class SimpleTextPostingsFormat extends PostingsFormat { /** Extension of freq postings file */ static final String POSTINGS_EXTENSION = "pst"; - static String getPostingsFileName(String segment, String segmentSuffix) { - return IndexFileNames.segmentFileName(segment, segmentSuffix, POSTINGS_EXTENSION); + static String getPostingsFileName(String segment, int id) { + return IndexFileNames.segmentFileName(segment, id, POSTINGS_EXTENSION); + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { + files.add(getPostingsFileName(segmentInfo.name, id)); + DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); + } + + @Override + public void getExtensions(Set extensions) { + extensions.add(POSTINGS_EXTENSION); + DefaultDocValuesConsumer.getExtensions(extensions); + } + + // TODO: would be great if these used a plain text impl + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new DefaultDocValuesConsumer(state); } @Override - public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { - files.add(getPostingsFileName(segmentInfo.name, segmentSuffix)); + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new DefaultDocValuesProducer(state); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java index 1c415f1..7db26cb 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java @@ -60,7 +60,7 @@ class SimpleTextFieldsReader extends FieldsProducer { final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD; public SimpleTextFieldsReader(SegmentReadState state) throws IOException { - in = state.dir.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context); + in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, state.codecId), state.context); fieldInfos = state.fieldInfos; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java index 4deec0c..9427da6 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java @@ -47,7 +47,7 @@ class SimpleTextFieldsWriter extends FieldsConsumer { final static BytesRef PAYLOAD = new BytesRef(" payload "); public SimpleTextFieldsWriter(SegmentWriteState state) throws IOException { - final String fileName = SimpleTextPostingsFormat.getPostingsFileName(state.segmentName, state.segmentSuffix); + final String fileName = SimpleTextCodec.getPostingsFileName(state.segmentName, state.codecId); out = state.directory.createOutput(fileName, state.context); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListReader.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListReader.java similarity index 98% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListReader.java rename to lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListReader.java index 17906d1..32f52c8 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListReader.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene40; +package org.apache.lucene.index.codecs.standard; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java similarity index 99% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListWriter.java rename to lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java index eb993ad..c2435c8 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene40; +package org.apache.lucene.index.codecs.standard; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java similarity index 64% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java rename to lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java index 9104acb..a8a546a 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene40; +package org.apache.lucene.index.codecs.standard; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.lucene40; import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; @@ -27,28 +28,28 @@ import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.BlockTreeTermsReader; import org.apache.lucene.index.codecs.BlockTreeTermsWriter; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; +import org.apache.lucene.index.codecs.DefaultDocValuesProducer; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.store.Directory; /** Default codec. * @lucene.experimental */ - -// TODO: this class could be created by wrapping -// BlockTreeTermsDict around Lucene40PostingsBaseFormat; ie -// we should not duplicate the code from that class here: -public class Lucene40PostingsFormat extends PostingsFormat { +public class StandardCodec extends Codec { private final int minBlockSize; private final int maxBlockSize; - public Lucene40PostingsFormat() { + public StandardCodec() { this(BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); } - public Lucene40PostingsFormat(int minBlockSize, int maxBlockSize) { - super("Lucene40"); + public StandardCodec(int minBlockSize, int maxBlockSize) { + super("Standard"); this.minBlockSize = minBlockSize; assert minBlockSize > 1; this.maxBlockSize = maxBlockSize; @@ -56,7 +57,7 @@ public class Lucene40PostingsFormat extends PostingsFormat { @Override public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase docs = new Lucene40PostingsWriter(state); + PostingsWriterBase docs = new StandardPostingsWriter(state); // TODO: should we make the terms index more easily // pluggable? Ie so that this codec would record which @@ -78,7 +79,7 @@ public class Lucene40PostingsFormat extends PostingsFormat { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postings = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); boolean success = false; try { @@ -88,7 +89,7 @@ public class Lucene40PostingsFormat extends PostingsFormat { state.segmentInfo.name, postings, state.context, - state.segmentSuffix, + state.codecId, state.termsIndexDivisor); success = true; return ret; @@ -106,13 +107,36 @@ public class Lucene40PostingsFormat extends PostingsFormat { static final String PROX_EXTENSION = "prx"; @Override - public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { - Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); - BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); + public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set files) throws IOException { + StandardPostingsReader.files(dir, segmentInfo, codecID, files); + BlockTreeTermsReader.files(dir, segmentInfo, codecID, files); + DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files); + } + + @Override + public void getExtensions(Set extensions) { + getStandardExtensions(extensions); + } + + public static void getStandardExtensions(Set extensions) { + extensions.add(FREQ_EXTENSION); + extensions.add(PROX_EXTENSION); + BlockTreeTermsReader.getExtensions(extensions); + DefaultDocValuesConsumer.getExtensions(extensions); } @Override public String toString() { - return getName() + "(minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")"; + return name + "(minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")"; + } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new DefaultDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new DefaultDocValuesProducer(state); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java similarity index 97% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java rename to lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java index 5283c56..488c81f 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene40; +package org.apache.lucene.index.codecs.standard; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -42,7 +42,7 @@ import org.apache.lucene.util.CodecUtil; * postings format. * @lucene.experimental */ -public class Lucene40PostingsReader extends PostingsReaderBase { +public class StandardPostingsReader extends PostingsReaderBase { private final IndexInput freqIn; private final IndexInput proxIn; @@ -54,14 +54,14 @@ public class Lucene40PostingsReader extends PostingsReaderBase { // private String segment; - public Lucene40PostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext ioContext, String segmentSuffix) throws IOException { - freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION), + public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext ioContext, int codecId) throws IOException { + freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.FREQ_EXTENSION), ioContext); // this.segment = segmentInfo.name; if (segmentInfo.getHasProx()) { boolean success = false; try { - proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION), + proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.PROX_EXTENSION), ioContext); success = true; } finally { @@ -74,10 +74,10 @@ public class Lucene40PostingsReader extends PostingsReaderBase { } } - public static void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Collection files) throws IOException { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION)); + public static void files(Directory dir, SegmentInfo segmentInfo, int codecID, Collection files) throws IOException { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, StandardCodec.FREQ_EXTENSION)); if (segmentInfo.getHasProx()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, StandardCodec.PROX_EXTENSION)); } } @@ -85,8 +85,8 @@ public class Lucene40PostingsReader extends PostingsReaderBase { public void init(IndexInput termsIn) throws IOException { // Make sure we are talking to the matching past writer - CodecUtil.checkHeader(termsIn, Lucene40PostingsWriter.CODEC, - Lucene40PostingsWriter.VERSION_START, Lucene40PostingsWriter.VERSION_START); + CodecUtil.checkHeader(termsIn, StandardPostingsWriter.CODEC, + StandardPostingsWriter.VERSION_START, StandardPostingsWriter.VERSION_START); skipInterval = termsIn.readInt(); maxSkipLevels = termsIn.readInt(); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java similarity index 96% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java rename to lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java index ef93fc5..1a31693 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene40; +package org.apache.lucene.index.codecs.standard; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -38,8 +38,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CodecUtil; /** @lucene.experimental */ -public final class Lucene40PostingsWriter extends PostingsWriterBase { - final static String CODEC = "Lucene40PostingsWriter"; +public final class StandardPostingsWriter extends PostingsWriterBase { + final static String CODEC = "StandardPostingsWriter"; //private static boolean DEBUG = BlockTreeTermsWriter.DEBUG; @@ -81,21 +81,21 @@ public final class Lucene40PostingsWriter extends PostingsWriterBase { // private String segment; - public Lucene40PostingsWriter(SegmentWriteState state) throws IOException { + public StandardPostingsWriter(SegmentWriteState state) throws IOException { this(state, DEFAULT_SKIP_INTERVAL); } - public Lucene40PostingsWriter(SegmentWriteState state, int skipInterval) throws IOException { + public StandardPostingsWriter(SegmentWriteState state, int skipInterval) throws IOException { super(); this.skipInterval = skipInterval; this.skipMinimum = skipInterval; /* set to the same for now */ // this.segment = state.segmentName; - String fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION); + String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.FREQ_EXTENSION); freqOut = state.directory.createOutput(fileName, state.context); if (state.fieldInfos.hasProx()) { // At least one field does not omit TF, so create the // prox file - fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION); + fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION); proxOut = state.directory.createOutput(fileName, state.context); } else { // Every field omits TF so we will write no prox file diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/package.html b/lucene/src/java/org/apache/lucene/index/codecs/standard/package.html similarity index 100% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene40/package.html rename to lucene/src/java/org/apache/lucene/index/codecs/standard/package.html diff --git a/lucene/src/java/org/apache/lucene/index/values/Bytes.java b/lucene/src/java/org/apache/lucene/index/values/Bytes.java index aac08af..9d942ca 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Bytes.java +++ b/lucene/src/java/org/apache/lucene/index/values/Bytes.java @@ -61,9 +61,6 @@ import org.apache.lucene.util.packed.PackedInts; * @lucene.experimental */ public final class Bytes { - - static final String DV_SEGMENT_SUFFIX = "dv"; - // TODO - add bulk copy where possible private Bytes() { /* don't instantiate! */ } @@ -247,7 +244,7 @@ public final class Bytes { if (datOut == null) { boolean success = false; try { - datOut = dir.createOutput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, + datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", DATA_EXTENSION), context); CodecUtil.writeHeader(datOut, codecName, version); success = true; @@ -272,7 +269,7 @@ public final class Bytes { boolean success = false; try { if (idxOut == null) { - idxOut = dir.createOutput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, + idxOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", INDEX_EXTENSION), context); CodecUtil.writeHeader(idxOut, codecName, version); } @@ -310,10 +307,10 @@ public final class Bytes { @Override public void files(Collection files) throws IOException { assert datOut != null; - files.add(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, DATA_EXTENSION)); + files.add(IndexFileNames.segmentFileName(id, "", DATA_EXTENSION)); if (idxOut != null) { // called after flush - so this must be initialized // if needed or present - final String idxFile = IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, + final String idxFile = IndexFileNames.segmentFileName(id, "", INDEX_EXTENSION); files.add(idxFile); } @@ -337,11 +334,11 @@ public final class Bytes { IndexInput indexIn = null; boolean success = false; try { - dataIn = dir.openInput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, + dataIn = dir.openInput(IndexFileNames.segmentFileName(id, "", Writer.DATA_EXTENSION), context); version = CodecUtil.checkHeader(dataIn, codecName, maxVersion, maxVersion); if (doIndex) { - indexIn = dir.openInput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, + indexIn = dir.openInput(IndexFileNames.segmentFileName(id, "", Writer.INDEX_EXTENSION), context); final int version2 = CodecUtil.checkHeader(indexIn, codecName, maxVersion, maxVersion); @@ -497,7 +494,8 @@ public final class Bytes { protected void releaseResources() { hash.close(); - bytesUsed.addAndGet((-docToEntry.length) * RamUsageEstimator.NUM_BYTES_INT); + bytesUsed + .addAndGet((-docToEntry.length) * RamUsageEstimator.NUM_BYTES_INT); docToEntry = null; } diff --git a/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java b/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java index 6d00aaa..1e1b9d5 100644 --- a/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java +++ b/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java @@ -24,7 +24,8 @@ import org.apache.lucene.document.IndexDocValuesField; import org.apache.lucene.index.Fields; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.BytesRef; /** @@ -39,11 +40,13 @@ import org.apache.lucene.util.BytesRef; * performance {@link IndexDocValues} should be consumed per-segment just like * IndexReader. *

- * {@link IndexDocValues} are fully integrated into the {@link DocValuesFormat} API. + * {@link IndexDocValues} are fully integrated into the {@link Codec} API. + * Custom implementations can be exposed on a per field basis via + * {@link CodecProvider}. * * @see ValueType for limitations and default implementation documentation * @see IndexDocValuesField for adding values to the index - * @see DocValuesFormat#docsConsumer(org.apache.lucene.index.PerDocWriteState) for + * @see Codec#docsConsumer(org.apache.lucene.index.PerDocWriteState) for * customization * @lucene.experimental */ diff --git a/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java b/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java index 97470ef..293558e 100644 --- a/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java +++ b/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java @@ -156,7 +156,7 @@ class PackedIntValues { protected PackedIntsReader(Directory dir, String id, int numDocs, IOContext context) throws IOException { datIn = dir.openInput( - IndexFileNames.segmentFileName(id, Bytes.DV_SEGMENT_SUFFIX, Writer.DATA_EXTENSION), + IndexFileNames.segmentFileName(id, "", Writer.DATA_EXTENSION), context); this.numDocs = numDocs; boolean success = false; diff --git a/lucene/src/java/org/apache/lucene/index/values/ValueType.java b/lucene/src/java/org/apache/lucene/index/values/ValueType.java index 5a267eb..8e4a133 100644 --- a/lucene/src/java/org/apache/lucene/index/values/ValueType.java +++ b/lucene/src/java/org/apache/lucene/index/values/ValueType.java @@ -17,7 +17,7 @@ package org.apache.lucene.index.values; * limitations under the License. */ -import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.util.BytesRef; @@ -27,7 +27,7 @@ import org.apache.lucene.util.packed.PackedInts; * ValueType specifies the {@link IndexDocValues} type for a * certain field. A ValueType only defines the data type for a field * while the actual implementation used to encode and decode the values depends - * on the the {@link DocValuesFormat#docsConsumer} and {@link DocValuesFormat#docsProducer} methods. + * on the the {@link Codec#docsConsumer} and {@link Codec#docsProducer} methods. * * @lucene.experimental */ diff --git a/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java b/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java index cb00e4d..c8ed772 100644 --- a/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java +++ b/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java @@ -106,9 +106,7 @@ public final class CompoundFileDirectory extends Directory { numEntries); for (int i = 0; i < numEntries; i++) { final FileEntry fileEntry = new FileEntry(); - final String id = input.readString(); - assert !mapping.containsKey(id): "id=" + id + " was written multiple times in the CFS"; - mapping.put(id, fileEntry); + mapping.put(input.readString(), fileEntry); fileEntry.offset = input.readLong(); fileEntry.length = input.readLong(); } @@ -172,9 +170,6 @@ public final class CompoundFileDirectory extends Directory { entry = new FileEntry(); entry.offset = offset; - - assert !entries.containsKey(id); - entries.put(id, entry); } @@ -276,7 +271,7 @@ public final class CompoundFileDirectory extends Directory { public long fileLength(String name) throws IOException { ensureOpen(); if (this.writer != null) { - return writer.fileLength(name); + return writer.fileLenght(name); } FileEntry e = entries.get(IndexFileNames.stripSegmentName(name)); if (e == null) @@ -328,9 +323,4 @@ public final class CompoundFileDirectory extends Directory { } }; } - - @Override - public String toString() { - return "CompoundFileDirectory(file=\"" + fileName + "\" in dir=" + directory + ")"; - } } diff --git a/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java b/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java index 37ae0a5..4508172 100644 --- a/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java +++ b/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java @@ -22,11 +22,9 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedList; import java.util.Map; import java.util.Queue; -import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.index.IndexFileNames; @@ -87,7 +85,6 @@ final class CompoundFileWriter implements Closeable{ private final Directory directory; private final Map entries = new HashMap(); - private final Set seenIDs = new HashSet(); // all entries that are written to a sep. file but not yet moved into CFS private final Queue pendingEntries = new LinkedList(); private boolean closed = false; @@ -241,9 +238,6 @@ final class CompoundFileWriter implements Closeable{ final FileEntry entry = new FileEntry(); entry.file = name; entries.put(name, entry); - final String id = IndexFileNames.stripSegmentName(name); - assert !seenIDs.contains(id): "file=\"" + name + "\" maps to id=\"" + id + "\", which was already written"; - seenIDs.add(id); final DirectCFSIndexOutput out; if (outputTaken.compareAndSet(false, true)) { out = new DirectCFSIndexOutput(dataOut, entry, false); @@ -290,7 +284,7 @@ final class CompoundFileWriter implements Closeable{ } } - long fileLength(String name) throws IOException { + long fileLenght(String name) throws IOException { FileEntry fileEntry = entries.get(name); if (fileEntry == null) { throw new FileNotFoundException(name + " does not exist"); diff --git a/lucene/src/java/org/apache/lucene/util/CodecUtil.java b/lucene/src/java/org/apache/lucene/util/CodecUtil.java index f7b2711..f4caa27 100644 --- a/lucene/src/java/org/apache/lucene/util/CodecUtil.java +++ b/lucene/src/java/org/apache/lucene/util/CodecUtil.java @@ -35,7 +35,7 @@ public final class CodecUtil { private final static int CODEC_MAGIC = 0x3fd76c17; - public static void writeHeader(DataOutput out, String codec, int version) + public static DataOutput writeHeader(DataOutput out, String codec, int version) throws IOException { BytesRef bytes = new BytesRef(codec); if (bytes.length != codec.length() || bytes.length >= 128) { @@ -44,6 +44,8 @@ public final class CodecUtil { out.writeInt(CODEC_MAGIC); out.writeString(codec); out.writeInt(version); + + return out; } public static int headerLength(String codec) { diff --git a/lucene/src/java/org/apache/lucene/util/NamedSPILoader.java b/lucene/src/java/org/apache/lucene/util/NamedSPILoader.java deleted file mode 100644 index 0f24daa..0000000 diff --git a/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec b/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec deleted file mode 100644 index 5d8a95e..0000000 diff --git a/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.PostingsFormat b/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.PostingsFormat deleted file mode 100644 index efbc25f..0000000 diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/lucene40ords/Lucene40WithOrds.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/lucene40ords/Lucene40WithOrds.java deleted file mode 100644 index 728c361..0000000 diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepDocValuesFormat.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepDocValuesFormat.java deleted file mode 100644 index 790bca1..0000000 diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/nestedpulsing/NestedPulsingPostingsFormat.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/nestedpulsing/NestedPulsingPostingsFormat.java deleted file mode 100644 index 659aa17..0000000 diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java deleted file mode 100644 index f259b54..0000000 diff --git a/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java b/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java rename to lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/analysis/CollationTestBase.java b/lucene/src/test-framework/org/apache/lucene/analysis/CollationTestBase.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/analysis/CollationTestBase.java rename to lucene/src/test-framework/org/apache/lucene/analysis/CollationTestBase.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/analysis/MockAnalyzer.java b/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/analysis/MockAnalyzer.java rename to lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java b/lucene/src/test-framework/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java rename to lucene/src/test-framework/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java b/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java rename to lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenFilter.java b/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenFilter.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenFilter.java rename to lucene/src/test-framework/org/apache/lucene/analysis/MockTokenFilter.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java b/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java rename to lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java b/lucene/src/test-framework/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java rename to lucene/src/test-framework/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/analysis/VocabularyAssert.java b/lucene/src/test-framework/org/apache/lucene/analysis/VocabularyAssert.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/analysis/VocabularyAssert.java rename to lucene/src/test-framework/org/apache/lucene/analysis/VocabularyAssert.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/DocHelper.java b/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/index/DocHelper.java rename to lucene/src/test-framework/org/apache/lucene/index/DocHelper.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/MockIndexInput.java b/lucene/src/test-framework/org/apache/lucene/index/MockIndexInput.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/index/MockIndexInput.java rename to lucene/src/test-framework/org/apache/lucene/index/MockIndexInput.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/MockRandomMergePolicy.java b/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/index/MockRandomMergePolicy.java rename to lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java b/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java similarity index 43% rename from lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java rename to lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java index 7216390..b92ce7a 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java +++ b/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java @@ -25,79 +25,83 @@ import java.util.Locale; import java.util.Map; import java.util.Random; -import org.apache.lucene.index.codecs.PostingsFormat; -import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; -import org.apache.lucene.index.codecs.lucene40ords.Lucene40WithOrds; -import org.apache.lucene.index.codecs.memory.MemoryPostingsFormat; -import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockPostingsFormat; -import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockPostingsFormat; -import org.apache.lucene.index.codecs.mockrandom.MockRandomPostingsFormat; -import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; -import org.apache.lucene.index.codecs.nestedpulsing.NestedPulsingPostingsFormat; -import org.apache.lucene.index.codecs.pulsing.Pulsing40PostingsFormat; -import org.apache.lucene.index.codecs.simpletext.SimpleTextPostingsFormat; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.memory.MemoryCodec; +import org.apache.lucene.index.codecs.preflex.PreFlexCodec; +import org.apache.lucene.index.codecs.pulsing.PulsingCodec; +import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; +import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.util._TestUtil; /** - * Codec that assigns per-field random postings formats. + * CodecProvider that assigns per-field random codecs. *

- * The same field/format assignment will happen regardless of order, + * The same field/codec assignment will happen regardless of order, * a hash is computed up front that determines the mapping. * This means fields can be put into things like HashSets and added to * documents in different orders and the test will still be deterministic * and reproducable. */ -public class RandomCodec extends Lucene40Codec { - /** shuffled list of postingsformats to use for new mappings */ - private List formats = new ArrayList(); - /** memorized field->postingsformat mappings */ - private Map previousMappings = new HashMap(); +public class RandomCodecProvider extends CodecProvider { + private List knownCodecs = new ArrayList(); + private Map previousMappings = new HashMap(); private final int perFieldSeed; - - @Override - public PostingsFormat getPostingsFormatForField(String name) { - PostingsFormat codec = previousMappings.get(name); - if (codec == null) { - codec = formats.get(Math.abs(perFieldSeed ^ name.hashCode()) % formats.size()); - if (codec instanceof SimpleTextPostingsFormat && perFieldSeed % 5 != 0) { - // make simpletext rarer, choose again - codec = formats.get(Math.abs(perFieldSeed ^ name.toUpperCase(Locale.ENGLISH).hashCode()) % formats.size()); - } - previousMappings.put(name, codec); - // Safety: - assert previousMappings.size() < 10000; - } - return codec; - } - - public RandomCodec(Random random, boolean useNoMemoryExpensiveCodec) { + + public RandomCodecProvider(Random random, boolean useNoMemoryExpensiveCodec) { this.perFieldSeed = random.nextInt(); // TODO: make it possible to specify min/max iterms per // block via CL: int minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); int maxItemsPerBlock = 2*(Math.max(2, minItemsPerBlock-1)) + random.nextInt(100); - formats.add(new Lucene40PostingsFormat(minItemsPerBlock, maxItemsPerBlock)); + register(new StandardCodec(minItemsPerBlock, maxItemsPerBlock)); + register(new PreFlexCodec()); // TODO: make it possible to specify min/max iterms per // block via CL: minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); maxItemsPerBlock = 2*(Math.max(1, minItemsPerBlock-1)) + random.nextInt(100); - formats.add(new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock)); - formats.add(new MockSepPostingsFormat()); - formats.add(new MockFixedIntBlockPostingsFormat(_TestUtil.nextInt(random, 1, 2000))); - formats.add(new MockVariableIntBlockPostingsFormat( _TestUtil.nextInt(random, 1, 127))); - formats.add(new MockRandomPostingsFormat(random)); - formats.add(new NestedPulsingPostingsFormat()); - formats.add(new Lucene40WithOrds()); + register(new PulsingCodec( 1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock)); if (!useNoMemoryExpensiveCodec) { - formats.add(new SimpleTextPostingsFormat()); - formats.add(new MemoryPostingsFormat()); + register(new SimpleTextCodec()); + register(new MemoryCodec()); } - Collections.shuffle(formats, random); + Collections.shuffle(knownCodecs, random); + } + + @Override + public synchronized void register(Codec codec) { + if (!codec.name.equals("PreFlex")) + knownCodecs.add(codec); + super.register(codec); + } + + @Override + public synchronized void unregister(Codec codec) { + knownCodecs.remove(codec); + super.unregister(codec); + } + + @Override + public synchronized String getFieldCodec(String name) { + Codec codec = previousMappings.get(name); + if (codec == null) { + codec = knownCodecs.get(Math.abs(perFieldSeed ^ name.hashCode()) % knownCodecs.size()); + if (codec instanceof SimpleTextCodec && perFieldSeed % 5 != 0) { + // make simpletext rarer, choose again + codec = knownCodecs.get(Math.abs(perFieldSeed ^ name.toUpperCase(Locale.ENGLISH).hashCode()) % knownCodecs.size()); + } + previousMappings.put(name, codec); + } + return codec.name; + } + + @Override + public synchronized boolean hasFieldCodec(String name) { + return true; // we have a codec for every field } @Override - public String toString() { - return super.toString() + ": " + previousMappings.toString(); + public synchronized String toString() { + return "RandomCodecProvider: " + previousMappings.toString(); } } diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java b/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java similarity index 96% rename from lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java rename to lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java index 6abf908..9e028ce 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java +++ b/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java @@ -27,7 +27,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.IndexDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexWriter; // javadoc -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; @@ -53,7 +53,7 @@ public class RandomIndexWriter implements Closeable { private final int fixedBytesLength; private final long docValuesFieldPrefix; private volatile boolean doDocValues; - private final Codec codec; // sugar + private CodecProvider codecProvider; // Randomly calls Thread.yield so we mixup thread scheduling private static final class MockIndexWriter extends IndexWriter { @@ -96,10 +96,9 @@ public class RandomIndexWriter implements Closeable { this.r = r; w = new MockIndexWriter(r, dir, c); flushAt = _TestUtil.nextInt(r, 10, 1000); - codec = w.getConfig().getCodec(); if (LuceneTestCase.VERBOSE) { System.out.println("RIW config=" + w.getConfig()); - System.out.println("codec default=" + codec.getName()); + System.out.println("codec default=" + w.getConfig().getCodecProvider().getDefaultFieldCodec()); w.setInfoStream(System.out); } /* TODO: find some what to make that random... @@ -110,6 +109,7 @@ public class RandomIndexWriter implements Closeable { */ fixedBytesLength = 37; docValuesFieldPrefix = r.nextLong(); + codecProvider = w.getConfig().getCodecProvider(); switchDoDocValues(); } @@ -171,7 +171,7 @@ public class RandomIndexWriter implements Closeable { ValueType[] values = ValueType.values(); ValueType type = values[random.nextInt(values.length)]; String name = "random_" + type.name() + "" + docValuesFieldPrefix; - if ("Lucene3x".equals(codec.getName()) || doc.getField(name) != null) + if ("PreFlex".equals(codecProvider.getFieldCodec(name)) || doc.getField(name) != null) return; IndexDocValuesField docValuesField = new IndexDocValuesField(name); switch (type) { @@ -367,7 +367,7 @@ public class RandomIndexWriter implements Closeable { // If we are writing with PreFlexRW, force a full // IndexReader.open so terms are sorted in codepoint // order during searching: - if (!applyDeletions || !codec.getName().equals("Lucene3x") && r.nextBoolean()) { + if (!applyDeletions || !w.codecs.getDefaultFieldCodec().equals("PreFlex") && r.nextBoolean()) { if (LuceneTestCase.VERBOSE) { System.out.println("RIW.getReader: use NRT reader"); } @@ -382,7 +382,7 @@ public class RandomIndexWriter implements Closeable { w.commit(); switchDoDocValues(); if (r.nextBoolean()) { - return IndexReader.open(w.getDirectory(), new KeepOnlyLastCommitDeletionPolicy(), r.nextBoolean(), _TestUtil.nextInt(r, 1, 10)); + return IndexReader.open(w.getDirectory(), new KeepOnlyLastCommitDeletionPolicy(), r.nextBoolean(), _TestUtil.nextInt(r, 1, 10), w.getConfig().getCodecProvider()); } else { return w.getReader(applyDeletions); } diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java b/lucene/src/test-framework/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java similarity index 98% rename from lucene/src/test-framework/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java rename to lucene/src/test-framework/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java index 8d3fb5b..425fe7b 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java +++ b/lucene/src/test-framework/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java @@ -35,6 +35,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; @@ -431,6 +432,12 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas final long t0 = System.currentTimeMillis(); + final String defaultCodec = CodecProvider.getDefault().getDefaultFieldCodec(); + if (defaultCodec.equals("SimpleText") || defaultCodec.equals("Memory")) { + // no + CodecProvider.getDefault().setDefaultFieldCodec("Standard"); + } + final LineFileDocs docs = new LineFileDocs(random); final File tempDir = _TestUtil.getTempDir(testName); dir = newFSDirectory(tempDir); diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java similarity index 79% rename from lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java rename to lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java index 427ebb2..ee471e5 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java @@ -20,22 +20,27 @@ package org.apache.lucene.index.codecs.mockintblock; import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.sep.IntStreamFactory; import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.index.codecs.sep.IntIndexOutput; +import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; +import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; import org.apache.lucene.index.codecs.sep.SepPostingsReader; import org.apache.lucene.index.codecs.sep.SepPostingsWriter; +import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput; import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.BlockTermsReader; @@ -52,22 +57,18 @@ import org.apache.lucene.util.IOUtils; * used here just writes each block as a series of vInt. */ -public class MockFixedIntBlockPostingsFormat extends PostingsFormat { +public class MockFixedIntBlockCodec extends Codec { private final int blockSize; - public MockFixedIntBlockPostingsFormat() { - this(1); - } - - public MockFixedIntBlockPostingsFormat(int blockSize) { + public MockFixedIntBlockCodec(int blockSize) { super("MockFixedIntBlock"); this.blockSize = blockSize; } @Override public String toString() { - return getName() + "(blockSize=" + blockSize + ")"; + return name + "(blockSize=" + blockSize + ")"; } // only for testing @@ -160,7 +161,7 @@ public class MockFixedIntBlockPostingsFormat extends PostingsFormat { PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, state.segmentInfo, state.context, - new MockIntFactory(blockSize), state.segmentSuffix); + new MockIntFactory(blockSize), state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -169,7 +170,7 @@ public class MockFixedIntBlockPostingsFormat extends PostingsFormat { state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator(), state.segmentSuffix, + BytesRef.getUTF8SortedAsUnicodeComparator(), state.codecId, IOContext.DEFAULT); success = true; } finally { @@ -186,8 +187,8 @@ public class MockFixedIntBlockPostingsFormat extends PostingsFormat { state.segmentInfo.name, postingsReader, state.context, - Lucene40PostingsFormat.TERMS_CACHE_SIZE, - state.segmentSuffix); + StandardCodec.TERMS_CACHE_SIZE, + state.codecId); success = true; return ret; } finally { @@ -202,9 +203,28 @@ public class MockFixedIntBlockPostingsFormat extends PostingsFormat { } @Override - public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { - SepPostingsReader.files(segmentInfo, segmentSuffix, files); - BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { + SepPostingsReader.files(segmentInfo, codecId, files); + BlockTermsReader.files(dir, segmentInfo, codecId, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); + SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); + } + + @Override + public void getExtensions(Set extensions) { + SepPostingsWriter.getExtensions(extensions); + BlockTermsReader.getExtensions(extensions); + FixedGapTermsIndexReader.getIndexExtensions(extensions); + SepDocValuesConsumer.getExtensions(extensions); + } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new SepDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new SepDocValuesProducer(state); } } diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java similarity index 81% rename from lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java rename to lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java index 6e3cde7..d0a0c0b 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java @@ -20,22 +20,27 @@ package org.apache.lucene.index.codecs.mockintblock; import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.sep.IntStreamFactory; import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.index.codecs.sep.IntIndexOutput; +import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; +import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; import org.apache.lucene.index.codecs.sep.SepPostingsReader; import org.apache.lucene.index.codecs.sep.SepPostingsWriter; +import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput; import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.BlockTermsReader; @@ -56,21 +61,17 @@ import org.apache.lucene.util.IOUtils; * int is <= 3, else 2*baseBlockSize. */ -public class MockVariableIntBlockPostingsFormat extends PostingsFormat { +public class MockVariableIntBlockCodec extends Codec { private final int baseBlockSize; - public MockVariableIntBlockPostingsFormat() { - this(1); - } - - public MockVariableIntBlockPostingsFormat(int baseBlockSize) { + public MockVariableIntBlockCodec(int baseBlockSize) { super("MockVariableIntBlock"); this.baseBlockSize = baseBlockSize; } @Override public String toString() { - return getName() + "(baseBlockSize="+ baseBlockSize + ")"; + return name + "(baseBlockSize="+ baseBlockSize + ")"; } public static class MockIntFactory extends IntStreamFactory { @@ -183,7 +184,7 @@ public class MockVariableIntBlockPostingsFormat extends PostingsFormat { PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, state.segmentInfo, state.context, - new MockIntFactory(baseBlockSize), state.segmentSuffix); + new MockIntFactory(baseBlockSize), state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -193,7 +194,7 @@ public class MockVariableIntBlockPostingsFormat extends PostingsFormat { state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.segmentSuffix, state.context); + state.codecId, state.context); success = true; } finally { if (!success) { @@ -209,8 +210,8 @@ public class MockVariableIntBlockPostingsFormat extends PostingsFormat { state.segmentInfo.name, postingsReader, state.context, - Lucene40PostingsFormat.TERMS_CACHE_SIZE, - state.segmentSuffix); + StandardCodec.TERMS_CACHE_SIZE, + state.codecId); success = true; return ret; } finally { @@ -225,9 +226,28 @@ public class MockVariableIntBlockPostingsFormat extends PostingsFormat { } @Override - public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { - SepPostingsReader.files(segmentInfo, segmentSuffix, files); - BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { + SepPostingsReader.files(segmentInfo, codecId, files); + BlockTermsReader.files(dir, segmentInfo, codecId, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); + SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); + } + + @Override + public void getExtensions(Set extensions) { + SepPostingsWriter.getExtensions(extensions); + BlockTermsReader.getExtensions(extensions); + FixedGapTermsIndexReader.getIndexExtensions(extensions); + SepDocValuesConsumer.getExtensions(extensions); + } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new SepDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new SepDocValuesProducer(state); } } diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockrandom/MockRandomPostingsFormat.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java similarity index 81% rename from lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockrandom/MockRandomPostingsFormat.java rename to lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java index 898229f..9cd824c 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockrandom/MockRandomPostingsFormat.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java @@ -26,6 +26,7 @@ import java.util.Set; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; @@ -33,11 +34,15 @@ import org.apache.lucene.index.codecs.BlockTreeTermsReader; import org.apache.lucene.index.codecs.BlockTreeTermsWriter; import org.apache.lucene.index.codecs.BlockTermsReader; import org.apache.lucene.index.codecs.BlockTermsWriter; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesProducer; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; +import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.TermStats; @@ -45,18 +50,20 @@ import org.apache.lucene.index.codecs.TermsIndexReaderBase; import org.apache.lucene.index.codecs.TermsIndexWriterBase; import org.apache.lucene.index.codecs.VariableGapTermsIndexReader; import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; -import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockPostingsFormat; -import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec; +import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec; import org.apache.lucene.index.codecs.mocksep.MockSingleIntFactory; import org.apache.lucene.index.codecs.pulsing.PulsingPostingsReader; import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriter; import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.index.codecs.sep.IntIndexOutput; import org.apache.lucene.index.codecs.sep.IntStreamFactory; +import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; +import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; import org.apache.lucene.index.codecs.sep.SepPostingsReader; import org.apache.lucene.index.codecs.sep.SepPostingsWriter; +import org.apache.lucene.index.codecs.standard.StandardPostingsReader; +import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -69,17 +76,14 @@ import org.apache.lucene.util._TestUtil; * Randomly combines terms index impl w/ postings impls. */ -public class MockRandomPostingsFormat extends PostingsFormat { +public class MockRandomCodec extends Codec { + private final boolean useSepDocValues; private final Random seedRandom; private final String SEED_EXT = "sd"; - public MockRandomPostingsFormat() { - // just for reading, we are gonna setSeed from the .seed file... right? - this(new Random()); - } - - public MockRandomPostingsFormat(Random random) { + public MockRandomCodec(Random random) { super("MockRandom"); + this.useSepDocValues = random.nextBoolean(); this.seedRandom = new Random(random.nextLong()); } @@ -92,9 +96,9 @@ public class MockRandomPostingsFormat extends PostingsFormat { salt = random.nextInt(); delegates.add(new MockSingleIntFactory()); final int blockSize = _TestUtil.nextInt(random, 1, 2000); - delegates.add(new MockFixedIntBlockPostingsFormat.MockIntFactory(blockSize)); + delegates.add(new MockFixedIntBlockCodec.MockIntFactory(blockSize)); final int baseBlockSize = _TestUtil.nextInt(random, 1, 127); - delegates.add(new MockVariableIntBlockPostingsFormat.MockIntFactory(baseBlockSize)); + delegates.add(new MockVariableIntBlockCodec.MockIntFactory(baseBlockSize)); // TODO: others } @@ -138,10 +142,10 @@ public class MockRandomPostingsFormat extends PostingsFormat { final long seed = seedRandom.nextLong(); if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: writing to seg=" + state.segmentName + " formatID=" + state.segmentSuffix + " seed=" + seed); + System.out.println("MockRandomCodec: writing to seg=" + state.segmentName + " codecID=" + state.codecId + " seed=" + seed); } - final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, SEED_EXT); + final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SEED_EXT); final IndexOutput out = state.directory.createOutput(seedFileName, state.context); try { out.writeLong(seed); @@ -160,7 +164,7 @@ public class MockRandomPostingsFormat extends PostingsFormat { if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: writing Standard postings"); } - postingsWriter = new Lucene40PostingsWriter(state, skipInterval); + postingsWriter = new StandardPostingsWriter(state, skipInterval); } if (random.nextBoolean()) { @@ -272,11 +276,11 @@ public class MockRandomPostingsFormat extends PostingsFormat { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, SEED_EXT); + final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, SEED_EXT); final IndexInput in = state.dir.openInput(seedFileName, state.context); final long seed = in.readLong(); if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " formatID=" + state.segmentSuffix + " seed=" + seed); + System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " codecID=" + state.codecId + " seed=" + seed); } in.close(); @@ -294,12 +298,12 @@ public class MockRandomPostingsFormat extends PostingsFormat { System.out.println("MockRandomCodec: reading Sep postings"); } postingsReader = new SepPostingsReader(state.dir, state.segmentInfo, - state.context, new MockIntStreamFactory(random), state.segmentSuffix); + state.context, new MockIntStreamFactory(random), state.codecId); } else { if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: reading Standard postings"); } - postingsReader = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + postingsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); } if (random.nextBoolean()) { @@ -325,7 +329,7 @@ public class MockRandomPostingsFormat extends PostingsFormat { state.segmentInfo.name, postingsReader, state.context, - state.segmentSuffix, + state.codecId, state.termsIndexDivisor); success = true; } finally { @@ -359,7 +363,7 @@ public class MockRandomPostingsFormat extends PostingsFormat { state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.segmentSuffix, state.context); + state.codecId, state.context); } else { final int n2 = random.nextInt(3); if (n2 == 1) { @@ -374,7 +378,7 @@ public class MockRandomPostingsFormat extends PostingsFormat { state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - state.segmentSuffix, state.context); + state.codecId, state.context); } @@ -396,7 +400,7 @@ public class MockRandomPostingsFormat extends PostingsFormat { postingsReader, state.context, termsCacheSize, - state.segmentSuffix); + state.codecId); success = true; } finally { if (!success) { @@ -413,15 +417,20 @@ public class MockRandomPostingsFormat extends PostingsFormat { } @Override - public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { - final String seedFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SEED_EXT); + public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { + final String seedFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SEED_EXT); files.add(seedFileName); - SepPostingsReader.files(segmentInfo, segmentSuffix, files); - Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); - BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); - BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); - VariableGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + SepPostingsReader.files(segmentInfo, codecId, files); + StandardPostingsReader.files(dir, segmentInfo, codecId, files); + BlockTermsReader.files(dir, segmentInfo, codecId, files); + BlockTreeTermsReader.files(dir, segmentInfo, codecId, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); + VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files); + if (useSepDocValues) { + SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); + } else { + DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files); + } // hackish! Iterator it = files.iterator(); while(it.hasNext()) { @@ -432,4 +441,39 @@ public class MockRandomPostingsFormat extends PostingsFormat { } //System.out.println("MockRandom.files return " + files); } + + @Override + public void getExtensions(Set extensions) { + SepPostingsWriter.getExtensions(extensions); + BlockTermsReader.getExtensions(extensions); + BlockTreeTermsReader.getExtensions(extensions); + FixedGapTermsIndexReader.getIndexExtensions(extensions); + VariableGapTermsIndexReader.getIndexExtensions(extensions); + if (useSepDocValues) { + SepDocValuesConsumer.getExtensions(extensions); + } else { + DefaultDocValuesConsumer.getExtensions(extensions); + } + extensions.add(SEED_EXT); + //System.out.println("MockRandom.getExtensions return " + extensions); + } + + // can we make this more evil? + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + if (useSepDocValues) { + return new SepDocValuesConsumer(state); + } else { + return new DefaultDocValuesConsumer(state); + } + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + if (useSepDocValues) { + return new SepDocValuesProducer(state); + } else { + return new DefaultDocValuesProducer(state); + } + } } diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepPostingsFormat.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java similarity index 70% rename from lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepPostingsFormat.java rename to lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java index 1e8baa1..0b24ed4 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepPostingsFormat.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java @@ -20,21 +20,26 @@ package org.apache.lucene.index.codecs.mocksep; import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.BlockTermsReader; import org.apache.lucene.index.codecs.BlockTermsWriter; import org.apache.lucene.index.codecs.TermsIndexReaderBase; import org.apache.lucene.index.codecs.TermsIndexWriterBase; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; +import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; import org.apache.lucene.index.codecs.sep.SepPostingsWriter; import org.apache.lucene.index.codecs.sep.SepPostingsReader; import org.apache.lucene.store.Directory; @@ -46,9 +51,9 @@ import org.apache.lucene.util.BytesRef; * This is here just to test the core sep codec * classes. */ -public class MockSepPostingsFormat extends PostingsFormat { +public class MockSepCodec extends Codec { - public MockSepPostingsFormat() { + public MockSepCodec() { super("MockSep"); } @@ -88,7 +93,7 @@ public class MockSepPostingsFormat extends PostingsFormat { public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, state.segmentInfo, - state.context, new MockSingleIntFactory(), state.segmentSuffix); + state.context, new MockSingleIntFactory(), state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -98,7 +103,7 @@ public class MockSepPostingsFormat extends PostingsFormat { state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.segmentSuffix, state.context); + state.codecId, state.context); success = true; } finally { if (!success) { @@ -114,8 +119,8 @@ public class MockSepPostingsFormat extends PostingsFormat { state.segmentInfo.name, postingsReader, state.context, - Lucene40PostingsFormat.TERMS_CACHE_SIZE, - state.segmentSuffix); + StandardCodec.TERMS_CACHE_SIZE, + state.codecId); success = true; return ret; } finally { @@ -130,9 +135,32 @@ public class MockSepPostingsFormat extends PostingsFormat { } @Override - public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { - SepPostingsReader.files(segmentInfo, segmentSuffix, files); - BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { + SepPostingsReader.files(segmentInfo, codecId, files); + BlockTermsReader.files(dir, segmentInfo, codecId, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); + SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); + } + + @Override + public void getExtensions(Set extensions) { + getSepExtensions(extensions); + } + + public static void getSepExtensions(Set extensions) { + SepPostingsWriter.getExtensions(extensions); + BlockTermsReader.getExtensions(extensions); + FixedGapTermsIndexReader.getIndexExtensions(extensions); + SepDocValuesConsumer.getExtensions(extensions); + } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new SepDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new SepDocValuesProducer(state); } } diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java rename to lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java rename to lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java rename to lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java similarity index 95% rename from lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java rename to lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java index ca3c45e..05bbbfb 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java @@ -29,9 +29,9 @@ import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.PostingsConsumer; import org.apache.lucene.index.codecs.TermStats; import org.apache.lucene.index.codecs.TermsConsumer; -import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; -import org.apache.lucene.index.codecs.lucene3x.TermInfo; -import org.apache.lucene.index.codecs.lucene40.DefaultSkipListWriter; +import org.apache.lucene.index.codecs.preflex.PreFlexCodec; +import org.apache.lucene.index.codecs.preflex.TermInfo; +import org.apache.lucene.index.codecs.standard.DefaultSkipListWriter; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; @@ -50,12 +50,12 @@ class PreFlexFieldsWriter extends FieldsConsumer { state.fieldInfos, state.termIndexInterval); - final String freqFile = IndexFileNames.segmentFileName(state.segmentName, "", Lucene3xPostingsFormat.FREQ_EXTENSION); + final String freqFile = IndexFileNames.segmentFileName(state.segmentName, "", PreFlexCodec.FREQ_EXTENSION); freqOut = state.directory.createOutput(freqFile, state.context); totalNumDocs = state.numDocs; if (state.fieldInfos.hasProx()) { - final String proxFile = IndexFileNames.segmentFileName(state.segmentName, "", Lucene3xPostingsFormat.PROX_EXTENSION); + final String proxFile = IndexFileNames.segmentFileName(state.segmentName, "", PreFlexCodec.PROX_EXTENSION); proxOut = state.directory.createOutput(proxFile, state.context); } else { proxOut = null; diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWPostingsFormat.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java similarity index 87% rename from lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWPostingsFormat.java rename to lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java index 633ea47..f911ef2 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWPostingsFormat.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java @@ -21,8 +21,8 @@ import java.io.IOException; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.lucene3x.Lucene3xFields; -import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; +import org.apache.lucene.index.codecs.preflex.PreFlexCodec; +import org.apache.lucene.index.codecs.preflex.PreFlexFields; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.util.LuceneTestCase; @@ -32,9 +32,9 @@ import org.apache.lucene.util.LuceneTestCase; * * @lucene.experimental */ -public class PreFlexRWPostingsFormat extends Lucene3xPostingsFormat { +public class PreFlexRWCodec extends PreFlexCodec { - public PreFlexRWPostingsFormat() { + public PreFlexRWCodec() { // NOTE: we impersonate the PreFlex codec so that it can // read the segments we write! } @@ -50,7 +50,7 @@ public class PreFlexRWPostingsFormat extends Lucene3xPostingsFormat { // Whenever IW opens readers, eg for merging, we have to // keep terms order in UTF16: - return new Lucene3xFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor) { + return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor) { @Override protected boolean sortTermsByUnicode() { // We carefully peek into stack track above us: if diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java similarity index 95% rename from lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java rename to lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java index 8ca9be5..91e07a1 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java @@ -23,8 +23,8 @@ import java.io.IOException; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; -import org.apache.lucene.index.codecs.lucene3x.TermInfo; +import org.apache.lucene.index.codecs.preflex.PreFlexCodec; +import org.apache.lucene.index.codecs.preflex.TermInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -106,8 +106,8 @@ final class TermInfosWriter implements Closeable { try { directory.deleteFile(IndexFileNames.segmentFileName(segment, "", - (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION - : Lucene3xPostingsFormat.TERMS_EXTENSION))); + (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION + : PreFlexCodec.TERMS_EXTENSION))); } catch (IOException ignored) { } } @@ -125,8 +125,8 @@ final class TermInfosWriter implements Closeable { fieldInfos = fis; isIndex = isi; output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", - (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION - : Lucene3xPostingsFormat.TERMS_EXTENSION)), IOContext.DEFAULT); + (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION + : PreFlexCodec.TERMS_EXTENSION)), IOContext.DEFAULT); boolean success = false; try { output.writeInt(FORMAT_CURRENT); // write format @@ -147,8 +147,8 @@ final class TermInfosWriter implements Closeable { try { directory.deleteFile(IndexFileNames.segmentFileName(segment, "", - (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION - : Lucene3xPostingsFormat.TERMS_EXTENSION))); + (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION + : PreFlexCodec.TERMS_EXTENSION))); } catch (IOException ignored) { } } diff --git a/lucene/src/test-framework/java/org/apache/lucene/search/AssertingIndexSearcher.java b/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/search/AssertingIndexSearcher.java rename to lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/search/CachingWrapperFilterHelper.java b/lucene/src/test-framework/org/apache/lucene/search/CachingWrapperFilterHelper.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/search/CachingWrapperFilterHelper.java rename to lucene/src/test-framework/org/apache/lucene/search/CachingWrapperFilterHelper.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java b/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java rename to lucene/src/test-framework/org/apache/lucene/search/CheckHits.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java b/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java rename to lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/search/RandomSimilarityProvider.java b/lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/search/RandomSimilarityProvider.java rename to lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java similarity index 97% rename from lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java rename to lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java index c4c00fa..a4cd957 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java @@ -33,6 +33,7 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.ThrottledIndexOutput; import org.apache.lucene.util._TestUtil; @@ -485,8 +486,14 @@ public class MockDirectoryWrapper extends Directory { if (LuceneTestCase.VERBOSE) { System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex"); } - if (IndexReader.indexExists(this)) { - _TestUtil.checkIndex(this); + if (codecProvider != null) { + if (IndexReader.indexExists(this, codecProvider)) { + _TestUtil.checkIndex(this, codecProvider); + } + } else { + if (IndexReader.indexExists(this)) { + _TestUtil.checkIndex(this); + } } } delegate.close(); @@ -516,6 +523,13 @@ public class MockDirectoryWrapper extends Directory { public synchronized void removeIndexInput(IndexInput in, String name) { removeOpenFile(in, name); } + + private CodecProvider codecProvider; + + // We pass this CodecProvider to checkIndex when dir is closed... + public void setCodecProvider(CodecProvider cp) { + codecProvider = cp; + } boolean open = true; diff --git a/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexInputWrapper.java b/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/store/MockIndexInputWrapper.java rename to lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexOutputWrapper.java b/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/store/MockIndexOutputWrapper.java rename to lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/store/MockLockFactoryWrapper.java b/lucene/src/test-framework/org/apache/lucene/store/MockLockFactoryWrapper.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/store/MockLockFactoryWrapper.java rename to lucene/src/test-framework/org/apache/lucene/store/MockLockFactoryWrapper.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/store/_TestHelper.java b/lucene/src/test-framework/org/apache/lucene/store/_TestHelper.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/store/_TestHelper.java rename to lucene/src/test-framework/org/apache/lucene/store/_TestHelper.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/LineFileDocs.java b/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/util/LineFileDocs.java rename to lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitDividingSelector.java b/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitDividingSelector.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitDividingSelector.java rename to lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitDividingSelector.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitResultFormatter.java b/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitResultFormatter.java similarity index 98% rename from lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitResultFormatter.java rename to lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitResultFormatter.java index a03f780..c67b9bc 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitResultFormatter.java +++ b/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitResultFormatter.java @@ -66,7 +66,8 @@ public class LuceneJUnitResultFormatter implements JUnitResultFormatter { private static final org.apache.lucene.store.Lock lock; static { - File lockDir = new File(System.getProperty("java.io.tmpdir"), + File lockDir = new File( + System.getProperty("tests.lockdir", System.getProperty("java.io.tmpdir")), "lucene_junit_lock"); lockDir.mkdirs(); if (!lockDir.exists()) { diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java similarity index 87% rename from lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java rename to lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java index 37bb746..826e165 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java @@ -40,18 +40,14 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.*; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.PostingsFormat; -import org.apache.lucene.index.codecs.lucene3x.Lucene3xCodec; -import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; -import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; -import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockPostingsFormat; -import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockPostingsFormat; -import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; -import org.apache.lucene.index.codecs.mockrandom.MockRandomPostingsFormat; -import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec; +import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec; +import org.apache.lucene.index.codecs.mocksep.MockSepCodec; +import org.apache.lucene.index.codecs.mockrandom.MockRandomCodec; +import org.apache.lucene.index.codecs.preflex.PreFlexCodec; import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; -import org.apache.lucene.index.codecs.preflexrw.PreFlexRWPostingsFormat; -import org.apache.lucene.index.codecs.pulsing.PulsingPostingsFormat; +import org.apache.lucene.index.codecs.pulsing.PulsingCodec; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.FieldCache.CacheEntry; @@ -139,8 +135,10 @@ public abstract class LuceneTestCase extends Assert { // by default we randomly pick a different codec for // each test case (non-J4 tests) and each test class (J4 // tests) - /** Gets the postingsFormat to run tests with. */ - public static final String TEST_POSTINGSFORMAT = System.getProperty("tests.postingsformat", "random"); + /** Gets the codec to run tests with. */ + public static final String TEST_CODEC = System.getProperty("tests.codec", "randomPerField"); + /** Gets the codecprovider to run tests with */ + public static final String TEST_CODECPROVIDER = System.getProperty("tests.codecprovider", "random"); /** Gets the locale to run tests with */ public static final String TEST_LOCALE = System.getProperty("tests.locale", "random"); /** Gets the timezone to run tests with */ @@ -170,9 +168,6 @@ public abstract class LuceneTestCase extends Assert { */ public static final int RANDOM_MULTIPLIER = Integer.parseInt(System.getProperty("tests.multiplier", "1")); - /** @lucene.internal */ - public static boolean PREFLEX_IMPERSONATION_IS_ACTIVE; - private int savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount(); private volatile Thread.UncaughtExceptionHandler savedUncaughtExceptionHandler = null; @@ -213,8 +208,12 @@ public abstract class LuceneTestCase extends Assert { } private List uncaughtExceptions = Collections.synchronizedList(new ArrayList()); - // default codec - private static Codec savedCodec; + // saves default codec: we do this statically as many build indexes in @beforeClass + private static String savedDefaultCodec; + // default codec: not set when we use a per-field provider. + private static Codec codec; + // default codec provider + private static CodecProvider savedCodecProvider; private static SimilarityProvider similarityProvider; @@ -225,6 +224,96 @@ public abstract class LuceneTestCase extends Assert { protected static Map stores; + private static final String[] TEST_CODECS = new String[] {"MockSep", "MockFixedIntBlock", "MockVariableIntBlock", "MockRandom"}; + + private static void swapCodec(Codec c, CodecProvider cp) { + Codec prior = null; + try { + prior = cp.lookup(c.name); + } catch (IllegalArgumentException iae) { + } + if (prior != null) { + cp.unregister(prior); + } + cp.register(c); + } + + // returns current default codec + static Codec installTestCodecs(String codec, CodecProvider cp) { + savedDefaultCodec = cp.getDefaultFieldCodec(); + + final boolean codecHasParam; + int codecParam = 0; + if (codec.equals("randomPerField")) { + // lie + codec = "Standard"; + codecHasParam = false; + } else if (codec.equals("random")) { + codec = pickRandomCodec(random); + codecHasParam = false; + } else { + Matcher m = codecWithParam.matcher(codec); + if (m.matches()) { + // codec has a fixed param + codecHasParam = true; + codec = m.group(1); + codecParam = Integer.parseInt(m.group(2)); + } else { + codecHasParam = false; + } + } + + cp.setDefaultFieldCodec(codec); + + if (codec.equals("PreFlex")) { + // If we're running w/ PreFlex codec we must swap in the + // test-only PreFlexRW codec (since core PreFlex can + // only read segments): + swapCodec(new PreFlexRWCodec(), cp); + } + + swapCodec(new MockSepCodec(), cp); + // TODO: make it possible to specify min/max iterms per + // block via CL: + int minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); + int maxItemsPerBlock = 2*(Math.max(2, minItemsPerBlock-1)) + random.nextInt(100); + swapCodec(new PulsingCodec(codecHasParam && "Pulsing".equals(codec) ? codecParam : 1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock), cp); + swapCodec(new MockFixedIntBlockCodec(codecHasParam && "MockFixedIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 2000)), cp); + // baseBlockSize cannot be over 127: + swapCodec(new MockVariableIntBlockCodec(codecHasParam && "MockVariableIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 127)), cp); + swapCodec(new MockRandomCodec(random), cp); + + return cp.lookup(codec); + } + + // returns current PreFlex codec + static void removeTestCodecs(Codec codec, CodecProvider cp) { + if (codec.name.equals("PreFlex")) { + final Codec preFlex = cp.lookup("PreFlex"); + if (preFlex != null) { + cp.unregister(preFlex); + } + cp.register(new PreFlexCodec()); + } + cp.unregister(cp.lookup("MockSep")); + cp.unregister(cp.lookup("MockFixedIntBlock")); + cp.unregister(cp.lookup("MockVariableIntBlock")); + cp.unregister(cp.lookup("MockRandom")); + swapCodec(new PulsingCodec(), cp); + cp.setDefaultFieldCodec(savedDefaultCodec); + } + + // randomly picks from core and test codecs + static String pickRandomCodec(Random rnd) { + int idx = rnd.nextInt(CodecProvider.CORE_CODECS.length + + TEST_CODECS.length); + if (idx < CodecProvider.CORE_CODECS.length) { + return CodecProvider.CORE_CODECS[idx]; + } else { + return TEST_CODECS[idx - CodecProvider.CORE_CODECS.length]; + } + } + /** @deprecated (4.0) until we fix no-fork problems in solr tests */ @Deprecated static List testClassesRun = new ArrayList(); @@ -253,44 +342,40 @@ public abstract class LuceneTestCase extends Assert { System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockDirectoryFactory"); } - // if verbose: print some debugging stuff about which codecs are loaded - if (VERBOSE) { - Set codecs = Codec.availableCodecs(); - for (String codec : codecs) { - System.out.println("Loaded codec: '" + codec + "': " + Codec.forName(codec).getClass().getName()); - } - - Set postingsFormats = PostingsFormat.availablePostingsFormats(); - for (String postingsFormat : postingsFormats) { - System.out.println("Loaded postingsFormat: '" + postingsFormat + "': " + PostingsFormat.forName(postingsFormat).getClass().getName()); + savedCodecProvider = CodecProvider.getDefault(); + if ("random".equals(TEST_CODECPROVIDER)) { + if ("randomPerField".equals(TEST_CODEC)) { + if (random.nextInt(4) == 0) { // preflex-only setup + codec = installTestCodecs("PreFlex", CodecProvider.getDefault()); + } else { // per-field setup + CodecProvider.setDefault(new RandomCodecProvider(random, useNoMemoryExpensiveCodec)); + codec = installTestCodecs(TEST_CODEC, CodecProvider.getDefault()); + } + } else { // ordinary setup + codec = installTestCodecs(TEST_CODEC, CodecProvider.getDefault()); } - } - - PREFLEX_IMPERSONATION_IS_ACTIVE = false; - savedCodec = Codec.getDefault(); - final Codec codec; - if ("Lucene3x".equals(TEST_POSTINGSFORMAT) || ("random".equals(TEST_POSTINGSFORMAT) && random.nextInt(4) == 0)) { // preflex-only setup - codec = new PreFlexRWCodec(); - PREFLEX_IMPERSONATION_IS_ACTIVE = true; - } else if ("random".equals(TEST_POSTINGSFORMAT)) { - codec = new RandomCodec(random, useNoMemoryExpensiveCodec); } else { - codec = new Lucene40Codec() { - private final PostingsFormat format = PostingsFormat.forName(TEST_POSTINGSFORMAT); - - @Override - public PostingsFormat getPostingsFormatForField(String field) { - return format; - } - - @Override - public String toString() { - return super.toString() + ": " + format.toString(); + // someone specified their own codecprovider by class + try { + Class cpClazz = Class.forName(TEST_CODECPROVIDER).asSubclass(CodecProvider.class); + CodecProvider cp = cpClazz.newInstance(); + String codecName; + if (TEST_CODEC.startsWith("random")) { // TODO: somehow do random per-field?! + Set codecSet = cp.listAll(); + String availableCodecs[] = codecSet.toArray(new String[codecSet.size()]); + codecName = availableCodecs[random.nextInt(availableCodecs.length)]; + } else { + codecName = TEST_CODEC; } - }; + + codec = cp.lookup(codecName); + cp.setDefaultFieldCodec(codecName); + CodecProvider.setDefault(cp); + } catch (Exception e) { + System.err.println("Could not instantiate CodecProvider: " + TEST_CODECPROVIDER); + throw new RuntimeException(e); + } } - - Codec.setDefault(codec); savedLocale = Locale.getDefault(); @@ -315,6 +400,11 @@ public abstract class LuceneTestCase extends Assert { TimeZone.setDefault(timeZone); similarityProvider = new RandomSimilarityProvider(random); testsFailed = false; + + // verify assertions are enabled (do last, for smooth cleanup) + if (!Boolean.parseBoolean(System.getProperty("tests.asserts.gracious", "false"))) { + assertTrue("assertions are not enabled!", assertionsEnabled()); + } } @AfterClass @@ -340,8 +430,7 @@ public abstract class LuceneTestCase extends Assert { } } - String codecDescription = Codec.getDefault().toString(); - Codec.setDefault(savedCodec); + String codecDescription = uninstallCodecsAfterClass(); Locale.setDefault(savedLocale); TimeZone.setDefault(savedTimeZone); System.clearProperty("solr.solr.home"); @@ -406,6 +495,24 @@ public abstract class LuceneTestCase extends Assert { + "total=" + Runtime.getRuntime().totalMemory()); } + /** uninstalls test codecs, returns description of the codec used for debugging */ + private static String uninstallCodecsAfterClass() { + String codecDescription; + CodecProvider cp = CodecProvider.getDefault(); + + if ("randomPerField".equals(TEST_CODEC) && cp instanceof RandomCodecProvider) { + codecDescription = cp.toString(); + } else { + codecDescription = codec.toString(); + } + + if ("random".equals(TEST_CODECPROVIDER) && CodecProvider.getDefault() == savedCodecProvider) + removeTestCodecs(codec, CodecProvider.getDefault()); + CodecProvider.setDefault(savedCodecProvider); + + return codecDescription; + } + /** check that directories and their resources were closed */ private static void checkResourcesAfterClass() { for (MockDirectoryWrapper d : stores.keySet()) { @@ -538,12 +645,12 @@ public abstract class LuceneTestCase extends Assert { } if (useNoMemoryExpensiveCodec) { - String defFormat = _TestUtil.getPostingsFormat("thisCodeMakesAbsolutelyNoSenseCanWeDeleteIt"); + final String defCodec = CodecProvider.getDefault().getDefaultFieldCodec(); // Stupid: assumeFalse in setUp() does not print any information, because // TestWatchman does not watch test during setUp() - getName() is also not defined... // => print info directly and use assume without message: - if ("SimpleText".equals(defFormat) || "Memory".equals(defFormat)) { - System.err.println("NOTE: A test method in " + getClass().getSimpleName() + " was ignored, as it uses too much memory with " + defFormat + "."); + if ("SimpleText".equals(defCodec) || "Memory".equals(defCodec)) { + System.err.println("NOTE: A test method in " + getClass().getSimpleName() + " was ignored, as it uses too much memory with " + defCodec + "."); Assume.assumeTrue(false); } } @@ -1273,7 +1380,7 @@ public abstract class LuceneTestCase extends Assert { // extra params that were overridden needed to reproduce the command private static String reproduceWithExtraParams() { StringBuilder sb = new StringBuilder(); - if (!TEST_POSTINGSFORMAT.equals("random")) sb.append(" -Dtests.postingsformat=").append(TEST_POSTINGSFORMAT); + if (!TEST_CODEC.equals("randomPerField")) sb.append(" -Dtests.codec=").append(TEST_CODEC); if (!TEST_LOCALE.equals("random")) sb.append(" -Dtests.locale=").append(TEST_LOCALE); if (!TEST_TIMEZONE.equals("random")) sb.append(" -Dtests.timezone=").append(TEST_TIMEZONE); if (!TEST_DIRECTORY.equals("random")) sb.append(" -Dtests.directory=").append(TEST_DIRECTORY); @@ -1340,4 +1447,15 @@ public abstract class LuceneTestCase extends Assert { @Ignore("just a hack") public final void alwaysIgnoredTestMethod() {} + + /** check if assertions are enabled */ + private static boolean assertionsEnabled() { + try { + assert Boolean.FALSE.booleanValue(); + return false; // should never get here + } catch (AssertionError e) { + return true; + } + } + } diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java rename to lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/SmartRandom.java b/lucene/src/test-framework/org/apache/lucene/util/SmartRandom.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/util/SmartRandom.java rename to lucene/src/test-framework/org/apache/lucene/util/SmartRandom.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/ThreeLongs.java b/lucene/src/test-framework/org/apache/lucene/util/ThreeLongs.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/util/ThreeLongs.java rename to lucene/src/test-framework/org/apache/lucene/util/ThreeLongs.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/ThrottledIndexOutput.java b/lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/util/ThrottledIndexOutput.java rename to lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/_TestIgnoredException.java b/lucene/src/test-framework/org/apache/lucene/util/_TestIgnoredException.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/util/_TestIgnoredException.java rename to lucene/src/test-framework/org/apache/lucene/util/_TestIgnoredException.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java b/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java similarity index 94% rename from lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java rename to lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java index c736503..b298aad 100644 --- a/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java +++ b/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java @@ -45,9 +45,7 @@ import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.PostingsFormat; -import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; -import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; @@ -146,10 +144,17 @@ public class _TestUtil { * issues are hit, a RuntimeException is thrown; else, * true is returned. */ public static CheckIndex.Status checkIndex(Directory dir) throws IOException { + return checkIndex(dir, CodecProvider.getDefault()); + } + + /** This runs the CheckIndex tool on the index in. If any + * issues are hit, a RuntimeException is thrown; else, + * true is returned. */ + public static CheckIndex.Status checkIndex(Directory dir, CodecProvider codecs) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); CheckIndex checker = new CheckIndex(dir); checker.setInfoStream(new PrintStream(bos), false); - CheckIndex.Status indexStatus = checker.checkIndex(null); + CheckIndex.Status indexStatus = checker.checkIndex(null, codecs); if (indexStatus == null || indexStatus.clean == false) { System.out.println("CheckIndex failed"); System.out.println(bos.toString()); @@ -346,28 +351,28 @@ public class _TestUtil { return new String(buffer, 0, i); } - - /** Return a Codec that can read any of the - * default codecs and formats, but always writes in the specified - * format. */ - public static Codec alwaysPostingsFormat(final PostingsFormat format) { - return new Lucene40Codec() { + public static CodecProvider alwaysCodec(final Codec c) { + CodecProvider p = new CodecProvider() { @Override - public PostingsFormat getPostingsFormatForField(String field) { - return format; + public Codec lookup(String name) { + // can't do this until we fix PreFlexRW to not + //impersonate PreFlex: + if (name.equals(c.name)) { + return c; + } else { + return CodecProvider.getDefault().lookup(name); + } } }; + p.setDefaultFieldCodec(c.name); + return p; } - // TODO: generalize all 'test-checks-for-crazy-codecs' to - // annotations (LUCENE-3489) - public static String getPostingsFormat(String field) { - PostingsFormat p = Codec.getDefault().postingsFormat(); - if (p instanceof PerFieldPostingsFormat) { - return ((PerFieldPostingsFormat)p).getPostingsFormatForField(field).getName(); - } else { - return p.getName(); - } + /** Return a CodecProvider that can read any of the + * default codecs, but always writes in the specified + * codec. */ + public static CodecProvider alwaysCodec(final String codec) { + return alwaysCodec(CodecProvider.getDefault().lookup(codec)); } public static boolean anyFilesExceptWriteLock(Directory dir) throws IOException { diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java b/lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java rename to lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java b/lucene/src/test-framework/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java rename to lucene/src/test-framework/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/util/europarl.lines.txt.gz b/lucene/src/test-framework/org/apache/lucene/util/europarl.lines.txt.gz similarity index 100% rename from lucene/src/test-framework/java/org/apache/lucene/util/europarl.lines.txt.gz rename to lucene/src/test-framework/org/apache/lucene/util/europarl.lines.txt.gz diff --git a/lucene/src/test-framework/java/overview.html b/lucene/src/test-framework/overview.html similarity index 100% rename from lucene/src/test-framework/java/overview.html rename to lucene/src/test-framework/overview.html diff --git a/lucene/src/test-framework/resources/META-INF/services/org.apache.lucene.index.codecs.Codec b/lucene/src/test-framework/resources/META-INF/services/org.apache.lucene.index.codecs.Codec deleted file mode 100644 index fdd6f8e..0000000 diff --git a/lucene/src/test-framework/resources/META-INF/services/org.apache.lucene.index.codecs.PostingsFormat b/lucene/src/test-framework/resources/META-INF/services/org.apache.lucene.index.codecs.PostingsFormat deleted file mode 100644 index 505d010..0000000 diff --git a/lucene/src/test/org/apache/lucene/TestExternalCodecs.java b/lucene/src/test/org/apache/lucene/TestExternalCodecs.java index 7269691..b738976 100644 --- a/lucene/src/test/org/apache/lucene/TestExternalCodecs.java +++ b/lucene/src/test/org/apache/lucene/TestExternalCodecs.java @@ -17,47 +17,512 @@ package org.apache.lucene; * limitations under the License. */ -import java.io.*; -import java.util.*; - -import org.apache.lucene.analysis.*; -import org.apache.lucene.document.*; +import org.apache.lucene.util.*; +import org.apache.lucene.util.Bits; import org.apache.lucene.index.*; -import org.apache.lucene.index.codecs.*; -import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; -import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.document.*; import org.apache.lucene.search.*; +import org.apache.lucene.analysis.*; +import org.apache.lucene.index.codecs.*; import org.apache.lucene.store.*; -import org.apache.lucene.util.*; -import org.apache.lucene.util.Bits; +import java.util.*; +import java.io.*; /* Intentionally outside of oal.index to verify fully external codecs work fine */ public class TestExternalCodecs extends LuceneTestCase { - private static final class CustomPerFieldCodec extends Lucene40Codec { + // For fun, test that we can override how terms are + // sorted, and basic things still work -- this comparator + // sorts in reversed unicode code point order: + private static final Comparator reverseUnicodeComparator = new Comparator() { + public int compare(BytesRef t1, BytesRef t2) { + byte[] b1 = t1.bytes; + byte[] b2 = t2.bytes; + int b1Stop; + int b1Upto = t1.offset; + int b2Upto = t2.offset; + if (t1.length < t2.length) { + b1Stop = t1.offset + t1.length; + } else { + b1Stop = t1.offset + t2.length; + } + while(b1Upto < b1Stop) { + final int bb1 = b1[b1Upto++] & 0xff; + final int bb2 = b2[b2Upto++] & 0xff; + if (bb1 != bb2) { + //System.out.println("cmp 1=" + t1 + " 2=" + t2 + " return " + (bb2-bb1)); + return bb2 - bb1; + } + } + + // One is prefix of another, or they are equal + return t2.length-t1.length; + } + + @Override + public boolean equals(Object other) { + return this == other; + } + }; + + // TODO + // - good improvement would be to write through to disk, + // and then load into ram from disk + public static class RAMOnlyCodec extends Codec { - private final PostingsFormat ramFormat = PostingsFormat.forName("RAMOnly"); - private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene40"); - private final PostingsFormat pulsingFormat = PostingsFormat.forName("Pulsing40"); + public RAMOnlyCodec() { + super("RamOnly"); + } + // Postings state: + static class RAMPostings extends FieldsProducer { + final Map fieldToTerms = new TreeMap(); + + @Override + public Terms terms(String field) { + return fieldToTerms.get(field); + } + + @Override + public FieldsEnum iterator() { + return new RAMFieldsEnum(this); + } + + @Override + public void close() { + } + } + + static class RAMField extends Terms { + final String field; + final SortedMap termToDocs = new TreeMap(); + long sumTotalTermFreq; + long sumDocFreq; + int docCount; + + RAMField(String field) { + this.field = field; + } + + @Override + public long getUniqueTermCount() { + return termToDocs.size(); + } + + @Override + public long getSumTotalTermFreq() { + return sumTotalTermFreq; + } + + @Override + public long getSumDocFreq() throws IOException { + return sumDocFreq; + } + + @Override + public int getDocCount() throws IOException { + return docCount; + } + + @Override + public TermsEnum iterator() { + return new RAMTermsEnum(RAMOnlyCodec.RAMField.this); + } + + @Override + public Comparator getComparator() { + return reverseUnicodeComparator; + } + } + + static class RAMTerm { + final String term; + long totalTermFreq; + final List docs = new ArrayList(); + public RAMTerm(String term) { + this.term = term; + } + } + + static class RAMDoc { + final int docID; + final int[] positions; + byte[][] payloads; + + public RAMDoc(int docID, int freq) { + this.docID = docID; + positions = new int[freq]; + } + } + + // Classes for writing to the postings state + private static class RAMFieldsConsumer extends FieldsConsumer { + + private final RAMPostings postings; + private final RAMTermsConsumer termsConsumer = new RAMTermsConsumer(); + + public RAMFieldsConsumer(RAMPostings postings) { + this.postings = postings; + } + + @Override + public TermsConsumer addField(FieldInfo field) { + RAMField ramField = new RAMField(field.name); + postings.fieldToTerms.put(field.name, ramField); + termsConsumer.reset(ramField); + return termsConsumer; + } + + @Override + public void close() { + // TODO: finalize stuff + } + } + + private static class RAMTermsConsumer extends TermsConsumer { + private RAMField field; + private final RAMPostingsWriterImpl postingsWriter = new RAMPostingsWriterImpl(); + RAMTerm current; + + void reset(RAMField field) { + this.field = field; + } + + @Override + public PostingsConsumer startTerm(BytesRef text) { + final String term = text.utf8ToString(); + current = new RAMTerm(term); + postingsWriter.reset(current); + return postingsWriter; + } + + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public void finishTerm(BytesRef text, TermStats stats) { + assert stats.docFreq > 0; + assert stats.docFreq == current.docs.size(); + current.totalTermFreq = stats.totalTermFreq; + field.termToDocs.put(current.term, current); + } + + @Override + public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) { + field.sumTotalTermFreq = sumTotalTermFreq; + field.sumDocFreq = sumDocFreq; + field.docCount = docCount; + } + } + + public static class RAMPostingsWriterImpl extends PostingsConsumer { + private RAMTerm term; + private RAMDoc current; + private int posUpto = 0; + + public void reset(RAMTerm term) { + this.term = term; + } + + @Override + public void startDoc(int docID, int freq) { + current = new RAMDoc(docID, freq); + term.docs.add(current); + posUpto = 0; + } + + @Override + public void addPosition(int position, BytesRef payload) { + current.positions[posUpto] = position; + if (payload != null && payload.length > 0) { + if (current.payloads == null) { + current.payloads = new byte[current.positions.length][]; + } + byte[] bytes = current.payloads[posUpto] = new byte[payload.length]; + System.arraycopy(payload.bytes, payload.offset, bytes, 0, payload.length); + } + posUpto++; + } + + @Override + public void finishDoc() { + assert posUpto == current.positions.length; + } + } + + // Classes for reading from the postings state + static class RAMFieldsEnum extends FieldsEnum { + private final RAMPostings postings; + private final Iterator it; + private String current; + + public RAMFieldsEnum(RAMPostings postings) { + this.postings = postings; + this.it = postings.fieldToTerms.keySet().iterator(); + } + + @Override + public String next() { + if (it.hasNext()) { + current = it.next(); + } else { + current = null; + } + return current; + } + + @Override + public TermsEnum terms() { + return new RAMTermsEnum(postings.fieldToTerms.get(current)); + } + } + + static class RAMTermsEnum extends TermsEnum { + Iterator it; + String current; + private final RAMField ramField; + + public RAMTermsEnum(RAMField field) { + this.ramField = field; + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public BytesRef next() { + if (it == null) { + if (current == null) { + it = ramField.termToDocs.keySet().iterator(); + } else { + it = ramField.termToDocs.tailMap(current).keySet().iterator(); + } + } + if (it.hasNext()) { + current = it.next(); + return new BytesRef(current); + } else { + return null; + } + } + + @Override + public SeekStatus seekCeil(BytesRef term, boolean useCache) { + current = term.utf8ToString(); + it = null; + if (ramField.termToDocs.containsKey(current)) { + return SeekStatus.FOUND; + } else { + if (current.compareTo(ramField.termToDocs.lastKey()) > 0) { + return SeekStatus.END; + } else { + return SeekStatus.NOT_FOUND; + } + } + } + + @Override + public void seekExact(long ord) { + throw new UnsupportedOperationException(); + } + + @Override + public long ord() { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef term() { + // TODO: reuse BytesRef + return new BytesRef(current); + } + + @Override + public int docFreq() { + return ramField.termToDocs.get(current).docs.size(); + } + + @Override + public long totalTermFreq() { + return ramField.termToDocs.get(current).totalTermFreq; + } + + @Override + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) { + return new RAMDocsEnum(ramField.termToDocs.get(current), liveDocs); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) { + return new RAMDocsAndPositionsEnum(ramField.termToDocs.get(current), liveDocs); + } + } + + private static class RAMDocsEnum extends DocsEnum { + private final RAMTerm ramTerm; + private final Bits liveDocs; + private RAMDoc current; + int upto = -1; + int posUpto = 0; + + public RAMDocsEnum(RAMTerm ramTerm, Bits liveDocs) { + this.ramTerm = ramTerm; + this.liveDocs = liveDocs; + } + + @Override + public int advance(int targetDocID) { + do { + nextDoc(); + } while (upto < ramTerm.docs.size() && current.docID < targetDocID); + return NO_MORE_DOCS; + } + + // TODO: override bulk read, for better perf + @Override + public int nextDoc() { + while(true) { + upto++; + if (upto < ramTerm.docs.size()) { + current = ramTerm.docs.get(upto); + if (liveDocs == null || liveDocs.get(current.docID)) { + posUpto = 0; + return current.docID; + } + } else { + return NO_MORE_DOCS; + } + } + } + + @Override + public int freq() { + return current.positions.length; + } + + @Override + public int docID() { + return current.docID; + } + } + + private static class RAMDocsAndPositionsEnum extends DocsAndPositionsEnum { + private final RAMTerm ramTerm; + private final Bits liveDocs; + private RAMDoc current; + int upto = -1; + int posUpto = 0; + + public RAMDocsAndPositionsEnum(RAMTerm ramTerm, Bits liveDocs) { + this.ramTerm = ramTerm; + this.liveDocs = liveDocs; + } + + @Override + public int advance(int targetDocID) { + do { + nextDoc(); + } while (upto < ramTerm.docs.size() && current.docID < targetDocID); + return NO_MORE_DOCS; + } + + // TODO: override bulk read, for better perf + @Override + public int nextDoc() { + while(true) { + upto++; + if (upto < ramTerm.docs.size()) { + current = ramTerm.docs.get(upto); + if (liveDocs == null || liveDocs.get(current.docID)) { + posUpto = 0; + return current.docID; + } + } else { + return NO_MORE_DOCS; + } + } + } + + @Override + public int freq() { + return current.positions.length; + } + + @Override + public int docID() { + return current.docID; + } + + @Override + public int nextPosition() { + return current.positions[posUpto++]; + } + + @Override + public boolean hasPayload() { + return current.payloads != null && current.payloads[posUpto-1] != null; + } + + @Override + public BytesRef getPayload() { + return new BytesRef(current.payloads[posUpto-1]); + } + } + + // Holds all indexes created + private final Map state = new HashMap(); + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState writeState) { + RAMPostings postings = new RAMPostings(); + RAMFieldsConsumer consumer = new RAMFieldsConsumer(postings); + synchronized(state) { + state.put(writeState.segmentName, postings); + } + return consumer; + } @Override - public PostingsFormat getPostingsFormatForField(String field) { - if (field.equals("field2") || field.equals("id")) { - return pulsingFormat; - } else if (field.equals("field1")) { - return defaultFormat; - } else { - return ramFormat; + public FieldsProducer fieldsProducer(SegmentReadState readState) + throws IOException { + + synchronized(state) { + return state.get(readState.segmentInfo.name); } } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return null; + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return null; + } + + @Override + public void getExtensions(Set extensions) { + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) { + } } // tests storing "id" and "field2" fields as pulsing codec, // whose term sort is backwards unicode code point, and // storing "field1" as a custom entirely-in-RAM codec public void testPerFieldCodec() throws Exception { + CodecProvider provider = new CoreCodecProvider(); + provider.register(new RAMOnlyCodec()); + provider.setDefaultFieldCodec("RamOnly"); final int NUM_DOCS = atLeast(173); MockDirectoryWrapper dir = newDirectory(); @@ -65,7 +530,7 @@ public class TestExternalCodecs extends LuceneTestCase { IndexWriter w = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setCodec(new CustomPerFieldCodec()). + setCodecProvider(provider). setMergePolicy(newLogMergePolicy(3)) ); w.setInfoStream(VERBOSE ? System.out : null); @@ -74,9 +539,11 @@ public class TestExternalCodecs extends LuceneTestCase { doc.add(newField("field1", "this field uses the standard codec as the test", TextField.TYPE_UNSTORED)); // uses pulsing codec: Field field2 = newField("field2", "this field uses the pulsing codec as the test", TextField.TYPE_UNSTORED); + provider.setFieldCodec(field2.name(), "Pulsing"); doc.add(field2); Field idField = newField("id", "", StringField.TYPE_UNSTORED); + provider.setFieldCodec(idField.name(), "Pulsing"); doc.add(idField); for(int i=0;i Integer.MAX_VALUE terms/docs pairs * @lucene.experimental */ -@UseNoMemoryExpensiveCodec public class Test2BPostings extends LuceneTestCase { @Nightly public void test() throws Exception { + + assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory")); + assumeFalse("This test is super-slow and very disk-space-consuming with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText")); + MockDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BPostings")); dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER); dir.setCheckIndexOnClose(false); // don't double-checkindex diff --git a/lucene/src/test/org/apache/lucene/index/Test2BTerms.java b/lucene/src/test/org/apache/lucene/index/Test2BTerms.java index 03a3031..c146ba1 100644 --- a/lucene/src/test/org/apache/lucene/index/Test2BTerms.java +++ b/lucene/src/test/org/apache/lucene/index/Test2BTerms.java @@ -24,8 +24,7 @@ import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.*; import org.apache.lucene.document.*; import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.codecs.Codec; - +import org.apache.lucene.index.codecs.CodecProvider; import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -143,7 +142,7 @@ public class Test2BTerms extends LuceneTestCase { @Ignore("Takes ~4 hours to run on a fast machine!! And requires that you don't use PreFlex codec.") public void test2BTerms() throws IOException { - if ("Lucene3x".equals(Codec.getDefault().getName())) { + if ("PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())) { throw new RuntimeException("thist test cannot run with PreFlex codec"); } diff --git a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java index 3e9f14b..909609c 100755 --- a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java +++ b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java @@ -17,10 +17,9 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.io.FileNotFoundException; import java.io.IOException; +import java.io.FileNotFoundException; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; @@ -30,22 +29,11 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.DefaultDocValuesFormat; -import org.apache.lucene.index.codecs.DefaultFieldsFormat; -import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; -import org.apache.lucene.index.codecs.DocValuesFormat; -import org.apache.lucene.index.codecs.FieldsFormat; -import org.apache.lucene.index.codecs.PostingsFormat; -import org.apache.lucene.index.codecs.SegmentInfosFormat; -import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsBaseFormat; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; -import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; -import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; -import org.apache.lucene.index.codecs.pulsing.Pulsing40PostingsFormat; -import org.apache.lucene.index.codecs.pulsing.PulsingPostingsFormat; -import org.apache.lucene.index.codecs.simpletext.SimpleTextPostingsFormat; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.mocksep.MockSepCodec; +import org.apache.lucene.index.codecs.pulsing.PulsingCodec; +import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; +import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.store.AlreadyClosedException; @@ -987,29 +975,30 @@ public class TestAddIndexes extends LuceneTestCase { } } - public void testSimpleCaseCustomCodec() throws IOException { + public void testSimpleCaseCustomCodecProvider() throws IOException { // main directory Directory dir = newDirectory(); // two auxiliary directories Directory aux = newDirectory(); Directory aux2 = newDirectory(); - Codec codec = new CustomPerFieldCodec(); + CodecProvider provider = new MockCodecProvider(); IndexWriter writer = null; writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodec(codec)); + new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodecProvider( + provider)); // add 100 documents addDocs3(writer, 100); assertEquals(100, writer.maxDoc()); writer.commit(); writer.close(); - _TestUtil.checkIndex(dir); + _TestUtil.checkIndex(dir, provider); writer = newWriter( aux, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.CREATE). - setCodec(codec). + setCodecProvider(provider). setMaxBufferedDocs(10). setMergePolicy(newLogMergePolicy(false)) ); @@ -1023,7 +1012,7 @@ public class TestAddIndexes extends LuceneTestCase { aux2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.CREATE). - setCodec(codec) + setCodecProvider(provider) ); // add 40 documents in compound files addDocs2(writer, 50); @@ -1036,7 +1025,7 @@ public class TestAddIndexes extends LuceneTestCase { dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.APPEND). - setCodec(codec) + setCodecProvider(provider) ); assertEquals(100, writer.maxDoc()); writer.addIndexes(aux, aux2); @@ -1048,24 +1037,19 @@ public class TestAddIndexes extends LuceneTestCase { aux2.close(); } - private static final class CustomPerFieldCodec extends Lucene40Codec { - private final PostingsFormat simpleTextFormat = PostingsFormat.forName("SimpleText"); - private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene40"); - private final PostingsFormat mockSepFormat = PostingsFormat.forName("MockSep"); - - @Override - public PostingsFormat getPostingsFormatForField(String field) { - if (field.equals("id")) { - return simpleTextFormat; - } else if (field.equals("content")) { - return mockSepFormat; - } else { - return defaultFormat; - } + public static class MockCodecProvider extends CodecProvider { + public MockCodecProvider() { + StandardCodec standardCodec = new StandardCodec(); + SimpleTextCodec simpleTextCodec = new SimpleTextCodec(); + MockSepCodec mockSepCodec = new MockSepCodec(); + register(standardCodec); + register(mockSepCodec); + register(simpleTextCodec); + setFieldCodec("id", simpleTextCodec.name); + setFieldCodec("content", mockSepCodec.name); } } - // LUCENE-2790: tests that the non CFS files were deleted by addIndexes public void testNonCFSLeftovers() throws Exception { Directory[] dirs = new Directory[2]; @@ -1082,19 +1066,16 @@ public class TestAddIndexes extends LuceneTestCase { IndexReader[] readers = new IndexReader[] { IndexReader.open(dirs[0]), IndexReader.open(dirs[1]) }; - Directory dir = new MockDirectoryWrapper(random, new RAMDirectory()); + Directory dir = new RAMDirectory(); IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(true); lmp.setNoCFSRatio(1.0); // Force creation of CFS IndexWriter w3 = new IndexWriter(dir, conf); - w3.setInfoStream(VERBOSE ? System.out : null); w3.addIndexes(readers); w3.close(); - // we should now see segments_X, - // segments.gen,_Y.cfs,_Y.cfe, _Z.fnx - assertEquals("Only one compound segment should exist, but got: " + Arrays.toString(dir.listAll()), 5, dir.listAll().length); - dir.close(); + // we should now see segments_X, segments.gen,_Y.cfs,_Y.cfe, _Z.fnx + assertEquals("Only one compound segment should exist", 5, dir.listAll().length); } // LUCENE-3126: tests that if a non-CFS segment is copied, it is converted to @@ -1155,45 +1136,18 @@ public class TestAddIndexes extends LuceneTestCase { src.close(); target.close(); } - - private static class UnRegisteredCodec extends Codec { - public UnRegisteredCodec() { - super("NotRegistered"); - } - - @Override - public PostingsFormat postingsFormat() { - return PostingsFormat.forName("Lucene40"); - } - - @Override - public DocValuesFormat docValuesFormat() { - return new DefaultDocValuesFormat(); - } - - @Override - public FieldsFormat fieldsFormat() { - return new DefaultFieldsFormat(); - } - - @Override - public SegmentInfosFormat segmentInfosFormat() { - return new DefaultSegmentInfosFormat(); - } - } /* * simple test that ensures we getting expected exceptions */ public void testAddIndexMissingCodec() throws IOException { - MockDirectoryWrapper toAdd = newDirectory(); - // Disable checkIndex, else we get an exception because - // of the unregistered codec: - toAdd.setCheckIndexOnClose(false); + Directory toAdd = newDirectory(); { IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); - conf.setCodec(new UnRegisteredCodec()); + CodecProvider provider = new CodecProvider(); + provider.register(new StandardCodec()); + conf.setCodecProvider(provider); IndexWriter w = new IndexWriter(toAdd, conf); Document doc = new Document(); FieldType customType = new FieldType(); @@ -1202,12 +1156,13 @@ public class TestAddIndexes extends LuceneTestCase { w.addDocument(doc); w.close(); } - { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); - conf.setCodec(_TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1 + random.nextInt(20)))); + CodecProvider provider = new CodecProvider(); + provider.register(new PulsingCodec(1 + random.nextInt(20))); + conf.setCodecProvider(provider); IndexWriter w = new IndexWriter(dir, conf); try { w.addIndexes(toAdd); @@ -1222,11 +1177,27 @@ public class TestAddIndexes extends LuceneTestCase { dir.close(); } - try { + { + Directory dir = newDirectory(); + IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random)); + CodecProvider provider = new CodecProvider(); + provider.register(new PulsingCodec(1 + random.nextInt(20))); + conf.setCodecProvider(provider); + IndexWriter w = new IndexWriter(dir, conf); IndexReader indexReader = IndexReader.open(toAdd); - fail("no such codec"); - } catch (IllegalArgumentException ex) { - // expected + try { + w.addIndexes(indexReader); + fail("no such codec"); + } catch (IllegalArgumentException ex) { + // expected + } + indexReader.close(); + w.close(); + IndexReader open = IndexReader.open(dir); + assertEquals(0, open.numDocs()); + open.close(); + dir.close(); } toAdd.close(); } diff --git a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index f2cde36..8cd628e 100644 --- a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -568,7 +568,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase { "_0_1.s" + contentFieldIndex, "segments_2", "segments.gen", - "_1.fnx"}; + "1.fnx"}; String[] actual = dir.listAll(); Arrays.sort(expected); diff --git a/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java b/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java index 83dc9b1..d05b8d0 100644 --- a/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java +++ b/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java @@ -23,7 +23,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; @@ -37,7 +37,7 @@ import org.apache.lucene.util.LuceneTestCase; public class TestBinaryTerms extends LuceneTestCase { public void testBinary() throws IOException { assumeFalse("PreFlex codec cannot work with binary terms!", - Codec.getDefault().getName().equals("Lucene3x")); + "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir); diff --git a/lucene/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/src/test/org/apache/lucene/index/TestCodecs.java index 3cce510..8db99ab 100644 --- a/lucene/src/test/org/apache/lucene/index/TestCodecs.java +++ b/lucene/src/test/org/apache/lucene/index/TestCodecs.java @@ -26,15 +26,14 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.PostingsConsumer; import org.apache.lucene.index.codecs.TermStats; import org.apache.lucene.index.codecs.TermsConsumer; -import org.apache.lucene.index.codecs.lucene3x.Lucene3xCodec; -import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; -import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepCodec; +import org.apache.lucene.index.codecs.preflex.PreFlexCodec; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; @@ -65,7 +64,6 @@ import org.junit.BeforeClass; // goes to 1 before next one known to exist // - skipTo(term) // - skipTo(doc) - public class TestCodecs extends LuceneTestCase { private static String[] fieldNames = new String[] {"one", "two", "three", "four"}; @@ -257,10 +255,9 @@ public class TestCodecs extends LuceneTestCase { final Directory dir = newDirectory(); FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone(); this.write(fieldInfos, dir, fields, true); - Codec codec = Codec.getDefault(); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, codec, clonedFieldInfos); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, clonedFieldInfos.buildSegmentCodecs(false), clonedFieldInfos); - final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); + final FieldsProducer reader = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); final FieldsEnum fieldsEnum = reader.iterator(); assertNotNull(fieldsEnum.next()); @@ -310,13 +307,12 @@ public class TestCodecs extends LuceneTestCase { FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone(); this.write(fieldInfos, dir, fields, false); - Codec codec = Codec.getDefault(); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, codec, clonedFieldInfos); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, clonedFieldInfos.buildSegmentCodecs(false), clonedFieldInfos); if (VERBOSE) { System.out.println("TEST: now read postings"); } - final FieldsProducer terms = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); + final FieldsProducer terms = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); final Verify[] threads = new Verify[NUM_TEST_THREADS-1]; for(int i=0;i filesToDelete = merger.createCompoundFile(merged + ".cfs", info, newIOContext(random)); diff --git a/lucene/src/test/org/apache/lucene/index/TestDocCount.java b/lucene/src/test/org/apache/lucene/index/TestDocCount.java index bba8e30..4c12aa0 100644 --- a/lucene/src/test/org/apache/lucene/index/TestDocCount.java +++ b/lucene/src/test/org/apache/lucene/index/TestDocCount.java @@ -19,7 +19,7 @@ package org.apache.lucene.index; import org.apache.lucene.document.Document; import org.apache.lucene.document.StringField; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.util.FixedBitSet; @@ -32,7 +32,7 @@ import org.apache.lucene.util._TestUtil; public class TestDocCount extends LuceneTestCase { public void testSimple() throws Exception { assumeFalse("PreFlex codec does not support docCount statistic!", - "Lucene3x".equals(Codec.getDefault().getName())); + "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir); int numDocs = atLeast(100); diff --git a/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java b/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java index aef44ab..eb3641a 100644 --- a/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java +++ b/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java @@ -34,17 +34,21 @@ import org.apache.lucene.index.DocTermOrds.TermOrdsIterator; import org.apache.lucene.index.codecs.BlockTermsReader; import org.apache.lucene.index.codecs.BlockTermsWriter; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.CoreCodecProvider; +import org.apache.lucene.index.codecs.DefaultDocValuesProducer; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; +import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.TermsIndexReaderBase; import org.apache.lucene.index.codecs.TermsIndexWriterBase; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; +import org.apache.lucene.index.codecs.standard.StandardPostingsReader; +import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; @@ -102,6 +106,130 @@ public class TestDocTermOrds extends LuceneTestCase { dir.close(); } + private static class StandardCodecWithOrds extends Codec { + + public StandardCodecWithOrds() { + super("StandardOrds"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docs = new StandardPostingsWriter(state); + + // TODO: should we make the terms index more easily + // pluggable? Ie so that this codec would record which + // index impl was used, and switch on loading? + // Or... you must make a new Codec for this? + TermsIndexWriterBase indexWriter; + boolean success = false; + try { + indexWriter = new FixedGapTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + docs.close(); + } + } + + success = false; + try { + FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs); + success = true; + return ret; + } finally { + if (!success) { + try { + docs.close(); + } finally { + indexWriter.close(); + } + } + } + } + + public final static int TERMS_CACHE_SIZE = 1024; + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); + TermsIndexReaderBase indexReader; + + boolean success = false; + try { + indexReader = new FixedGapTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.codecId, state.context); + success = true; + } finally { + if (!success) { + postings.close(); + } + } + + success = false; + try { + FieldsProducer ret = new BlockTermsReader(indexReader, + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postings, + state.context, + TERMS_CACHE_SIZE, + state.codecId); + success = true; + return ret; + } finally { + if (!success) { + try { + postings.close(); + } finally { + indexReader.close(); + } + } + } + } + + /** Extension of freq postings file */ + static final String FREQ_EXTENSION = "frq"; + + /** Extension of prox postings file */ + static final String PROX_EXTENSION = "prx"; + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { + StandardPostingsReader.files(dir, segmentInfo, id, files); + BlockTermsReader.files(dir, segmentInfo, id, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, id, files); + DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); + } + + @Override + public void getExtensions(Set extensions) { + getStandardExtensions(extensions); + DefaultDocValuesConsumer.getExtensions(extensions); + } + + public static void getStandardExtensions(Set extensions) { + extensions.add(FREQ_EXTENSION); + extensions.add(PROX_EXTENSION); + BlockTermsReader.getExtensions(extensions); + FixedGapTermsIndexReader.getIndexExtensions(extensions); + } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new DefaultDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new DefaultDocValuesProducer(state); + } + } + public void testRandom() throws Exception { MockDirectoryWrapper dir = newDirectory(); @@ -124,8 +252,13 @@ public class TestDocTermOrds extends LuceneTestCase { // Sometimes swap in codec that impls ord(): if (random.nextInt(10) == 7) { // Make sure terms index has ords: - Codec codec = _TestUtil.alwaysPostingsFormat(PostingsFormat.forName("Lucene40WithOrds")); - conf.setCodec(codec); + CoreCodecProvider cp = new CoreCodecProvider(); + cp.register(new StandardCodecWithOrds()); + cp.setDefaultFieldCodec("StandardOrds"); + + // So checkIndex on close works + dir.setCodecProvider(cp); + conf.setCodecProvider(cp); } final RandomIndexWriter w = new RandomIndexWriter(random, dir, conf); @@ -221,8 +354,14 @@ public class TestDocTermOrds extends LuceneTestCase { // Sometimes swap in codec that impls ord(): if (random.nextInt(10) == 7) { - Codec codec = _TestUtil.alwaysPostingsFormat(PostingsFormat.forName("Lucene40WithOrds")); - conf.setCodec(codec); + // Make sure terms index has ords: + CoreCodecProvider cp = new CoreCodecProvider(); + cp.register(new StandardCodecWithOrds()); + cp.setDefaultFieldCodec("StandardOrds"); + + // So checkIndex on close works + dir.setCodecProvider(cp); + conf.setCodecProvider(cp); } final RandomIndexWriter w = new RandomIndexWriter(random, dir, conf); diff --git a/lucene/src/test/org/apache/lucene/index/TestFlex.java b/lucene/src/test/org/apache/lucene/index/TestFlex.java index 90e4628..7714afb 100644 --- a/lucene/src/test/org/apache/lucene/index/TestFlex.java +++ b/lucene/src/test/org/apache/lucene/index/TestFlex.java @@ -20,7 +20,6 @@ package org.apache.lucene.index; import org.apache.lucene.store.*; import org.apache.lucene.analysis.*; import org.apache.lucene.document.*; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.util.*; public class TestFlex extends LuceneTestCase { @@ -65,7 +64,7 @@ public class TestFlex extends LuceneTestCase { public void testTermOrd() throws Exception { Directory d = newDirectory(); IndexWriter w = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); + new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); Document doc = new Document(); doc.add(newField("f", "a b c", TextField.TYPE_UNSTORED)); w.addDocument(doc); diff --git a/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java b/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java index 5bdd9b0..ad997d1 100644 --- a/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java +++ b/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java @@ -22,6 +22,7 @@ import java.util.*; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TopDocs; @@ -35,9 +36,6 @@ public class TestForTooMuchCloning extends LuceneTestCase { // Make sure we don't clone IndexInputs too frequently // during merging: public void test() throws Exception { - // NOTE: if we see a fail on this test with "NestedPulsing" its because its - // reuse isnt perfect (but reasonable). see TestPulsingReuse.testNestedPulsing - // for more details final MockDirectoryWrapper dir = newDirectory(); final TieredMergePolicy tmp = new TieredMergePolicy(); tmp.setMaxMergeAtOnce(2); diff --git a/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java b/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java index 54e10fd..a7ac933 100644 --- a/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java +++ b/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java @@ -58,25 +58,25 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { } writer.commit(); Collection files = writer.getIndexFileNames(); - files.remove("_1.fnx"); + files.remove("1.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "_1.fnx"); + assertFNXFiles(dir, "1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); writer.commit(); files = writer.getIndexFileNames(); - files.remove("_2.fnx"); + files.remove("2.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "_2.fnx"); + assertFNXFiles(dir, "2.fnx"); writer.close(); - assertFNXFiles(dir, "_2.fnx"); + assertFNXFiles(dir, "2.fnx"); } { @@ -89,12 +89,12 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { writer.addDocument(d); writer.close(); Collection files = writer.getIndexFileNames(); - files.remove("_2.fnx"); + files.remove("2.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "_2.fnx"); + assertFNXFiles(dir, "2.fnx"); } IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( @@ -102,7 +102,7 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "_2.fnx"); + assertFNXFiles(dir, "2.fnx"); dir.close(); } @@ -121,29 +121,29 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "_1.fnx"); + assertFNXFiles(dir, "1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "_2.fnx"); + assertFNXFiles(dir, "2.fnx"); writer.close(); - assertFNXFiles(dir, "_2.fnx"); + assertFNXFiles(dir, "2.fnx"); } IndexReader reader = IndexReader.open(dir, false); reader.deleteDocument(0); reader.commit(); reader.close(); // make sure this reader can not modify the field map - assertFNXFiles(dir, "_2.fnx"); + assertFNXFiles(dir, "2.fnx"); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "_2.fnx"); + assertFNXFiles(dir, "2.fnx"); dir.close(); } @@ -162,7 +162,7 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "_1.fnx"); + assertFNXFiles(dir, "1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); @@ -170,9 +170,9 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { writer.commit(); writer.commit(); writer.commit(); - assertFNXFiles(dir, "_1.fnx", "_2.fnx"); + assertFNXFiles(dir, "1.fnx", "2.fnx"); writer.close(); - assertFNXFiles(dir, "_1.fnx", "_2.fnx"); + assertFNXFiles(dir, "1.fnx", "2.fnx"); } { @@ -184,14 +184,14 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { d.add(new BinaryField("f3", new byte[] { 1, 2, 3, 4, 5 })); writer.addDocument(d); writer.close(); - assertFNXFiles(dir, "_2.fnx"); + assertFNXFiles(dir, "2.fnx"); } IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "_2.fnx"); + assertFNXFiles(dir, "2.fnx"); dir.close(); } } @@ -208,14 +208,14 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "_1.fnx"); + assertFNXFiles(dir, "1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); - assertFNXFiles(dir, "_1.fnx"); + assertFNXFiles(dir, "1.fnx"); writer.close(); - assertFNXFiles(dir, "_1.fnx", "_2.fnx"); + assertFNXFiles(dir, "1.fnx", "2.fnx"); // open first commit List listCommits = IndexReader.listCommits(dir); assertEquals(2, listCommits.size()); @@ -229,18 +229,18 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { writer.addDocument(d); writer.commit(); // now we have 3 files since f3 is not present in the first commit - assertFNXFiles(dir, "_1.fnx", "_2.fnx", "_3.fnx"); + assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx"); writer.close(); - assertFNXFiles(dir, "_1.fnx", "_2.fnx", "_3.fnx"); + assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx"); writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.commit(); listCommits = IndexReader.listCommits(dir); assertEquals(1, listCommits.size()); - assertFNXFiles(dir, "_3.fnx"); + assertFNXFiles(dir, "3.fnx"); writer.close(); - assertFNXFiles(dir, "_3.fnx"); + assertFNXFiles(dir, "3.fnx"); dir.close(); } } @@ -494,7 +494,7 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { assertEquals(1, segmentInfos.getGlobalFieldMapVersion()); assertEquals(1, segmentInfos.getLastGlobalFieldMapVersion()); files = writer.getIndexFileNames(); - assertTrue(files.remove("_1.fnx")); + assertTrue(files.remove("1.fnx")); for (String string : files) { assertFalse(string.endsWith(".fnx")); } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexReader.java b/lucene/src/test/org/apache/lucene/index/TestIndexReader.java index 024dd49..7d10fa1 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexReader.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexReader.java @@ -39,7 +39,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader.FieldOption; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldCache; @@ -88,7 +88,6 @@ public class TestIndexReader extends LuceneTestCase writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode( OpenMode.APPEND).setMaxBufferedDocs(2)); - writer.setInfoStream(VERBOSE ? System.out : null); for(int i=0;i<7;i++) addDocumentWithFields(writer); writer.close(); @@ -948,7 +947,7 @@ public class TestIndexReader extends LuceneTestCase writer.close(); SegmentInfos sis = new SegmentInfos(); - sis.read(d); + sis.read(d, CodecProvider.getDefault()); IndexReader r = IndexReader.open(d, false); IndexCommit c = r.getIndexCommit(); @@ -1232,7 +1231,7 @@ public class TestIndexReader extends LuceneTestCase // LUCENE-1609: don't load terms index public void testNoTermsIndex() throws Throwable { Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); Document doc = new Document(); doc.add(newField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", TextField.TYPE_UNSTORED)); doc.add(newField("number", "0 1 2 3 4 5 6 7 8 9", TextField.TYPE_UNSTORED)); @@ -1252,7 +1251,7 @@ public class TestIndexReader extends LuceneTestCase writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())). + setCodecProvider(_TestUtil.alwaysCodec("Standard")). setMergePolicy(newLogMergePolicy(10)) ); writer.addDocument(doc); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java index ca85730..c58a795 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -52,17 +52,12 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; -import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.MockDirectoryWrapper; -import org.apache.lucene.store.NativeFSLockFactory; import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.store.SimpleFSDirectory; -import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; @@ -1942,22 +1937,4 @@ public class TestIndexWriter extends LuceneTestCase { assert(version3 > version2); d.close(); } - - public void testWhetherDeleteAllDeletesWriteLock() throws Exception { - Directory d = newFSDirectory(_TestUtil.getTempDir("TestIndexWriter.testWhetherDeleteAllDeletesWriteLock")); - // Must use SimpleFSLockFactory... NativeFSLockFactory - // somehow "knows" a lock is held against write.lock - // even if you remove that file: - d.setLockFactory(new SimpleFSLockFactory()); - RandomIndexWriter w1 = new RandomIndexWriter(random, d); - w1.deleteAll(); - try { - new RandomIndexWriter(random, d); - fail("should not be able to create another writer"); - } catch (LockObtainFailedException lofe) { - // expected - } - w1.close(); - d.close(); - } } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java index 07759d6..02d7c6d 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java @@ -28,6 +28,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -168,10 +169,8 @@ public class TestIndexWriterCommit extends LuceneTestCase { // them, the merged result can easily be larger than the // sum because the merged FST may use array encoding for // some arcs (which uses more space): - - final String idFormat = _TestUtil.getPostingsFormat("id"); - final String contentFormat = _TestUtil.getPostingsFormat("content"); - assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory")); + assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory")); + assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory")); MockDirectoryWrapper dir = newDirectory(); Analyzer analyzer; if (random.nextBoolean()) { diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java index 85fd1b4..816977e 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java @@ -26,7 +26,6 @@ import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.util.LuceneTestCase; @@ -72,7 +71,6 @@ public class TestIndexWriterConfig extends LuceneTestCase { assertEquals(ThreadAffinityDocumentsWriterThreadPool.class, conf.getIndexerThreadPool().getClass()); assertEquals(FlushByRamOrCountsPolicy.class, conf.getFlushPolicy().getClass()); assertEquals(IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB, conf.getRAMPerThreadHardLimitMB()); - assertEquals(Codec.getDefault(), conf.getCodec()); // Sanity check - validate that all getters are covered. Set getters = new HashSet(); getters.add("getAnalyzer"); @@ -90,6 +88,7 @@ public class TestIndexWriterConfig extends LuceneTestCase { getters.add("getMaxBufferedDocs"); getters.add("getIndexingChain"); getters.add("getMergedSegmentWarmer"); + getters.add("getCodecProvider"); getters.add("getMergePolicy"); getters.add("getMaxThreadStates"); getters.add("getReaderPooling"); @@ -97,7 +96,6 @@ public class TestIndexWriterConfig extends LuceneTestCase { getters.add("getReaderTermsIndexDivisor"); getters.add("getFlushPolicy"); getters.add("getRAMPerThreadHardLimitMB"); - getters.add("getCodec"); for (Method m : IndexWriterConfig.class.getDeclaredMethods()) { if (m.getDeclaringClass() == IndexWriterConfig.class && m.getName().startsWith("get")) { diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java index a2c5b13..2e4365e 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java @@ -31,6 +31,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -896,9 +897,8 @@ public class TestIndexWriterDelete extends LuceneTestCase { } public void testIndexingThenDeleting() throws Exception { - final String fieldFormat = _TestUtil.getPostingsFormat("field"); - assumeFalse("This test cannot run with Memory codec", fieldFormat.equals("Memory")); - assumeFalse("This test cannot run with SimpleText codec", fieldFormat.equals("SimpleText")); + assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory")); + assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText")); final Random r = random; Directory dir = newDirectory(); // note this test explicitly disables payloads diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java index 5c04b75..647d97b 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java @@ -23,7 +23,6 @@ import java.io.PrintStream; import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; -import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Random; @@ -928,10 +927,10 @@ public class TestIndexWriterExceptions extends LuceneTestCase { } catch (RuntimeException re) { // Expected } - assertTrue(dir.fileExists("_1.fnx")); + assertTrue(dir.fileExists("1.fnx")); assertTrue(failure.failOnCommit && failure.failOnDeleteFile); w.rollback(); - assertFalse(dir.fileExists("_1.fnx")); + assertFalse(dir.fileExists("1.fnx")); assertEquals(0, dir.listAll().length); dir.close(); } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java index 46abe85..357a48b 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java @@ -24,7 +24,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -154,10 +154,8 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase { // them, the merged result can easily be larger than the // sum because the merged FST may use array encoding for // some arcs (which uses more space): - - final String idFormat = _TestUtil.getPostingsFormat("id"); - final String contentFormat = _TestUtil.getPostingsFormat("content"); - assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory")); + assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory")); + assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory")); int START_COUNT = 57; int NUM_DIR = TEST_NIGHTLY ? 50 : 5; diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java index ff5e522..c012cda 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java @@ -30,7 +30,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -989,16 +988,15 @@ public class TestIndexWriterReader extends LuceneTestCase { public void testNoTermsIndex() throws Exception { // Some Codecs don't honor the ReaderTermsIndexDivisor, so skip the test if // they're picked. - assumeFalse("PreFlex codec does not support ReaderTermsIndexDivisor!", - "Lucene3x".equals(Codec.getDefault().getName())); + HashSet illegalCodecs = new HashSet(); + illegalCodecs.add("PreFlex"); + illegalCodecs.add("SimpleText"); + illegalCodecs.add("Memory"); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setReaderTermsIndexDivisor(-1); - // Don't proceed if picked Codec is in the list of illegal ones. - final String format = _TestUtil.getPostingsFormat("f"); - assumeFalse("Format: " + format + " does not support ReaderTermsIndexDivisor!", - (format.equals("SimpleText") || format.equals("Memory"))); + if (illegalCodecs.contains(conf.getCodecProvider().getFieldCodec("f"))) return; Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, conf); @@ -1008,7 +1006,7 @@ public class TestIndexWriterReader extends LuceneTestCase { IndexReader r = IndexReader.open(w, true).getSequentialSubReaders()[0]; try { r.termDocsEnum(null, "f", new BytesRef("val")); - fail("should have failed to seek since terms index was not loaded."); + fail("should have failed to seek since terms index was not loaded. Codec used " + conf.getCodecProvider().getFieldCodec("f")); } catch (IllegalStateException e) { // expected - we didn't load the term index } finally { diff --git a/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java b/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java index 1933a4a..65787b0 100755 --- a/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java +++ b/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java @@ -23,6 +23,7 @@ import java.io.Reader; import org.apache.lucene.analysis.*; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.ScoreDoc; @@ -33,7 +34,6 @@ import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util._TestUtil; /** * Tests lazy skipping on the proximity file. @@ -131,10 +131,8 @@ public class TestLazyProxSkipping extends LuceneTestCase { } public void testLazySkipping() throws IOException { - final String fieldFormat = _TestUtil.getPostingsFormat(this.field); - assumeFalse("This test cannot run with Memory codec", fieldFormat.equals("Memory")); - assumeFalse("This test cannot run with SimpleText codec", fieldFormat.equals("SimpleText")); - + assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("SimpleText")); + assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("Memory")); // test whether only the minimum amount of seeks() // are performed performTest(5); diff --git a/lucene/src/test/org/apache/lucene/index/TestLongPostings.java b/lucene/src/test/org/apache/lucene/index/TestLongPostings.java index 84f5f52..034c6f1 100644 --- a/lucene/src/test/org/apache/lucene/index/TestLongPostings.java +++ b/lucene/src/test/org/apache/lucene/index/TestLongPostings.java @@ -29,6 +29,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; diff --git a/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java b/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java index 67d3027..0a911ec 100644 --- a/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java +++ b/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java @@ -25,7 +25,6 @@ import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -69,7 +68,7 @@ public class TestMultiLevelSkipList extends LuceneTestCase { public void testSimpleSkip() throws IOException { Directory dir = new CountingRAMDirectory(new RAMDirectory()); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new PayloadAnalyzer()).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())).setMergePolicy(newLogMergePolicy())); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new PayloadAnalyzer()).setCodecProvider(_TestUtil.alwaysCodec("Standard")).setMergePolicy(newLogMergePolicy())); Term term = new Term("test", "a"); for (int i = 0; i < 5000; i++) { Document d1 = new Document(); diff --git a/lucene/src/test/org/apache/lucene/index/codecs/perfield/TestPerFieldPostingsFormat.java b/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java similarity index 55% rename from lucene/src/test/org/apache/lucene/index/codecs/perfield/TestPerFieldPostingsFormat.java rename to lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java index d178403..f55b50f 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/perfield/TestPerFieldPostingsFormat.java +++ b/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.perfield; +package org.apache.lucene.index; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -25,18 +25,17 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.LogDocMergePolicy; -import org.apache.lucene.index.Term; +import org.apache.lucene.index.CheckIndex.Status.SegmentInfoStatus; +import org.apache.lucene.index.CheckIndex.Status; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.PostingsFormat; -import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; -import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; -import org.apache.lucene.index.codecs.simpletext.SimpleTextPostingsFormat; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec; +import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec; +import org.apache.lucene.index.codecs.mocksep.MockSepCodec; +import org.apache.lucene.index.codecs.pulsing.PulsingCodec; +import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; +import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; @@ -49,10 +48,7 @@ import org.junit.Test; * * */ -//TODO: would be better in this test to pull termsenums and instanceof or something? -// this way we can verify PFPF is doing the right thing. -// for now we do termqueries. -public class TestPerFieldPostingsFormat extends LuceneTestCase { +public class TestPerFieldCodecSupport extends LuceneTestCase { private IndexWriter newWriter(Directory dir, IndexWriterConfig conf) throws IOException { @@ -92,13 +88,15 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { } /* - * Test that heterogeneous index segments are merge successfully + * Test is hetrogenous index segements are merge sucessfully */ @Test public void testMergeUnusedPerFieldCodec() throws IOException { Directory dir = newDirectory(); + CodecProvider provider = new MockCodecProvider(); IndexWriterConfig iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodec(new MockCodec()); + new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodecProvider( + provider); IndexWriter writer = newWriter(dir, iwconf); addDocs(writer, 10); writer.commit(); @@ -107,7 +105,7 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { addDocs2(writer, 10); writer.commit(); assertEquals(30, writer.maxDoc()); - _TestUtil.checkIndex(dir); + _TestUtil.checkIndex(dir, provider); writer.optimize(); assertEquals(30, writer.maxDoc()); writer.close(); @@ -117,22 +115,22 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { /* * Test that heterogeneous index segments are merged sucessfully */ - // TODO: not sure this test is that great, we should probably peek inside PerFieldPostingsFormat or something?! @Test public void testChangeCodecAndMerge() throws IOException { Directory dir = newDirectory(); + CodecProvider provider = new MockCodecProvider(); if (VERBOSE) { System.out.println("TEST: make new index"); } IndexWriterConfig iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodec(new MockCodec()); + new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodecProvider(provider); iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10); IndexWriter writer = newWriter(dir, iwconf); addDocs(writer, 10); writer.commit(); - assertQuery(new Term("content", "aaa"), dir, 10); + assertQuery(new Term("content", "aaa"), dir, 10, provider); if (VERBOSE) { System.out.println("TEST: addDocs3"); } @@ -140,17 +138,19 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { writer.commit(); writer.close(); - assertQuery(new Term("content", "ccc"), dir, 10); - assertQuery(new Term("content", "aaa"), dir, 10); - Lucene40Codec codec = (Lucene40Codec)iwconf.getCodec(); + assertQuery(new Term("content", "ccc"), dir, 10, provider); + assertQuery(new Term("content", "aaa"), dir, 10, provider); + assertCodecPerField(_TestUtil.checkIndex(dir, provider), "content", + provider.lookup("MockSep")); iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) - .setOpenMode(OpenMode.APPEND).setCodec(codec); + .setOpenMode(OpenMode.APPEND).setCodecProvider(provider); //((LogMergePolicy) iwconf.getMergePolicy()).setUseCompoundFile(false); //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10); iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); - iwconf.setCodec(new MockCodec2()); // uses standard for field content + provider = new MockCodecProvider2(); // uses standard for field content + iwconf.setCodecProvider(provider); writer = newWriter(dir, iwconf); // swap in new codec for currently written segments if (VERBOSE) { @@ -158,22 +158,23 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { } addDocs2(writer, 10); writer.commit(); - codec = (Lucene40Codec)iwconf.getCodec(); - PostingsFormat origContentCodec = PostingsFormat.forName("MockSep"); - PostingsFormat newContentCodec = PostingsFormat.forName("Lucene40"); + Codec origContentCodec = provider.lookup("MockSep"); + Codec newContentCodec = provider.lookup("Standard"); + assertHybridCodecPerField(_TestUtil.checkIndex(dir, provider), "content", + origContentCodec, origContentCodec, newContentCodec); assertEquals(30, writer.maxDoc()); - assertQuery(new Term("content", "bbb"), dir, 10); - assertQuery(new Term("content", "ccc"), dir, 10); //// - assertQuery(new Term("content", "aaa"), dir, 10); + assertQuery(new Term("content", "bbb"), dir, 10, provider); + assertQuery(new Term("content", "ccc"), dir, 10, provider); //// + assertQuery(new Term("content", "aaa"), dir, 10, provider); if (VERBOSE) { System.out.println("TEST: add more docs w/ new codec"); } addDocs2(writer, 10); writer.commit(); - assertQuery(new Term("content", "ccc"), dir, 10); - assertQuery(new Term("content", "bbb"), dir, 20); - assertQuery(new Term("content", "aaa"), dir, 10); + assertQuery(new Term("content", "ccc"), dir, 10, provider); + assertQuery(new Term("content", "bbb"), dir, 20, provider); + assertQuery(new Term("content", "aaa"), dir, 10, provider); assertEquals(40, writer.maxDoc()); if (VERBOSE) { @@ -182,19 +183,52 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { writer.optimize(); assertEquals(40, writer.maxDoc()); writer.close(); - assertQuery(new Term("content", "ccc"), dir, 10); - assertQuery(new Term("content", "bbb"), dir, 20); - assertQuery(new Term("content", "aaa"), dir, 10); + assertCodecPerFieldOptimized(_TestUtil.checkIndex(dir, provider), + "content", newContentCodec); + assertQuery(new Term("content", "ccc"), dir, 10, provider); + assertQuery(new Term("content", "bbb"), dir, 20, provider); + assertQuery(new Term("content", "aaa"), dir, 10, provider); dir.close(); } - public void assertQuery(Term t, Directory dir, int num) + public void assertCodecPerFieldOptimized(Status checkIndex, String field, + Codec codec) { + assertEquals(1, checkIndex.segmentInfos.size()); + final CodecProvider provider = checkIndex.segmentInfos.get(0).codec.provider; + assertEquals(codec, provider.lookup(provider.getFieldCodec(field))); + + } + + public void assertCodecPerField(Status checkIndex, String field, Codec codec) { + for (SegmentInfoStatus info : checkIndex.segmentInfos) { + final CodecProvider provider = info.codec.provider; + assertEquals(codec, provider.lookup(provider.getFieldCodec(field))); + } + } + + public void assertHybridCodecPerField(Status checkIndex, String field, + Codec... codec) throws IOException { + List segmentInfos = checkIndex.segmentInfos; + assertEquals(segmentInfos.size(), codec.length); + for (int i = 0; i < codec.length; i++) { + SegmentCodecs codecInfo = segmentInfos.get(i).codec; + FieldInfos fieldInfos = new FieldInfos(checkIndex.dir, IndexFileNames + .segmentFileName(segmentInfos.get(i).name, "", + IndexFileNames.FIELD_INFOS_EXTENSION)); + FieldInfo fieldInfo = fieldInfos.fieldInfo(field); + assertEquals("faild for segment index: " + i, codec[i], + codecInfo.codecs[fieldInfo.getCodecId()]); + } + } + + public void assertQuery(Term t, Directory dir, int num, CodecProvider codecs) throws CorruptIndexException, IOException { if (VERBOSE) { System.out.println("\nTEST: assertQuery " + t); } - IndexReader reader = IndexReader.open(dir, null, true, 1); + IndexReader reader = IndexReader.open(dir, null, true, + IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, codecs); IndexSearcher searcher = newSearcher(reader); TopDocs search = searcher.search(new TermQuery(t), num + 10); assertEquals(num, search.totalHits); @@ -203,34 +237,33 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { } - public static class MockCodec extends Lucene40Codec { - final PostingsFormat lucene40 = new Lucene40PostingsFormat(); - final PostingsFormat simpleText = new SimpleTextPostingsFormat(); - final PostingsFormat mockSep = new MockSepPostingsFormat(); - - @Override - public PostingsFormat getPostingsFormatForField(String field) { - if (field.equals("id")) { - return simpleText; - } else if (field.equals("content")) { - return mockSep; - } else { - return lucene40; - } + public static class MockCodecProvider extends CodecProvider { + + public MockCodecProvider() { + StandardCodec standardCodec = new StandardCodec(); + setDefaultFieldCodec(standardCodec.name); + SimpleTextCodec simpleTextCodec = new SimpleTextCodec(); + MockSepCodec mockSepCodec = new MockSepCodec(); + register(standardCodec); + register(mockSepCodec); + register(simpleTextCodec); + setFieldCodec("id", simpleTextCodec.name); + setFieldCodec("content", mockSepCodec.name); } } - public static class MockCodec2 extends Lucene40Codec { - final PostingsFormat lucene40 = new Lucene40PostingsFormat(); - final PostingsFormat simpleText = new SimpleTextPostingsFormat(); - - @Override - public PostingsFormat getPostingsFormatForField(String field) { - if (field.equals("id")) { - return simpleText; - } else { - return lucene40; - } + public static class MockCodecProvider2 extends CodecProvider { + + public MockCodecProvider2() { + StandardCodec standardCodec = new StandardCodec(); + setDefaultFieldCodec(standardCodec.name); + SimpleTextCodec simpleTextCodec = new SimpleTextCodec(); + MockSepCodec mockSepCodec = new MockSepCodec(); + register(standardCodec); + register(mockSepCodec); + register(simpleTextCodec); + setFieldCodec("id", simpleTextCodec.name); + setFieldCodec("content", standardCodec.name); } } @@ -243,10 +276,23 @@ public class TestPerFieldPostingsFormat extends LuceneTestCase { final int docsPerRound = 97; int numRounds = atLeast(1); for (int i = 0; i < numRounds; i++) { + CodecProvider provider = new CodecProvider(); + Codec[] codecs = new Codec[] { new StandardCodec(), + new SimpleTextCodec(), new MockSepCodec(), + new PulsingCodec(1 + random.nextInt(20)), + new MockVariableIntBlockCodec(1 + random.nextInt(10)), + new MockFixedIntBlockCodec(1 + random.nextInt(10)) }; + for (Codec codec : codecs) { + provider.register(codec); + } int num = _TestUtil.nextInt(random, 30, 60); + for (int j = 0; j < num; j++) { + provider.setFieldCodec("" + j, codecs[random.nextInt(codecs.length)].name); + } IndexWriterConfig config = newIndexWriterConfig(random, TEST_VERSION_CURRENT, new MockAnalyzer(random)); config.setOpenMode(OpenMode.CREATE_OR_APPEND); + config.setCodecProvider(provider); IndexWriter writer = newWriter(dir, config); for (int j = 0; j < docsPerRound; j++) { final Document doc = new Document(); diff --git a/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java b/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java index 9590215..1e994d1 100644 --- a/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java +++ b/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java @@ -22,6 +22,7 @@ import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.store.AlreadyClosedException; diff --git a/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java b/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java index e0e7c2f..248baf0 100644 --- a/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java +++ b/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java @@ -19,8 +19,7 @@ package org.apache.lucene.index; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.*; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.memory.MemoryPostingsFormat; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.*; import org.apache.lucene.util.*; import org.junit.Test; @@ -36,12 +35,13 @@ public class TestRollingUpdates extends LuceneTestCase { dir.setCheckIndexOnClose(false); // we use a custom codec provider final LineFileDocs docs = new LineFileDocs(random); + CodecProvider provider = CodecProvider.getDefault(); //provider.register(new MemoryCodec()); - if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random.nextBoolean()) { - Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat())); + if ( (!"PreFlex".equals(provider.getDefaultFieldCodec())) && random.nextBoolean()) { + provider.setFieldCodec("docid", "Memory"); } - final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); + final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(provider)); w.setInfoStream(VERBOSE ? System.out : null); final int SIZE = atLeast(TEST_NIGHTLY ? 100 : 20); int id = 0; diff --git a/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java index fa6c256..0348bd8 100644 --- a/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java +++ b/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java @@ -23,7 +23,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.util.BytesRef; import java.io.IOException; @@ -75,7 +74,7 @@ public class TestSegmentMerger extends LuceneTestCase { } public void testMerge() throws IOException { - SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, null, new FieldInfos(), Codec.getDefault(), newIOContext(random)); + SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, null, new FieldInfos(), newIOContext(random)); merger.add(reader1); merger.add(reader2); int docsMerged = merger.merge(); @@ -83,7 +82,7 @@ public class TestSegmentMerger extends LuceneTestCase { final FieldInfos fieldInfos = merger.fieldInfos(); //Should be able to open a new SegmentReader against the new directory SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, - merger.getCodec(), fieldInfos), + merger.getSegmentCodecs(), fieldInfos), true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); assertTrue(mergedReader != null); assertTrue(mergedReader.numDocs() == 2); @@ -146,7 +145,7 @@ public class TestSegmentMerger extends LuceneTestCase { w.close(); // Assert that SM fails if .del exists - SegmentMerger sm = new SegmentMerger(dir, 1, "a", null, null, null, Codec.getDefault(), newIOContext(random)); + SegmentMerger sm = new SegmentMerger(dir, 1, "a", null, null, null, newIOContext(random)); boolean doFail = false; try { sm.createCompoundFile("b1", w.segmentInfos.info(0), newIOContext(random)); diff --git a/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java b/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java index c1a393e..bd0fefc 100644 --- a/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java +++ b/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java @@ -26,7 +26,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.store.Directory; @@ -75,7 +74,7 @@ public class TestSegmentTermEnum extends LuceneTestCase { public void testPrevTermAtEnd() throws IOException { - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); addDoc(writer, "aaa bbb"); writer.close(); SegmentReader reader = getOnlySegmentReader(IndexReader.open(dir, false)); diff --git a/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java b/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java index c945fbc..00bcbfb 100644 --- a/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java +++ b/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java @@ -358,7 +358,11 @@ public class TestTermsEnum extends LuceneTestCase { IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); /* - iwc.setCodec(new StandardCodec(minTermsInBlock, maxTermsInBlock)); + CoreCodecProvider cp = new CoreCodecProvider(); + cp.unregister(cp.lookup("Standard")); + cp.register(new StandardCodec(minTermsInBlock, maxTermsInBlock)); + cp.setDefaultFieldCodec("Standard"); + iwc.setCodecProvider(cp); */ final RandomIndexWriter w = new RandomIndexWriter(random, d, iwc); diff --git a/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java b/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java index 2d3d334..dad404e 100644 --- a/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java +++ b/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java @@ -29,7 +29,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.CheckHits; import org.apache.lucene.search.IndexSearcher; @@ -57,7 +57,7 @@ public class TestTermsEnum2 extends LuceneTestCase { super.setUp(); // we generate aweful regexps: good for testing. // but for preflex codec, the test can be very slow, so use less iterations. - numIterations = Codec.getDefault().getName().equals("Lucene3x") ? 10 * RANDOM_MULTIPLIER : atLeast(50); + numIterations = CodecProvider.getDefault().getFieldCodec("field").equals("PreFlex") ? 10 * RANDOM_MULTIPLIER : atLeast(50); dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, diff --git a/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java b/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java index 2762aae..dbd01a0 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java @@ -27,7 +27,7 @@ public class TestIntBlockCodec extends LuceneTestCase { public void testSimpleIntBlocks() throws Exception { Directory dir = newDirectory(); - IntStreamFactory f = new MockFixedIntBlockPostingsFormat(128).getIntFactory(); + IntStreamFactory f = new MockFixedIntBlockCodec(128).getIntFactory(); IntIndexOutput out = f.createOutput(dir, "test", newIOContext(random)); for(int i=0;i<11777;i++) { @@ -49,7 +49,7 @@ public class TestIntBlockCodec extends LuceneTestCase { public void testEmptySimpleIntBlocks() throws Exception { Directory dir = newDirectory(); - IntStreamFactory f = new MockFixedIntBlockPostingsFormat(128).getIntFactory(); + IntStreamFactory f = new MockFixedIntBlockCodec(128).getIntFactory(); IntIndexOutput out = f.createOutput(dir, "test", newIOContext(random)); // write no ints diff --git a/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestImpersonation.java b/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestImpersonation.java deleted file mode 100644 index 25d4331..0000000 diff --git a/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestSurrogates.java b/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java similarity index 97% rename from lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestSurrogates.java rename to lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java index fdf3a42..03051da 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestSurrogates.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -27,15 +27,9 @@ import org.apache.lucene.util.*; import java.util.*; import java.io.IOException; -import org.junit.BeforeClass; import org.junit.Test; public class TestSurrogates extends LuceneTestCase { - /** we will manually instantiate preflex-rw here */ - @BeforeClass - public static void beforeClass() { - LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true; - } private static String makeDifficultRandomUnicodeString(Random r) { final int end = r.nextInt(20); @@ -288,7 +282,7 @@ public class TestSurrogates extends LuceneTestCase { RandomIndexWriter w = new RandomIndexWriter(random, dir, newIndexWriterConfig( TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setCodec(new PreFlexRWCodec())); + new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec(new PreFlexRWCodec()))); final int numField = _TestUtil.nextInt(random, 2, 5); diff --git a/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestTermInfosReaderIndex.java b/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java similarity index 91% rename from lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestTermInfosReaderIndex.java rename to lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java index b78db06..d42efa3 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestTermInfosReaderIndex.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.lucene.index.codecs.preflex; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -39,6 +39,9 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.CoreCodecProvider; import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; @@ -46,9 +49,9 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; -import org.junit.BeforeClass; public class TestTermInfosReaderIndex extends LuceneTestCase { @@ -62,12 +65,6 @@ public class TestTermInfosReaderIndex extends LuceneTestCase { private IndexReader reader; private List sampleTerms; - /** we will manually instantiate preflex-rw here */ - @BeforeClass - public static void beforeClass() { - LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true; - } - @Override public void setUp() throws Exception { super.setUp(); @@ -81,10 +78,10 @@ public class TestTermInfosReaderIndex extends LuceneTestCase { r.close(); FieldInfos fieldInfos = new FieldInfos(directory, IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELD_INFOS_EXTENSION)); - String segmentFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); + String segmentFileName = IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION); long tiiFileLength = directory.fileLength(segmentFileName); IndexInput input = directory.openInput(segmentFileName, newIOContext(random)); - termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), newIOContext(random)), fieldInfos, false); + termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_EXTENSION), newIOContext(random)), fieldInfos, false); int totalIndexInterval = termEnum.indexInterval * indexDivisor; SegmentTermEnum indexEnum = new SegmentTermEnum(input, fieldInfos, true); @@ -167,7 +164,11 @@ public class TestTermInfosReaderIndex extends LuceneTestCase { private int populate(Directory directory) throws CorruptIndexException, LockObtainFailedException, IOException { IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.KEYWORD, false)); - config.setCodec(new PreFlexRWCodec()); + CoreCodecProvider cp = new CoreCodecProvider(); + cp.unregister(cp.lookup("PreFlex")); + cp.register(new PreFlexRWCodec()); + cp.setDefaultFieldCodec("PreFlex"); + config.setCodecProvider(cp); // turn off compound file, this test will open some index files directly. LogMergePolicy mp = newLogMergePolicy(); mp.setUseCompoundFile(false); diff --git a/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java b/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java index 4ce4a0d..b1c1761 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java @@ -35,8 +35,7 @@ import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.PostingsFormat; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsBaseFormat; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; @@ -52,13 +51,13 @@ import org.junit.Ignore; public class Test10KPulsings extends LuceneTestCase { public void test10kPulsed() throws Exception { // we always run this test with pulsing codec. - Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1)); + CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); File f = _TestUtil.getTempDir("10kpulsed"); MockDirectoryWrapper dir = newFSDirectory(f); dir.setCheckIndexOnClose(false); // we do this ourselves explicitly RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); Document document = new Document(); FieldType ft = new FieldType(TextField.TYPE_STORED); @@ -102,14 +101,13 @@ public class Test10KPulsings extends LuceneTestCase { */ public void test10kNotPulsed() throws Exception { // we always run this test with pulsing codec. - int freqCutoff = _TestUtil.nextInt(random, 1, 10); - Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(freqCutoff)); + CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); File f = _TestUtil.getTempDir("10knotpulsed"); MockDirectoryWrapper dir = newFSDirectory(f); dir.setCheckIndexOnClose(false); // we do this ourselves explicitly RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); Document document = new Document(); FieldType ft = new FieldType(TextField.TYPE_STORED); @@ -125,7 +123,10 @@ public class Test10KPulsings extends LuceneTestCase { NumberFormat df = new DecimalFormat("00000", new DecimalFormatSymbols(Locale.ENGLISH)); - final int freq = freqCutoff + 1; + Codec codec = cp.lookup(cp.getFieldCodec("field")); + assertTrue(codec instanceof PulsingCodec); + PulsingCodec pulsing = (PulsingCodec) codec; + final int freq = pulsing.getFreqCutoff() + 1; for (int i = 0; i < 10050; i++) { StringBuilder sb = new StringBuilder(); diff --git a/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java b/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java index c1addf5..e4dcc84 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java @@ -17,8 +17,10 @@ package org.apache.lucene.index.codecs.pulsing; * limitations under the License. */ +import java.io.IOException; import java.util.IdentityHashMap; import java.util.Map; +import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -28,10 +30,27 @@ import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.BlockTreeTermsReader; +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.nestedpulsing.NestedPulsingPostingsFormat; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; +import org.apache.lucene.index.codecs.DefaultDocValuesProducer; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.index.codecs.standard.StandardPostingsReader; +import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; @@ -44,10 +63,10 @@ public class TestPulsingReuse extends LuceneTestCase { // TODO: this is a basic test. this thing is complicated, add more public void testSophisticatedReuse() throws Exception { // we always run this test with pulsing codec. - Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1)); + CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); Document doc = new Document(); doc.add(new Field("foo", "a b b c c c d e f g g h i i j j k", TextField.TYPE_UNSTORED)); iw.addDocument(doc); @@ -82,11 +101,11 @@ public class TestPulsingReuse extends LuceneTestCase { /** tests reuse with Pulsing1(Pulsing2(Standard)) */ public void testNestedPulsing() throws Exception { // we always run this test with pulsing codec. - Codec cp = _TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat()); + CodecProvider cp = _TestUtil.alwaysCodec(new NestedPulsing()); MockDirectoryWrapper dir = newDirectory(); dir.setCheckIndexOnClose(false); // will do this ourselves, custom codec RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); Document doc = new Document(); doc.add(new Field("foo", "a b b c c c d e f g g g h i i j j k l l m m m", TextField.TYPE_UNSTORED)); // note: the reuse is imperfect, here we would have 4 enums (lost reuse when we get an enum for 'm') @@ -119,7 +138,79 @@ public class TestPulsingReuse extends LuceneTestCase { ir.close(); CheckIndex ci = new CheckIndex(dir); - ci.checkIndex(null); + ci.checkIndex(null, cp); dir.close(); } + + static class NestedPulsing extends Codec { + public NestedPulsing() { + super("NestedPulsing"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docsWriter = new StandardPostingsWriter(state); + + PostingsWriterBase pulsingWriterInner = new PulsingPostingsWriter(2, docsWriter); + PostingsWriterBase pulsingWriter = new PulsingPostingsWriter(1, pulsingWriterInner); + + // Terms dict + boolean success = false; + try { + FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, + BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); + success = true; + return ret; + } finally { + if (!success) { + pulsingWriter.close(); + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); + PostingsReaderBase pulsingReaderInner = new PulsingPostingsReader(docsReader); + PostingsReaderBase pulsingReader = new PulsingPostingsReader(pulsingReaderInner); + boolean success = false; + try { + FieldsProducer ret = new BlockTreeTermsReader( + state.dir, state.fieldInfos, state.segmentInfo.name, + pulsingReader, + state.context, + state.codecId, + state.termsIndexDivisor); + success = true; + return ret; + } finally { + if (!success) { + pulsingReader.close(); + } + } + } + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new DefaultDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new DefaultDocValuesProducer(state); + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { + StandardPostingsReader.files(dir, segmentInfo, id, files); + BlockTreeTermsReader.files(dir, segmentInfo, id, files); + DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); + } + + @Override + public void getExtensions(Set extensions) { + StandardCodec.getStandardExtensions(extensions); + DefaultDocValuesConsumer.getExtensions(extensions); + } + } } diff --git a/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java b/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java index a28465a..526fc2b 100644 --- a/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java +++ b/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java @@ -39,7 +39,7 @@ import org.apache.lucene.index.LogMergePolicy; import org.apache.lucene.index.MultiPerDocValues; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.search.*; @@ -65,7 +65,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { @Before public void setUp() throws Exception { super.setUp(); - assumeFalse("cannot work with preflex codec", Codec.getDefault().getName().equals("Lucene3x")); + assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); } /* @@ -140,7 +140,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { indexValues(w_1, valuesPerIndex, first, values, false, 7); w_1.commit(); assertEquals(valuesPerIndex, w_1.maxDoc()); - _TestUtil.checkIndex(d_1); + _TestUtil.checkIndex(d_1, w_1.getConfig().getCodecProvider()); // index second index Directory d_2 = newDirectory(); @@ -148,7 +148,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { indexValues(w_2, valuesPerIndex, second, values, false, 7); w_2.commit(); assertEquals(valuesPerIndex, w_2.maxDoc()); - _TestUtil.checkIndex(d_2); + _TestUtil.checkIndex(d_2, w_2.getConfig().getCodecProvider()); Directory target = newDirectory(); IndexWriter w = new IndexWriter(target, writerConfig(random.nextBoolean())); @@ -162,7 +162,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { w.optimize(true); w.commit(); - _TestUtil.checkIndex(target); + _TestUtil.checkIndex(target, w.getConfig().getCodecProvider()); assertEquals(valuesPerIndex * 2, w.maxDoc()); // check values @@ -546,6 +546,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { } public void testMultiValuedIndexDocValuesField() throws Exception { + assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); Directory d = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random, d); Document doc = new Document(); @@ -574,6 +575,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { } public void testDifferentTypedDocValuesField() throws Exception { + assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); Directory d = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random, d); Document doc = new Document(); diff --git a/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java b/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java index 6bb4341..53733af 100644 --- a/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java +++ b/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java @@ -15,7 +15,7 @@ import org.apache.lucene.index.IndexReader.ReaderContext; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -42,7 +42,8 @@ public class TestTypePromotion extends LuceneTestCase { @Before public void setUp() throws Exception { super.setUp(); - assumeFalse("cannot work with preflex codec", Codec.getDefault().getName().equals("Lucene3x")); + assumeFalse("cannot work with preflex codec", CodecProvider.getDefault() + .getDefaultFieldCodec().equals("PreFlex")); } private static EnumSet INTEGERS = EnumSet.of(ValueType.VAR_INTS, diff --git a/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java b/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java index 21463ac..3099014 100644 --- a/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java +++ b/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java @@ -28,7 +28,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.search.similarities.Similarity; @@ -49,7 +49,7 @@ public class TestDocValuesScoring extends LuceneTestCase { public void testSimple() throws Exception { assumeFalse("PreFlex codec cannot work with IndexDocValues!", - "Lucene3x".equals(Codec.getDefault().getName())); + "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir); diff --git a/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java b/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java index e43ab23..000c13a 100644 --- a/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java +++ b/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java @@ -30,7 +30,7 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; @@ -60,8 +60,8 @@ public class TestPrefixRandom extends LuceneTestCase { // we generate aweful prefixes: good for testing. // but for preflex codec, the test can be very slow, so use less iterations. - final String codec = Codec.getDefault().getName(); - int num = codec.equals("Lucene3x") ? 200 * RANDOM_MULTIPLIER : atLeast(1000); + final String codec = CodecProvider.getDefault().getFieldCodec("field"); + int num = codec.equals("PreFlex") ? 200 * RANDOM_MULTIPLIER : atLeast(1000); for (int i = 0; i < num; i++) { field.setValue(_TestUtil.randomUnicodeString(random, 10)); writer.addDocument(doc); diff --git a/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java b/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java index 9fdd6a9..83db492 100644 --- a/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java +++ b/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java @@ -33,7 +33,7 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; @@ -143,7 +143,7 @@ public class TestRegexpRandom2 extends LuceneTestCase { public void testRegexps() throws Exception { // we generate aweful regexps: good for testing. // but for preflex codec, the test can be very slow, so use less iterations. - int num = Codec.getDefault().getName().equals("Lucene3x") ? 100 * RANDOM_MULTIPLIER : atLeast(1000); + int num = CodecProvider.getDefault().getFieldCodec(fieldName).equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : atLeast(1000); for (int i = 0; i < num; i++) { String reg = AutomatonTestUtil.randomRegexp(random); if (VERBOSE) { diff --git a/lucene/src/test/org/apache/lucene/search/TestSort.java b/lucene/src/test/org/apache/lucene/search/TestSort.java index a45c40f..4b82e1f 100644 --- a/lucene/src/test/org/apache/lucene/search/TestSort.java +++ b/lucene/src/test/org/apache/lucene/search/TestSort.java @@ -40,7 +40,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.FieldValueHitQueue.Entry; @@ -70,7 +70,7 @@ import org.junit.BeforeClass; public class TestSort extends LuceneTestCase { // true if our codec supports docvalues: true unless codec is preflex (3.x) - boolean supportsDocValues = Codec.getDefault().getName().equals("Lucene3x") == false; + boolean supportsDocValues = CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex") == false; private static int NUM_STRINGS; private IndexSearcher full; private IndexSearcher searchX; diff --git a/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java b/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java index aff92fc..8591be6 100644 --- a/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java +++ b/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java @@ -29,7 +29,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.OrdTermState; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -566,7 +566,7 @@ public class TestSimilarityBase extends LuceneTestCase { /** Test whether all similarities return document 3 before documents 7 and 8. */ public void testHeartRanking() throws IOException { assumeFalse("PreFlex codec does not support the stats necessary for this test!", - "Lucene3x".equals(Codec.getDefault().getName())); + "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); Query q = new TermQuery(new Term(FIELD_BODY, "heart")); diff --git a/lucene/src/test/org/apache/lucene/util/TestNamedSPILoader.java b/lucene/src/test/org/apache/lucene/util/TestNamedSPILoader.java deleted file mode 100644 index 60c2a49..0000000 diff --git a/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java index 68b264e..33496b6 100644 --- a/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java +++ b/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java @@ -40,8 +40,7 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; @@ -1014,11 +1013,10 @@ public class TestFSTs extends LuceneTestCase { // file, up until a time limit public void testRealTerms() throws Exception { - // TODO: is this necessary? we use the annotation... - final String defaultFormat = _TestUtil.getPostingsFormat("abracadabra"); - if (defaultFormat.equals("SimpleText") || defaultFormat.equals("Memory")) { + final String defaultCodec = CodecProvider.getDefault().getDefaultFieldCodec(); + if (defaultCodec.equals("SimpleText") || defaultCodec.equals("Memory")) { // no - Codec.setDefault(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())); + CodecProvider.getDefault().setDefaultFieldCodec("Standard"); } final LineFileDocs docs = new LineFileDocs(random); diff --git a/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java b/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java index 00c035b..e3a0a29 100644 --- a/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java +++ b/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java @@ -20,7 +20,7 @@ package org.apache.lucene.collation; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CollationTestBase; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.BytesRef; import java.text.Collator; @@ -47,7 +47,7 @@ public class TestCollationKeyAnalyzer extends CollationTestBase { @Override public void setUp() throws Exception { super.setUp(); - assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); } public void testFarsiRangeFilterCollating() throws Exception { diff --git a/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java b/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java index 681290a..305ee27 100644 --- a/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java +++ b/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java @@ -22,7 +22,7 @@ import com.ibm.icu.text.Collator; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CollationTestBase; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.BytesRef; import java.util.Locale; @@ -45,7 +45,7 @@ public class TestICUCollationKeyAnalyzer extends CollationTestBase { @Override public void setUp() throws Exception { super.setUp(); - assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); } public void testFarsiRangeFilterCollating() throws Exception { diff --git a/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java b/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java index ac00ffa..217818c 100644 --- a/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java +++ b/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java @@ -33,7 +33,7 @@ import org.apache.lucene.index.NoDeletionPolicy; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.NoMergeScheduler; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.Version; @@ -133,12 +133,7 @@ public class CreateIndexTask extends PerfTask { final String defaultCodec = config.get("default.codec", null); if (defaultCodec != null) { - try { - Class clazz = Class.forName(defaultCodec).asSubclass(Codec.class); - Codec.setDefault(clazz.newInstance()); - } catch (Exception e) { - throw new RuntimeException("Couldn't instantiate Codec: " + defaultCodec, e); - } + CodecProvider.getDefault().setDefaultFieldCodec(defaultCodec); } final String mergePolicy = config.get("merge.policy", diff --git a/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java b/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java index 97c4eed..253d82e 100755 --- a/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java +++ b/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java @@ -486,17 +486,11 @@ public class SpellChecker implements java.io.Closeable { * @throws IOException */ public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB, boolean optimize) throws IOException { - IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null) - .setRAMBufferSizeMB(ramMB); - ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(mergeFactor); - indexDictionary(dict, config, optimize); - } - - public final void indexDictionary(Dictionary dict, IndexWriterConfig config, boolean optimize) throws IOException { synchronized (modifyCurrentIndexLock) { ensureOpen(); final Directory dir = this.spellIndex; - final IndexWriter writer = new IndexWriter(dir, config); + final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, null).setRAMBufferSizeMB(ramMB)); + ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setMaxMergeAtOnce(mergeFactor); IndexSearcher indexSearcher = obtainSearcher(); final List termsEnums = new ArrayList(); diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index c6e5359..c12c10a 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -133,8 +133,10 @@ New Features fq={!join from=name to=parent}eyes:blue (yonik) -* SOLR-1942: Added the ability to select postings format per fieldType in schema.xml - as well as support custom Codecs in solrconfig.xml. +* SOLR-1942: Added the ability to select codec per fieldType in schema.xml + as well as support custom CodecProviders in solrconfig.xml. + NOTE: IndexReaderFactory now has a codecProvider that should be passed + to IndexReader.open (in the case you have a custom IndexReaderFactory). (simonw via rmuir) * SOLR-2136: Boolean type added to function queries, along with diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java index 8d2ebbe..63fcfbc 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java @@ -22,7 +22,7 @@ import java.io.FileOutputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; @@ -37,7 +37,7 @@ public class TestICUCollationField extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); String home = setupSolrHome(); initCore("solrconfig.xml","schema.xml", home); // add some docs diff --git a/solr/core/src/java/org/apache/solr/core/CodecFactory.java b/solr/core/src/java/org/apache/solr/core/CodecProviderFactory.java similarity index 79% rename from solr/core/src/java/org/apache/solr/core/CodecFactory.java rename to solr/core/src/java/org/apache/solr/core/CodecProviderFactory.java index e047181..e3bc985 100644 --- a/solr/core/src/java/org/apache/solr/core/CodecFactory.java +++ b/solr/core/src/java/org/apache/solr/core/CodecProviderFactory.java @@ -17,17 +17,16 @@ package org.apache.solr.core; * limitations under the License. */ -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.solr.common.util.NamedList; -import org.apache.solr.schema.IndexSchema; import org.apache.solr.util.plugin.NamedListInitializedPlugin; /** - * Factory for plugging in a custom {@link Codec} + * Factory for plugging in a custom {@link CodecProvider} */ -public abstract class CodecFactory implements NamedListInitializedPlugin { +public abstract class CodecProviderFactory implements NamedListInitializedPlugin { public void init(NamedList args) { } - public abstract Codec create(IndexSchema Schema); + public abstract CodecProvider create(); } diff --git a/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java b/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java index 4204a9e..fb87650 100644 --- a/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java +++ b/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java @@ -19,6 +19,7 @@ package org.apache.solr.core; import java.io.IOException; import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.solr.common.util.NamedList; import org.apache.solr.util.plugin.NamedListInitializedPlugin; @@ -28,6 +29,7 @@ import org.apache.solr.util.plugin.NamedListInitializedPlugin; */ public abstract class IndexReaderFactory implements NamedListInitializedPlugin { protected int termInfosIndexDivisor = 1;//IndexReader.DEFAULT_TERMS_INDEX_DIVISOR; Set this once Lucene makes this public. + protected CodecProvider provider; /** * Potentially initializes {@link #termInfosIndexDivisor}. Overriding classes should call super.init() in order * to make sure termInfosIndexDivisor is set. @@ -63,4 +65,11 @@ public abstract class IndexReaderFactory implements NamedListInitializedPlugin { */ public abstract IndexReader newReader(Directory indexDir, boolean readOnly) throws IOException; + + /** + * Sets the codec provider for this IndexReaderFactory + */ + public void setCodecProvider(CodecProvider provider) { + this.provider = provider; + } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java b/solr/core/src/java/org/apache/solr/core/SchemaCodecProvider.java similarity index 22% rename from lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java rename to solr/core/src/java/org/apache/solr/core/SchemaCodecProvider.java index ac4e8ef..3d0ffab 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java +++ b/solr/core/src/java/org/apache/solr/core/SchemaCodecProvider.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.lucene3x; +package org.apache.solr.core; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -16,75 +16,140 @@ package org.apache.lucene.index.codecs.lucene3x; * See the License for the specific language governing permissions and * limitations under the License. */ - -import java.io.IOException; +import java.util.Collection; import java.util.Set; -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.DefaultFieldsFormat; -import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; -import org.apache.lucene.index.codecs.DocValuesFormat; -import org.apache.lucene.index.codecs.FieldsFormat; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.index.codecs.PostingsFormat; -import org.apache.lucene.index.codecs.SegmentInfosFormat; -import org.apache.lucene.store.Directory; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.SegmentInfosReader; +import org.apache.lucene.index.codecs.SegmentInfosWriter; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; /** - * Supports the Lucene 3.x index format (readonly) + * Selects a codec based on a {@link IndexSchema}. This {@link CodecProvider} + * also supports dynamic fields such that not all field codecs need to be known + * in advance */ -public class Lucene3xCodec extends Codec { - public Lucene3xCodec() { - super("Lucene3x"); - } - - private final PostingsFormat postingsFormat = new Lucene3xPostingsFormat(); - - // TODO: this should really be a different impl - private final FieldsFormat fieldsFormat = new DefaultFieldsFormat(); - - // TODO: this should really be a different impl - // also if we want preflex to *really* be read-only it should throw exception for the writer? - // this way IR.commit fails on delete/undelete/setNorm/etc ? - private final SegmentInfosFormat infosFormat = new DefaultSegmentInfosFormat(); - - // 3.x doesn't support docvalues - private final DocValuesFormat docValuesFormat = new DocValuesFormat() { - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return null; +final class SchemaCodecProvider extends CodecProvider { + private final IndexSchema schema; + private final CodecProvider delegate; + + SchemaCodecProvider(IndexSchema schema, CodecProvider delegate) { + this.schema = schema; + this.delegate = delegate; + } + + @Override + public Codec lookup(String name) { + synchronized (delegate) { + return delegate.lookup(name); } + } + + @Override + public String getFieldCodec(String name) { + synchronized (delegate) { + if (!delegate.hasFieldCodec(name)) { + final SchemaField fieldOrNull = schema.getFieldOrNull(name); + if (fieldOrNull == null) { + throw new IllegalArgumentException("no such field " + name); + } + String codecName = fieldOrNull.getType().getCodec(); + if (codecName == null) { + codecName = delegate.getDefaultFieldCodec(); + } + delegate.setFieldCodec(name, codecName); + return codecName; + } + return delegate.getFieldCodec(name); + } + } + + @Override + public int hashCode() { + return delegate.hashCode(); + } + + @Override + public void register(Codec codec) { + throw new UnsupportedOperationException(); + } + + @Override + public void unregister(Codec codec) { + throw new UnsupportedOperationException(); + } + + @Override + public Collection getAllExtensions() { + return delegate.getAllExtensions(); + } + + @Override + public SegmentInfosWriter getSegmentInfosWriter() { + return delegate.getSegmentInfosWriter(); + } - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return null; + @Override + public SegmentInfosReader getSegmentInfosReader() { + return delegate.getSegmentInfosReader(); + } + + @Override + public boolean equals(Object obj) { + return delegate.equals(obj); + } + + @Override + public void setFieldCodec(String field, String codec) { + throw new UnsupportedOperationException(); + } + + @Override + public String getDefaultFieldCodec() { + return delegate.getDefaultFieldCodec(); + } + + @Override + public boolean isCodecRegistered(String name) { + synchronized (delegate) { + return delegate.isCodecRegistered(name); } + } - @Override - public void files(Directory dir, SegmentInfo info, Set files) throws IOException {} - }; - @Override - public PostingsFormat postingsFormat() { - return postingsFormat; + public void setDefaultFieldCodec(String codec) { + throw new UnsupportedOperationException(); } - + @Override - public DocValuesFormat docValuesFormat() { - return docValuesFormat; + public boolean hasFieldCodec(String fieldName) { + synchronized (delegate) { + if (!delegate.hasFieldCodec(fieldName)) { + final SchemaField fieldOrNull = schema.getFieldOrNull(fieldName); + if (fieldOrNull == null) { + return false; + } + String codecName = fieldOrNull.getType().getCodec(); + if (codecName == null) { + codecName = delegate.getDefaultFieldCodec(); + } + delegate.setFieldCodec(fieldName, codecName); + } + return true; + } } @Override - public FieldsFormat fieldsFormat() { - return fieldsFormat; + public String toString() { + return "SchemaCodecProvider(" + delegate.toString() + ")"; } @Override - public SegmentInfosFormat segmentInfosFormat() { - return infosFormat; + public Set listAll() { + synchronized (delegate) { + return delegate.listAll(); + } } } diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 4128d13..3a9257d 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -35,6 +35,8 @@ import org.apache.solr.update.processor.UpdateRequestProcessorChain; import org.apache.solr.spelling.QueryConverter; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.index.IndexDeletionPolicy; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.Version; import org.slf4j.Logger; @@ -198,7 +200,7 @@ public class SolrConfig extends Config { loadPluginInfo(DirectoryFactory.class,"directoryFactory",false, true); loadPluginInfo(IndexDeletionPolicy.class,"mainIndex/deletionPolicy",false, true); - loadPluginInfo(CodecFactory.class,"mainIndex/codecFactory",false, false); + loadPluginInfo(CodecProviderFactory.class,"mainIndex/codecProviderFactory",false, false); loadPluginInfo(IndexReaderFactory.class,"indexReaderFactory",false, true); loadPluginInfo(UpdateRequestProcessorChain.class,"updateRequestProcessorChain",false, false); loadPluginInfo(UpdateLog.class,"updateHandler/updateLog",false, false); diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index 26afa4e..fbea86f 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -20,7 +20,7 @@ package org.apache.solr.core; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.store.Directory; import org.apache.solr.common.SolrException; @@ -91,7 +91,7 @@ public final class SolrCore implements SolrInfoMBean { private IndexDeletionPolicyWrapper solrDelPolicy; private DirectoryFactory directoryFactory; private IndexReaderFactory indexReaderFactory; - private final Codec codec; + private final CodecProvider codecProvider; public long getStartTime() { return startTime; } @@ -347,6 +347,7 @@ public final class SolrCore implements SolrInfoMBean { indexReaderFactory = new StandardIndexReaderFactory(); } this.indexReaderFactory = indexReaderFactory; + this.indexReaderFactory.setCodecProvider(codecProvider); } // protect via synchronized(SolrCore.class) @@ -382,7 +383,7 @@ public final class SolrCore implements SolrInfoMBean { log.warn(logid+"Solr index directory '" + new File(indexDir) + "' doesn't exist." + " Creating new index..."); - SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.mainIndexConfig, solrDelPolicy, codec, false); + SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.mainIndexConfig, solrDelPolicy, codecProvider, false); writer.close(); } @@ -559,7 +560,7 @@ public final class SolrCore implements SolrInfoMBean { initDeletionPolicy(); - this.codec= initCodec(solrConfig, schema); + this.codecProvider = initCodecProvider(solrConfig, schema); if (updateHandler == null) { initDirectoryFactory(); @@ -637,16 +638,18 @@ public final class SolrCore implements SolrInfoMBean { resourceLoader.inform(infoRegistry); } - private Codec initCodec(SolrConfig solrConfig, final IndexSchema schema) { - final PluginInfo info = solrConfig.getPluginInfo(CodecFactory.class.getName()); - final CodecFactory factory; + private CodecProvider initCodecProvider(SolrConfig solrConfig, IndexSchema schema) { + final PluginInfo info = solrConfig.getPluginInfo(CodecProviderFactory.class.getName()); + CodecProvider cp; if (info != null) { - factory = (CodecFactory) schema.getResourceLoader().newInstance(info.className); + CodecProviderFactory factory = (CodecProviderFactory) schema.getResourceLoader().newInstance(info.className); factory.init(info.initArgs); + cp = factory.create(); } else { - factory = new DefaultCodecFactory(); + // make sure we use the default if nothing is configured + cp = CodecProvider.getDefault(); } - return factory.create(schema); + return new SchemaCodecProvider(schema, cp); } /** @@ -1850,8 +1853,8 @@ public final class SolrCore implements SolrInfoMBean { return lst; } - public Codec getCodec() { - return codec; + public CodecProvider getCodecProvider() { + return codecProvider; } public final class LazyQueryResponseWriterWrapper implements QueryResponseWriter { diff --git a/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java b/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java index 2695cb2..5767d29 100644 --- a/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java +++ b/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java @@ -35,6 +35,6 @@ public class StandardIndexReaderFactory extends IndexReaderFactory { @Override public IndexReader newReader(Directory indexDir, boolean readOnly) throws IOException { - return IndexReader.open(indexDir, null, readOnly, termInfosIndexDivisor); + return IndexReader.open(indexDir, null, readOnly, termInfosIndexDivisor, provider); } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java index 4fbc58e..d345e19 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java @@ -213,7 +213,6 @@ public class CoreAdminHandler extends RequestHandlerBase { for (int i = 0; i < dirNames.length; i++) { Directory dir = dirFactory.get(dirNames[i], core.getSolrConfig().mainIndexConfig.lockType); dirsToBeReleased[i] = dir; - // TODO: why doesn't this use the IR factory? what is going on here? readersToBeClosed[i] = IndexReader.open(dir, true); } } diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 2bdbeef..05633d1 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -167,10 +167,10 @@ public abstract class FieldType extends FieldProperties { initArgs.remove("positionIncrementGap"); } - final String postingsFormat = initArgs.get("postingsFormat"); - if (postingsFormat != null) { - this.postingsFormat = postingsFormat; - initArgs.remove("postingsFormat"); + final String codec = initArgs.get("codec"); + if (codec != null) { + this.codec = codec; + initArgs.remove("codec"); } if (initArgs.size() > 0) { @@ -527,12 +527,12 @@ public abstract class FieldType extends FieldProperties { } /** - * The postings format used for this field type + * The codec ID used for this field type */ - protected String postingsFormat; + protected String codec; - public String getPostingsFormat() { - return postingsFormat; + public String getCodec() { + return codec; } /** diff --git a/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java index 0afbd06..459feb3 100644 --- a/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java +++ b/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java @@ -29,7 +29,6 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.search.spell.HighFrequencyDictionary; import org.apache.lucene.search.spell.PlainTextDictionary; import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.util.Version; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.schema.FieldType; @@ -63,14 +62,7 @@ public class FileBasedSpellChecker extends AbstractLuceneSpellChecker { try { loadExternalFileDictionary(core); spellChecker.clearIndex(); - // TODO: you should be able to specify the IWC params? - IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null); - // TODO: if we enable this, codec gets angry since field won't exist in the schema - // config.setCodec(core.getCodec()); - ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(300); - // TODO: does Solr really want to continue passing 'optimize=true' to the spellchecker here? - // (its been doing this behind the scenes all along, but its wasteful. - spellChecker.indexDictionary(dictionary, config, true); + spellChecker.indexDictionary(dictionary); } catch (IOException e) { throw new RuntimeException(e); } @@ -102,8 +94,6 @@ public class FileBasedSpellChecker extends AbstractLuceneSpellChecker { setMaxBufferedDocs(150). setMergePolicy(mp). setOpenMode(IndexWriterConfig.OpenMode.CREATE) - // TODO: if we enable this, codec gets angry since field won't exist in the schema - // .setCodec(core.getCodec()) ); List lines = core.getResourceLoader().getLines(sourceLocation, characterEncoding); @@ -116,7 +106,7 @@ public class FileBasedSpellChecker extends AbstractLuceneSpellChecker { writer.optimize(); writer.close(); - dictionary = new HighFrequencyDictionary(IndexReader.open(ramDir, true), + dictionary = new HighFrequencyDictionary(IndexReader.open(ramDir), WORD_FIELD_NAME, 0.0f); } else { // check if character encoding is defined diff --git a/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java index 63eaba5..1963006 100644 --- a/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java +++ b/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java @@ -17,11 +17,8 @@ package org.apache.solr.spelling; */ import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.search.spell.HighFrequencyDictionary; -import org.apache.lucene.util.Version; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; @@ -66,7 +63,7 @@ public class IndexBasedSpellChecker extends AbstractLuceneSpellChecker { if (sourceLocation != null) { try { FSDirectory luceneIndexDir = FSDirectory.open(new File(sourceLocation)); - this.reader = IndexReader.open(luceneIndexDir, true); + this.reader = IndexReader.open(luceneIndexDir); } catch (IOException e) { throw new RuntimeException(e); } @@ -88,19 +85,8 @@ public class IndexBasedSpellChecker extends AbstractLuceneSpellChecker { // Create the dictionary dictionary = new HighFrequencyDictionary(reader, field, threshold); - // TODO: maybe whether or not to clear the index should be configurable? - // an incremental update is faster (just adds new terms), but if you 'expunged' - // old terms I think they might hang around. spellChecker.clearIndex(); - // TODO: you should be able to specify the IWC params? - IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null); - - // TODO: if we enable this, codec gets angry since field won't exist in the schema - // config.setCodec(core.getCodec()); - ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(300); - // TODO: does Solr really want to continue passing 'optimize=true' to the spellchecker here? - // (its been doing this behind the scenes all along, but its wasteful. - spellChecker.indexDictionary(dictionary, config, true); + spellChecker.indexDictionary(dictionary); } catch (IOException e) { throw new RuntimeException(e); diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java index 40f9460..bb6b699 100644 --- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java @@ -78,7 +78,7 @@ public final class DefaultSolrCoreState extends SolrCoreState { boolean removeAllExisting, boolean forceNewDirectory) throws IOException { return new SolrIndexWriter(name, core.getNewIndexDir(), core.getDirectoryFactory(), removeAllExisting, core.getSchema(), - core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy(), core.getCodec(), forceNewDirectory); + core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy(), core.getCodecProvider(), forceNewDirectory); } @Override diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java index f56fefe..b84e232 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java @@ -29,7 +29,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.schema.IndexSchema; @@ -52,12 +52,12 @@ public class SolrIndexWriter extends IndexWriter { private PrintStream infoStream; private DirectoryFactory directoryFactory; - public SolrIndexWriter(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec, boolean forceNewDirectory) throws IOException { + public SolrIndexWriter(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, CodecProvider codecProvider, boolean forceNewDirectory) throws IOException { super( directoryFactory.get(path, config.lockType, forceNewDirectory), config.toIndexWriterConfig(schema). setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND). - setIndexDeletionPolicy(delPolicy).setCodec(codec) + setIndexDeletionPolicy(delPolicy).setCodecProvider(codecProvider) ); log.debug("Opened Writer " + name); this.name = name; diff --git a/solr/core/src/test-files/solr/conf/schema_codec.xml b/solr/core/src/test-files/solr/conf/schema_codec.xml index e08ab8d..2a69f11 100644 --- a/solr/core/src/test-files/solr/conf/schema_codec.xml +++ b/solr/core/src/test-files/solr/conf/schema_codec.xml @@ -17,9 +17,9 @@ --> - - - + + + diff --git a/solr/core/src/test-files/solr/conf/solrconfig_codec.xml b/solr/core/src/test-files/solr/conf/solrconfig_codec.xml index 1ff30e0..0a1a7e1 100644 --- a/solr/core/src/test-files/solr/conf/solrconfig_codec.xml +++ b/solr/core/src/test-files/solr/conf/solrconfig_codec.xml @@ -19,5 +19,15 @@ ${tests.luceneMatchVersion:LUCENE_CURRENT} + + + Pulsing + + org.apache.lucene.index.codecs.simpletext.SimpleTextCodec + org.apache.lucene.index.codecs.preflex.PreFlexCodec + + + + diff --git a/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java b/solr/core/src/test/org/apache/solr/core/MockCodecProviderFactory.java similarity index 39% rename from solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java rename to solr/core/src/test/org/apache/solr/core/MockCodecProviderFactory.java index 602bfca..ace2e09 100644 --- a/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java +++ b/solr/core/src/test/org/apache/solr/core/MockCodecProviderFactory.java @@ -18,42 +18,46 @@ package org.apache.solr.core; */ import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.PostingsFormat; -import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.SchemaField; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.pulsing.PulsingCodec; +import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.solr.common.util.NamedList; /** - * Default CodecFactory implementation, extends Lucene's - * and returns postings format implementations according to the - * schema configuration. - * @lucene.experimental + * CodecProviderFactory for testing, it inits a CP with Standard and Pulsing, + * and also adds any codecs specified by classname in solrconfig. */ -public class DefaultCodecFactory extends CodecFactory { +public class MockCodecProviderFactory extends CodecProviderFactory { + private String defaultCodec; + private NamedList codecs; - // TODO: we need to change how solr does this? - // rather than a string like "Pulsing" you need to be able to pass parameters - // and everything to a field in the schema, e.g. we should provide factories for - // the Lucene's core formats (Memory, Pulsing, ...) and such. - // - // So I think a FieldType should return PostingsFormat, not a String. - // how it constructs this from the XML... i don't care. + @Override + public void init(NamedList args) { + super.init(args); + defaultCodec = (String) args.get("defaultCodec"); + codecs = (NamedList) args.get("codecs"); + } @Override - public Codec create(final IndexSchema schema) { - return new Lucene40Codec() { - @Override - public PostingsFormat getPostingsFormatForField(String field) { - final SchemaField fieldOrNull = schema.getFieldOrNull(field); - if (fieldOrNull == null) { - throw new IllegalArgumentException("no such field " + field); - } - String postingsFormatName = fieldOrNull.getType().getPostingsFormat(); - if (postingsFormatName != null) { - return PostingsFormat.forName(postingsFormatName); + public CodecProvider create() { + CodecProvider cp = new CodecProvider(); + cp.register(new StandardCodec()); + cp.register(new PulsingCodec()); + if (codecs != null) { + for (Object codec : codecs.getAll("name")) { + if (!cp.isCodecRegistered((String)codec)) { + try { + Class clazz = Class.forName((String)codec).asSubclass(Codec.class); + cp.register(clazz.newInstance()); + } catch (Exception e) { + throw new RuntimeException(e); + } } - return super.getPostingsFormatForField(field); } - }; + } + if (defaultCodec != null) { + cp.setDefaultFieldCodec(defaultCodec); + } + return cp; } } diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecProviderSupport.java similarity index 41% rename from solr/core/src/test/org/apache/solr/core/TestCodecSupport.java rename to solr/core/src/test/org/apache/solr/core/TestCodecProviderSupport.java index 459d765..10897a9 100644 --- a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java +++ b/solr/core/src/test/org/apache/solr/core/TestCodecProviderSupport.java @@ -19,51 +19,84 @@ package org.apache.solr.core; import java.util.Map; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.schema.SchemaField; import org.junit.BeforeClass; -public class TestCodecSupport extends SolrTestCaseJ4 { +public class TestCodecProviderSupport extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - initCore("solrconfig-basic.xml", "schema_codec.xml"); + initCore("solrconfig_codec.xml", "schema_codec.xml"); } - public void testPostingsFormats() { - Codec codec = h.getCore().getCodec(); + public void testCodecs() { + CodecProvider codecProvider = h.getCore().getCodecProvider(); Map fields = h.getCore().getSchema().getFields(); SchemaField schemaField = fields.get("string_pulsing_f"); - PerFieldPostingsFormat format = (PerFieldPostingsFormat) codec.postingsFormat(); - assertEquals("Pulsing40", format.getPostingsFormatForField(schemaField.getName()).getName()); + assertEquals("Pulsing", codecProvider.getFieldCodec(schemaField.getName())); schemaField = fields.get("string_simpletext_f"); assertEquals("SimpleText", - format.getPostingsFormatForField(schemaField.getName()).getName()); + codecProvider.getFieldCodec(schemaField.getName())); schemaField = fields.get("string_standard_f"); - assertEquals("Lucene40", format.getPostingsFormatForField(schemaField.getName()).getName()); + assertEquals("Standard", codecProvider.getFieldCodec(schemaField.getName())); schemaField = fields.get("string_f"); - assertEquals("Lucene40", format.getPostingsFormatForField(schemaField.getName()).getName()); + assertEquals("Pulsing", codecProvider.getFieldCodec(schemaField.getName())); + + assertTrue(codecProvider.hasFieldCodec("string_simpletext_f")); + assertTrue(codecProvider.hasFieldCodec("string_standard_f")); + assertTrue(codecProvider.hasFieldCodec("string_f")); } public void testDynamicFields() { - Codec codec = h.getCore().getCodec(); - PerFieldPostingsFormat format = (PerFieldPostingsFormat) codec.postingsFormat(); - - assertEquals("SimpleText", format.getPostingsFormatForField("foo_simple").getName()); - assertEquals("SimpleText", format.getPostingsFormatForField("bar_simple").getName()); - assertEquals("Pulsing40", format.getPostingsFormatForField("foo_pulsing").getName()); - assertEquals("Pulsing40", format.getPostingsFormatForField("bar_pulsing").getName()); - assertEquals("Lucene40", format.getPostingsFormatForField("foo_standard").getName()); - assertEquals("Lucene40", format.getPostingsFormatForField("bar_standard").getName()); + CodecProvider codecProvider = h.getCore().getCodecProvider(); + + assertTrue(codecProvider.hasFieldCodec("bar_simple")); + assertTrue(codecProvider.hasFieldCodec("bar_pulsing")); + assertTrue(codecProvider.hasFieldCodec("bar_standard")); + + assertEquals("SimpleText", codecProvider.getFieldCodec("foo_simple")); + assertEquals("Pulsing", codecProvider.getFieldCodec("foo_pulsing")); + assertEquals("Standard", codecProvider.getFieldCodec("foo_standard")); + } + + public void testUnmodifiable() { + CodecProvider codecProvider = h.getCore().getCodecProvider(); + try { + codecProvider.setDefaultFieldCodec("foo"); + fail("should be unmodifiable"); + } catch (UnsupportedOperationException e) { + // + } + + try { + codecProvider.setFieldCodec("foo", "bar"); + fail("should be unmodifiable"); + } catch (UnsupportedOperationException e) { + // + } + + try { + codecProvider.register(new StandardCodec()); + fail("should be unmodifiable"); + } catch (UnsupportedOperationException e) { + // + } + + try { + codecProvider.unregister(new StandardCodec()); + fail("should be unmodifiable"); + } catch (UnsupportedOperationException e) { + // + } } public void testUnknownField() { - Codec codec = h.getCore().getCodec(); - PerFieldPostingsFormat format = (PerFieldPostingsFormat) codec.postingsFormat(); + CodecProvider codecProvider = h.getCore().getCodecProvider(); try { - format.getPostingsFormatForField("notexisting"); + codecProvider.getFieldCodec("notexisting"); fail("field is not existing"); } catch (IllegalArgumentException e) { // diff --git a/solr/core/src/test/org/apache/solr/schema/TestCollationField.java b/solr/core/src/test/org/apache/solr/schema/TestCollationField.java index 175b0f8..e92c14e 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestCollationField.java +++ b/solr/core/src/test/org/apache/solr/schema/TestCollationField.java @@ -25,7 +25,7 @@ import java.util.Locale; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; @@ -36,8 +36,7 @@ public class TestCollationField extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - assumeFalse("preflex format only supports UTF-8 encoded bytes", - "Lucene3x".equals(Codec.getDefault().getName())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); String home = setupSolrHome(); initCore("solrconfig.xml","schema.xml", home); // add some docs diff --git a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java index fff65ec..b5065d4 100755 --- a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java +++ b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java @@ -18,7 +18,7 @@ package org.apache.solr.search.function; import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.TFIDFSimilarity; @@ -413,7 +413,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 { */ public void testTotalTermFreq() throws Exception { assumeFalse("PreFlex codec does not support collection-level term stats", - "Lucene3x".equals(Codec.getDefault().getName())); + "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); clearIndex();