diff --git a/dev-tools/eclipse/dot.classpath b/dev-tools/eclipse/dot.classpath index 5876841..ee6a742 100644 --- a/dev-tools/eclipse/dot.classpath +++ b/dev-tools/eclipse/dot.classpath @@ -1,7 +1,9 @@ + + - + @@ -13,6 +15,7 @@ + diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 247a47c..bc21a6d 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -215,7 +215,7 @@ Changes in Runtime Behavior * LUCENE-2881: FieldInfos is now tracked per segment. Before it was tracked per IndexWriter session, which resulted in FieldInfos that had the FieldInfo properties from all previous segments combined. Field numbers are now tracked - globally across IndexWriter sessions and persisted into a X.fnx file on + globally across IndexWriter sessions and persisted into a _X.fnx file on successful commit. The corresponding file format changes are backwards- compatible. (Michael Busch, Simon Willnauer) @@ -375,9 +375,8 @@ New features * LUCENE-1458, LUCENE-2111: With flexible indexing it is now possible for an application to create its own postings codec, to alter how fields, terms, docs and positions are encoded into the index. The - standard codec is the default codec. Both IndexWriter and - IndexReader accept a CodecProvider class to obtain codecs for newly - written segments as well as existing segments opened for reading. + standard codec is the default codec. IndexWriter accepts a Codec + class to obtain codecs for newly written segments. * LUCENE-1458, LUCENE-2111: Some experimental codecs have been added for flexible indexing, including pulsing codec (inlines @@ -403,7 +402,7 @@ New features * LUCENE-2489: Added PerFieldCodecWrapper (in oal.index.codecs) which lets you set the Codec per field (Mike McCandless) -* LUCENE-2373: Extend CodecProvider to use SegmentInfosWriter and +* LUCENE-2373: Extend Codec to use SegmentInfosWriter and SegmentInfosReader to allow customization of SegmentInfos data. (Andrzej Bialecki) @@ -445,10 +444,10 @@ New features (i.e. \* or "*") Custom QueryParser subclasses overriding getRangeQuery() will be passed null for any open endpoint. (Adriano Crestani, yonik) -* LUCENE-2742: Add native per-field codec support. CodecProvider lets you now - register a codec for each field and which is in turn recorded in the segment - and field information. Codecs are maintained on a per-segment basis and be - resolved without knowing the actual codec used for writing the segment. +* LUCENE-2742: Add native per-field postings format support. Codec lets you now + register a postings format for each field and which is in turn recorded + into the index. Postings formtas are maintained on a per-segment basis and be + resolved without knowing the actual postings format used for writing the segment. (Simon Willnauer) * LUCENE-2741: Add support for multiple codecs that use the same file diff --git a/lucene/build.xml b/lucene/build.xml index e6c904a..32a3547 100644 --- a/lucene/build.xml +++ b/lucene/build.xml @@ -30,15 +30,15 @@ - - + + - + @@ -556,11 +556,11 @@ - + diff --git a/lucene/common-build.xml b/lucene/common-build.xml index b897056..f86dd69 100644 --- a/lucene/common-build.xml +++ b/lucene/common-build.xml @@ -83,8 +83,7 @@ - - + @@ -473,8 +472,13 @@ - + + + + + @@ -551,9 +555,7 @@ - - - + diff --git a/lucene/contrib/contrib-build.xml b/lucene/contrib/contrib-build.xml index 30d0bc3..e2eb4bd 100644 --- a/lucene/contrib/contrib-build.xml +++ b/lucene/contrib/contrib-build.xml @@ -39,8 +39,8 @@ - + diff --git a/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java b/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java index 060778c..57a96e1 100644 --- a/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java +++ b/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java @@ -34,6 +34,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TopDocs; @@ -107,7 +108,7 @@ public class MemoryIndexTest extends BaseTokenStreamTestCase { Directory ramdir = newDirectory(); Analyzer analyzer = randomAnalyzer(); IndexWriter writer = new IndexWriter(ramdir, - new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); Document doc = new Document(); Field field1 = newField("foo", fooField.toString(), TextField.TYPE_UNSTORED); Field field2 = newField("term", termField.toString(), TextField.TYPE_UNSTORED); diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java index e1c9cde..4c10dfc 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java @@ -27,7 +27,6 @@ import java.util.ArrayList; import java.util.List; import org.apache.lucene.index.IndexWriter; // Required for javadocs -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.FSDirectory; /** @@ -54,8 +53,6 @@ import org.apache.lucene.store.FSDirectory; */ public class IndexSplitter { public SegmentInfos infos; - - private final CodecProvider codecs; FSDirectory fsDir; @@ -96,17 +93,12 @@ public class IndexSplitter { is.split(targetDir, segs.toArray(new String[0])); } } - - public IndexSplitter(File dir) throws IOException { - this(dir, CodecProvider.getDefault()); - } - public IndexSplitter(File dir, CodecProvider codecs) throws IOException { + public IndexSplitter(File dir) throws IOException { this.dir = dir; - this.codecs = codecs; fsDir = FSDirectory.open(dir); - infos = new SegmentInfos(codecs); - infos.read(fsDir, codecs); + infos = new SegmentInfos(); + infos.read(fsDir); } public void listSegments() throws IOException { @@ -140,13 +132,13 @@ public class IndexSplitter { infos.remove(idx); } infos.changed(); - infos.commit(fsDir); + infos.commit(fsDir, infos.codecFormat()); } public void split(File destDir, String[] segs) throws IOException { destDir.mkdirs(); FSDirectory destFSDir = FSDirectory.open(destDir); - SegmentInfos destInfos = new SegmentInfos(codecs); + SegmentInfos destInfos = new SegmentInfos(); destInfos.counter = infos.counter; for (String n : segs) { SegmentInfo info = getInfo(n); @@ -160,7 +152,7 @@ public class IndexSplitter { } } destInfos.changed(); - destInfos.commit(destFSDir); + destInfos.commit(destFSDir, infos.codecFormat()); // System.out.println("destDir:"+destDir.getAbsolutePath()); } diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java dissimilarity index 74% index 3272cf6..57ea9dc 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java @@ -?,? +1,69 @@ +package org.apache.lucene.index.codecs.appending; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesFormat; +import org.apache.lucene.index.codecs.DefaultFieldsFormat; +import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldsFormat; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.SegmentInfosFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; + +/** + * This codec extends {@link Lucene40Codec} to work on append-only outputs, such + * as plain output streams and append-only filesystems. + * + *

Note: compound file format feature is not compatible with + * this codec. You must call both + * LogMergePolicy.setUseCompoundFile(false) and + * LogMergePolicy.setUseCompoundDocStore(false) to disable + * compound file format.

+ * @lucene.experimental + */ +public class AppendingCodec extends Codec { + public AppendingCodec() { + super("Appending"); + } + + private final PostingsFormat postings = new AppendingPostingsFormat(); + private final SegmentInfosFormat infos = new AppendingSegmentInfosFormat(); + private final FieldsFormat fields = new DefaultFieldsFormat(); + private final DocValuesFormat docValues = new DefaultDocValuesFormat(); + + @Override + public PostingsFormat postingsFormat() { + return postings; + } + + @Override + public FieldsFormat fieldsFormat() { + return fields; + } + + @Override + public DocValuesFormat docValuesFormat() { + return docValues; + } + + @Override + public SegmentInfosFormat segmentInfosFormat() { + return infos; + } +} diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java new file mode 100644 index 0000000..e2dd314 --- /dev/null +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java @@ -0,0 +1,128 @@ +package org.apache.lucene.index.codecs.appending; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.BlockTermsReader; +import org.apache.lucene.index.codecs.TermsIndexReaderBase; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; + +/** + * Appending postings impl + */ +class AppendingPostingsFormat extends PostingsFormat { + public static String CODEC_NAME = "Appending"; + + public AppendingPostingsFormat() { + super(CODEC_NAME); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) + throws IOException { + PostingsWriterBase docsWriter = new Lucene40PostingsWriter(state); + boolean success = false; + AppendingTermsIndexWriter indexWriter = null; + try { + indexWriter = new AppendingTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + docsWriter.close(); + } + } + success = false; + try { + FieldsConsumer ret = new AppendingTermsDictWriter(indexWriter, state, docsWriter); + success = true; + return ret; + } finally { + if (!success) { + try { + docsWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) + throws IOException { + PostingsReaderBase docsReader = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + TermsIndexReaderBase indexReader; + + boolean success = false; + try { + indexReader = new AppendingTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.segmentSuffix, state.context); + success = true; + } finally { + if (!success) { + docsReader.close(); + } + } + success = false; + try { + FieldsProducer ret = new AppendingTermsDictReader(indexReader, + state.dir, state.fieldInfos, state.segmentInfo.name, + docsReader, + state.context, + Lucene40PostingsFormat.TERMS_CACHE_SIZE, + state.segmentSuffix); + success = true; + return ret; + } finally { + if (!success) { + try { + docsReader.close(); + } finally { + indexReader.close(); + } + } + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) + throws IOException { + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java new file mode 100644 index 0000000..03544ea --- /dev/null +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java @@ -0,0 +1,30 @@ +package org.apache.lucene.index.codecs.appending; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; +import org.apache.lucene.index.codecs.SegmentInfosWriter; + +public class AppendingSegmentInfosFormat extends DefaultSegmentInfosFormat { + private final SegmentInfosWriter writer = new AppendingSegmentInfosWriter(); + + @Override + public SegmentInfosWriter getSegmentInfosWriter() { + return writer; + } +} diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java index 1ecb3e2..dc49fad 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java @@ -20,8 +20,6 @@ package org.apache.lucene.index.codecs.appending; import java.io.IOException; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; public class AppendingSegmentInfosWriter extends DefaultSegmentInfosWriter { diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java index 7f885ee..742de0b 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java @@ -34,9 +34,9 @@ public class AppendingTermsDictReader extends BlockTermsReader { public AppendingTermsDictReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext context, - int termsCacheSize, int codecId) throws IOException { + int termsCacheSize, String segmentSuffix) throws IOException { super(indexReader, dir, fieldInfos, segment, postingsReader, context, - termsCacheSize, codecId); + termsCacheSize, segmentSuffix); } @Override diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java index 205dc15..ed393ad 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java @@ -31,9 +31,9 @@ import org.apache.lucene.util.CodecUtil; public class AppendingTermsIndexReader extends FixedGapTermsIndexReader { public AppendingTermsIndexReader(Directory dir, FieldInfos fieldInfos, - String segment, int indexDivisor, Comparator termComp, int codecId, IOContext context) + String segment, int indexDivisor, Comparator termComp, String segmentSuffix, IOContext context) throws IOException { - super(dir, fieldInfos, segment, indexDivisor, termComp, codecId, context); + super(dir, fieldInfos, segment, indexDivisor, termComp, segmentSuffix, context); } @Override diff --git a/lucene/contrib/misc/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec b/lucene/contrib/misc/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec new file mode 100644 index 0000000..c29837d --- /dev/null +++ b/lucene/contrib/misc/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.lucene.index.codecs.appending.AppendingCodec diff --git a/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java b/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java index cb79651..c35d823 100644 --- a/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java +++ b/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java @@ -27,7 +27,9 @@ import org.apache.lucene.search.SearcherWarmer; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NRTCachingDirectory; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; +@UseNoMemoryExpensiveCodec public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase { private final ThreadLocal lastGens = new ThreadLocal(); diff --git a/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java b/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java index 60463f7..9398551 100644 --- a/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java +++ b/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java @@ -34,11 +34,6 @@ import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.DefaultSegmentInfosReader; -import org.apache.lucene.index.codecs.SegmentInfosReader; -import org.apache.lucene.index.codecs.SegmentInfosWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -50,30 +45,7 @@ import org.apache.lucene.util.Version; public class TestAppendingCodec extends LuceneTestCase { - static class AppendingCodecProvider extends CodecProvider { - Codec appending = new AppendingCodec(); - SegmentInfosWriter infosWriter = new AppendingSegmentInfosWriter(); - SegmentInfosReader infosReader = new DefaultSegmentInfosReader(); - public AppendingCodecProvider() { - setDefaultFieldCodec(appending.name); - } - @Override - public Codec lookup(String name) { - return appending; - } - - @Override - public SegmentInfosReader getSegmentInfosReader() { - return infosReader; - } - @Override - public SegmentInfosWriter getSegmentInfosWriter() { - return infosWriter; - } - - } - - private static class AppendingIndexOutputWrapper extends IndexOutput { + private static class AppendingIndexOutputWrapper extends IndexOutput { IndexOutput wrapped; public AppendingIndexOutputWrapper(IndexOutput wrapped) { @@ -137,7 +109,7 @@ public class TestAppendingCodec extends LuceneTestCase { Directory dir = new AppendingRAMDirectory(random, new RAMDirectory()); IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(random)); - cfg.setCodecProvider(new AppendingCodecProvider()); + cfg.setCodec(new AppendingCodec()); ((TieredMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false); IndexWriter writer = new IndexWriter(dir, cfg); Document doc = new Document(); @@ -151,7 +123,7 @@ public class TestAppendingCodec extends LuceneTestCase { writer.addDocument(doc); writer.optimize(); writer.close(); - IndexReader reader = IndexReader.open(dir, null, true, 1, new AppendingCodecProvider()); + IndexReader reader = IndexReader.open(dir, null, true, 1); assertEquals(2, reader.numDocs()); Document doc2 = reader.document(0); assertEquals(text, doc2.get("f")); diff --git a/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java b/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java index c06d8f1..f86a20d 100644 --- a/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java +++ b/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java @@ -34,9 +34,11 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; import org.apache.lucene.util.NamedThreadFactory; import org.apache.lucene.util._TestUtil; +@UseNoMemoryExpensiveCodec public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase { boolean warmCalled; diff --git a/lucene/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/src/java/org/apache/lucene/index/CheckIndex.java index 9712f5c..6dd0909 100644 --- a/lucene/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/src/java/org/apache/lucene/index/CheckIndex.java @@ -25,7 +25,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.document.Document; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import java.io.File; import java.io.IOException; @@ -143,8 +143,8 @@ public class CheckIndex { /** Name of the segment. */ public String name; - /** CodecInfo used to read this segment. */ - public SegmentCodecs codec; + /** Codec used to read this segment. */ + public Codec codec; /** Document count (does not take deletions into account). */ public int docCount; @@ -322,10 +322,6 @@ public class CheckIndex { public Status checkIndex() throws IOException { return checkIndex(null); } - - public Status checkIndex(List onlySegments) throws IOException { - return checkIndex(onlySegments, CodecProvider.getDefault()); - } /** Returns a {@link Status} instance detailing * the state of the index. @@ -339,13 +335,13 @@ public class CheckIndex { *

WARNING: make sure * you only call this when the index is not opened by any * writer. */ - public Status checkIndex(List onlySegments, CodecProvider codecs) throws IOException { + public Status checkIndex(List onlySegments) throws IOException { NumberFormat nf = NumberFormat.getInstance(); - SegmentInfos sis = new SegmentInfos(codecs); + SegmentInfos sis = new SegmentInfos(); Status result = new Status(); result.dir = dir; try { - sis.read(dir, codecs); + sis.read(dir); } catch (Throwable t) { msg("ERROR: could not read any segments file in directory"); result.missingSegments = true; @@ -377,6 +373,7 @@ public class CheckIndex { final int numSegments = sis.size(); final String segmentsFileName = sis.getCurrentSegmentFileName(); + // note: we only read the format byte (required preamble) here! IndexInput input = null; try { input = dir.openInput(segmentsFileName, IOContext.DEFAULT); @@ -489,7 +486,7 @@ public class CheckIndex { SegmentReader reader = null; try { - final SegmentCodecs codec = info.getSegmentCodecs(); + final Codec codec = info.getCodec(); msg(" codec=" + codec); segInfoStat.codec = codec; msg(" compound=" + info.getUseCompoundFile()); @@ -1182,11 +1179,11 @@ public class CheckIndex { * *

WARNING: Make sure you only call this when the * index is not opened by any writer. */ - public void fixIndex(Status result) throws IOException { + public void fixIndex(Status result, Codec codec) throws IOException { if (result.partial) throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)"); result.newSegments.changed(); - result.newSegments.commit(result.dir); + result.newSegments.commit(result.dir, codec); } private static boolean assertsOn; @@ -1236,6 +1233,7 @@ public class CheckIndex { public static void main(String[] args) throws IOException, InterruptedException { boolean doFix = false; + Codec codec = Codec.getDefault(); // only used when fixing boolean verbose = false; List onlySegments = new ArrayList(); String indexPath = null; @@ -1244,6 +1242,13 @@ public class CheckIndex { if (args[i].equals("-fix")) { doFix = true; i++; + } else if (args[i].equals("-codec")) { + if (i == args.length-1) { + System.out.println("ERROR: missing name for -codec option"); + System.exit(1); + } + codec = Codec.forName(args[i+1]); + i+=2; } else if (args[i].equals("-verbose")) { verbose = true; i++; @@ -1269,6 +1274,7 @@ public class CheckIndex { System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + + " -codec X: when fixing, codec to write the new segments_N file with\n" + " -verbose: print additional details\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + @@ -1329,7 +1335,7 @@ public class CheckIndex { System.out.println(" " + (5-s) + "..."); } System.out.println("Writing..."); - checker.fixIndex(result); + checker.fixIndex(result, codec); System.out.println("OK"); System.out.println("Wrote new segments file \"" + result.newSegments.getCurrentSegmentFileName() + "\""); } diff --git a/lucene/src/java/org/apache/lucene/index/DirectoryReader.java b/lucene/src/java/org/apache/lucene/index/DirectoryReader.java index 190ebd8..f15f077 100644 --- a/lucene/src/java/org/apache/lucene/index/DirectoryReader.java +++ b/lucene/src/java/org/apache/lucene/index/DirectoryReader.java @@ -33,7 +33,6 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -45,8 +44,6 @@ import org.apache.lucene.util.MapBackedSet; class DirectoryReader extends IndexReader implements Cloneable { protected Directory directory; protected boolean readOnly; - - protected CodecProvider codecs; IndexWriter writer; @@ -78,15 +75,13 @@ class DirectoryReader extends IndexReader implements Cloneable { // } static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, - final int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException { - final CodecProvider codecProvider = codecs == null ? CodecProvider.getDefault() - : codecs; + final int termInfosIndexDivisor) throws CorruptIndexException, IOException { return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - SegmentInfos infos = new SegmentInfos(codecProvider); - infos.read(directory, segmentFileName, codecProvider); - return new DirectoryReader(directory, infos, deletionPolicy, readOnly, termInfosIndexDivisor, codecProvider); + SegmentInfos infos = new SegmentInfos(); + infos.read(directory, segmentFileName); + return new DirectoryReader(directory, infos, deletionPolicy, readOnly, termInfosIndexDivisor); } }.run(commit); } @@ -97,17 +92,12 @@ class DirectoryReader extends IndexReader implements Cloneable { // } /** Construct reading the named set of readers. */ - DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws IOException { + DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException { this.directory = directory; this.readOnly = readOnly; this.segmentInfos = sis; this.deletionPolicy = deletionPolicy; this.termInfosIndexDivisor = termInfosIndexDivisor; - if (codecs == null) { - this.codecs = CodecProvider.getDefault(); - } else { - this.codecs = codecs; - } readerFinishedListeners = new MapBackedSet(new ConcurrentHashMap()); applyAllDeletes = false; @@ -141,17 +131,12 @@ class DirectoryReader extends IndexReader implements Cloneable { } // Used by near real-time search - DirectoryReader(IndexWriter writer, SegmentInfos infos, CodecProvider codecs, boolean applyAllDeletes) throws IOException { + DirectoryReader(IndexWriter writer, SegmentInfos infos, boolean applyAllDeletes) throws IOException { this.directory = writer.getDirectory(); this.readOnly = true; this.applyAllDeletes = applyAllDeletes; // saved for reopen this.termInfosIndexDivisor = writer.getConfig().getReaderTermsIndexDivisor(); - if (codecs == null) { - this.codecs = CodecProvider.getDefault(); - } else { - this.codecs = codecs; - } readerFinishedListeners = writer.getReaderFinishedListeners(); // IndexWriter synchronizes externally before calling @@ -200,8 +185,7 @@ class DirectoryReader extends IndexReader implements Cloneable { /** This constructor is only used for {@link #doOpenIfChanged()} */ DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, - boolean readOnly, boolean doClone, int termInfosIndexDivisor, CodecProvider codecs, - Collection readerFinishedListeners) throws IOException { + boolean readOnly, boolean doClone, int termInfosIndexDivisor, Collection readerFinishedListeners) throws IOException { this.directory = directory; this.readOnly = readOnly; this.segmentInfos = infos; @@ -209,13 +193,6 @@ class DirectoryReader extends IndexReader implements Cloneable { this.readerFinishedListeners = readerFinishedListeners; applyAllDeletes = false; - if (codecs == null) { - this.codecs = CodecProvider.getDefault(); - } else { - this.codecs = codecs; - } - - // we put the old SegmentReaders in a map, that allows us // to lookup a reader using its segment name Map segmentReaders = new HashMap(); @@ -347,7 +324,7 @@ class DirectoryReader extends IndexReader implements Cloneable { starts[subReaders.length] = maxDoc; if (!readOnly) { - maxIndexVersion = SegmentInfos.readCurrentVersion(directory, codecs); + maxIndexVersion = SegmentInfos.readCurrentVersion(directory); } } @@ -498,15 +475,15 @@ class DirectoryReader extends IndexReader implements Cloneable { return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - final SegmentInfos infos = new SegmentInfos(codecs); - infos.read(directory, segmentFileName, codecs); + final SegmentInfos infos = new SegmentInfos(); + infos.read(directory, segmentFileName); return doOpenIfChanged(infos, false, openReadOnly); } }.run(commit); } private synchronized DirectoryReader doOpenIfChanged(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException { - return new DirectoryReader(directory, infos, subReaders, openReadOnly, doClone, termInfosIndexDivisor, codecs, readerFinishedListeners); + return new DirectoryReader(directory, infos, subReaders, openReadOnly, doClone, termInfosIndexDivisor, readerFinishedListeners); } /** Version number when this IndexReader was opened. */ @@ -712,7 +689,7 @@ class DirectoryReader extends IndexReader implements Cloneable { // we have to check whether index has changed since this reader was opened. // if so, this reader is no longer valid for deletion - if (SegmentInfos.readCurrentVersion(directory, codecs) > maxIndexVersion) { + if (SegmentInfos.readCurrentVersion(directory) > maxIndexVersion) { stale = true; this.writeLock.release(); this.writeLock = null; @@ -743,7 +720,7 @@ class DirectoryReader extends IndexReader implements Cloneable { // KeepOnlyLastCommitDeleter: IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, - segmentInfos, null, codecs, null); + segmentInfos, null, null); segmentInfos.updateGeneration(deleter.getLastSegmentInfos()); segmentInfos.changed(); @@ -764,7 +741,7 @@ class DirectoryReader extends IndexReader implements Cloneable { // Sync all files we just wrote directory.sync(segmentInfos.files(directory, false)); - segmentInfos.commit(directory); + segmentInfos.commit(directory, segmentInfos.codecFormat()); success = true; } finally { @@ -842,7 +819,7 @@ class DirectoryReader extends IndexReader implements Cloneable { ensureOpen(); if (writer == null || writer.isClosed()) { // we loaded SegmentInfos from the directory - return SegmentInfos.readCurrentVersion(directory, codecs) == segmentInfos.getVersion(); + return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion(); } else { return writer.nrtIsCurrent(segmentInfos); } @@ -925,17 +902,12 @@ class DirectoryReader extends IndexReader implements Cloneable { /** @see org.apache.lucene.index.IndexReader#listCommits */ public static List listCommits(Directory dir) throws IOException { - return listCommits(dir, CodecProvider.getDefault()); - } - - /** @see org.apache.lucene.index.IndexReader#listCommits */ - public static List listCommits(Directory dir, CodecProvider codecs) throws IOException { final String[] files = dir.listAll(); List commits = new ArrayList(); - SegmentInfos latest = new SegmentInfos(codecs); - latest.read(dir, codecs); + SegmentInfos latest = new SegmentInfos(); + latest.read(dir); final long currentGen = latest.getGeneration(); commits.add(new ReaderCommit(latest, dir)); @@ -948,11 +920,11 @@ class DirectoryReader extends IndexReader implements Cloneable { !fileName.equals(IndexFileNames.SEGMENTS_GEN) && SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) { - SegmentInfos sis = new SegmentInfos(codecs); + SegmentInfos sis = new SegmentInfos(); try { // IOException allowed to throw there, in case // segments_N is corrupt - sis.read(dir, fileName, codecs); + sis.read(dir, fileName); } catch (FileNotFoundException fnfe) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth diff --git a/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java b/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java index 02a4a73..f4512af 100644 --- a/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java +++ b/lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java @@ -25,7 +25,7 @@ import java.util.HashSet; import java.util.Map; import org.apache.lucene.index.DocumentsWriterPerThread.DocState; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.index.codecs.DocValuesConsumer; import org.apache.lucene.index.codecs.PerDocConsumer; import org.apache.lucene.index.values.PerDocFieldValues; @@ -320,14 +320,13 @@ final class DocFieldProcessor extends DocConsumer { docValuesConsumerAndDocID.docID = docState.docID; return docValuesConsumerAndDocID.docValuesConsumer; } - PerDocConsumer perDocConsumer = perDocConsumers.get(fieldInfo.getCodecId()); + + PerDocConsumer perDocConsumer = perDocConsumers.get(0); if (perDocConsumer == null) { - PerDocWriteState perDocWriteState = docState.docWriter.newPerDocWriteState(fieldInfo.getCodecId()); - SegmentCodecs codecs = perDocWriteState.segmentCodecs; - assert codecs.codecs.length > fieldInfo.getCodecId(); - Codec codec = codecs.codecs[fieldInfo.getCodecId()]; - perDocConsumer = codec.docsConsumer(perDocWriteState); - perDocConsumers.put(Integer.valueOf(fieldInfo.getCodecId()), perDocConsumer); + PerDocWriteState perDocWriteState = docState.docWriter.newPerDocWriteState(""); + DocValuesFormat dvFormat = docState.docWriter.codec.docValuesFormat(); + perDocConsumer = dvFormat.docsConsumer(perDocWriteState); + perDocConsumers.put(0, perDocConsumer); } boolean success = false; DocValuesConsumer docValuesConsumer = null; diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java index e414fe1..73931a4 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriter.java @@ -31,6 +31,7 @@ import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.Query; import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.AlreadyClosedException; @@ -125,8 +126,11 @@ final class DocumentsWriter { final DocumentsWriterPerThreadPool perThreadPool; final FlushPolicy flushPolicy; final DocumentsWriterFlushControl flushControl; - DocumentsWriter(IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers, + + final Codec codec; + DocumentsWriter(Codec codec, IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers, BufferedDeletesStream bufferedDeletesStream) throws IOException { + this.codec = codec; this.directory = directory; this.indexWriter = writer; this.similarityProvider = config.getSimilarityProvider(); diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index fae432f..1a42d77 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -26,7 +26,7 @@ import java.text.NumberFormat; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FlushInfo; @@ -152,7 +152,7 @@ public class DocumentsWriterPerThread { } private final static boolean INFO_VERBOSE = false; final DocumentsWriter parent; - final CodecProvider codecProvider; + final Codec codec; final IndexWriter writer; final Directory directory; final DocState docState; @@ -183,7 +183,7 @@ public class DocumentsWriterPerThread { this.fieldInfos = fieldInfos; this.writer = parent.indexWriter; this.infoStream = parent.infoStream; - this.codecProvider = this.writer.codecs; + this.codec = parent.codec; this.docState = new DocState(this); this.docState.similarityProvider = parent.indexWriter.getConfig() .getSimilarityProvider(); @@ -405,8 +405,8 @@ public class DocumentsWriterPerThread { return numDocsInRAM; } - SegmentCodecs getCodec() { - return flushState.segmentCodecs; + Codec getCodec() { + return flushState.codec; } /** Reset after a flush */ @@ -443,7 +443,7 @@ public class DocumentsWriterPerThread { assert deleteSlice == null : "all deletes must be applied in prepareFlush"; flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos, numDocsInRAM, writer.getConfig().getTermIndexInterval(), - fieldInfos.buildSegmentCodecs(true), pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed()))); + codec, pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed()))); final double startMBUsed = parent.flushControl.netBytes() / 1024. / 1024.; // Apply delete-by-docID now (delete-byDocID only // happens when an exception is hit processing that @@ -474,12 +474,12 @@ public class DocumentsWriterPerThread { try { consumer.flush(flushState); pendingDeletes.terms.clear(); - final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.segmentCodecs, fieldInfos.asReadOnly()); + final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.codec, fieldInfos.asReadOnly()); if (infoStream != null) { message("new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.liveDocs.count())) + " deleted docs"); message("new segment has " + (newSegment.getHasVectors() ? "vectors" : "no vectors")); message("flushedFiles=" + newSegment.files()); - message("flushed codecs=" + newSegment.getSegmentCodecs()); + message("flushed codec=" + newSegment.getCodec()); } flushedDocCount += flushState.numDocs; @@ -556,9 +556,9 @@ public class DocumentsWriterPerThread { bytesUsed.addAndGet(-(length *(INT_BLOCK_SIZE*RamUsageEstimator.NUM_BYTES_INT))); } - PerDocWriteState newPerDocWriteState(int codecId) { + PerDocWriteState newPerDocWriteState(String segmentSuffix) { assert segment != null; - return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, codecId, IOContext.DEFAULT); + return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, segmentSuffix, IOContext.DEFAULT); } void setInfoStream(PrintStream infoStream) { diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java index eb6f209..8b959ca 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java @@ -20,8 +20,6 @@ import java.util.Iterator; import java.util.concurrent.locks.ReentrantLock; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; -import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.SetOnce; /** @@ -128,7 +126,6 @@ public abstract class DocumentsWriterPerThreadPool { private final ThreadState[] perThreads; private volatile int numThreadStatesActive; - private CodecProvider codecProvider; private FieldNumberBiMap globalFieldMap; private final SetOnce documentsWriter = new SetOnce(); @@ -148,11 +145,9 @@ public abstract class DocumentsWriterPerThreadPool { public void initialize(DocumentsWriter documentsWriter, FieldNumberBiMap globalFieldMap, IndexWriterConfig config) { this.documentsWriter.set(documentsWriter); // thread pool is bound to DW - final CodecProvider codecs = config.getCodecProvider(); - this.codecProvider = codecs; this.globalFieldMap = globalFieldMap; for (int i = 0; i < perThreads.length; i++) { - final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)); + final FieldInfos infos = new FieldInfos(globalFieldMap); perThreads[i] = new ThreadState(new DocumentsWriterPerThread(documentsWriter.directory, documentsWriter, infos, documentsWriter.chain)); } } @@ -240,7 +235,7 @@ public abstract class DocumentsWriterPerThreadPool { assert threadState.isHeldByCurrentThread(); final DocumentsWriterPerThread dwpt = threadState.perThread; if (!closed) { - final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentCodecsBuilder.create(codecProvider)); + final FieldInfos infos = new FieldInfos(globalFieldMap); final DocumentsWriterPerThread newDwpt = new DocumentsWriterPerThread(dwpt, infos); newDwpt.initialize(); threadState.resetWriter(newDwpt); diff --git a/lucene/src/java/org/apache/lucene/index/FieldInfo.java b/lucene/src/java/org/apache/lucene/index/FieldInfo.java index 7507f69..2a6ec56 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldInfo.java +++ b/lucene/src/java/org/apache/lucene/index/FieldInfo.java @@ -21,7 +21,6 @@ import org.apache.lucene.index.values.ValueType; /** @lucene.experimental */ public final class FieldInfo { - public static final int UNASSIGNED_CODEC_ID = -1; public final String name; public final int number; @@ -38,7 +37,6 @@ public final class FieldInfo { public IndexOptions indexOptions; public boolean storePayloads; // whether this field stores payloads together with term positions - private int codecId = UNASSIGNED_CODEC_ID; // set inside SegmentCodecs#build() during segment flush - this is used to identify the codec used to write this field /** * Controls how much information is stored in the postings lists. @@ -77,21 +75,11 @@ public final class FieldInfo { } assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !storePayloads; } - - void setCodecId(int codecId) { - assert this.codecId == UNASSIGNED_CODEC_ID : "CodecId can only be set once."; - this.codecId = codecId; - } - - public int getCodecId() { - return codecId; - } @Override public Object clone() { FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues); - clone.codecId = this.codecId; return clone; } diff --git a/lucene/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/src/java/org/apache/lucene/index/FieldInfos.java index 08f7d60..306230e 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/src/java/org/apache/lucene/index/FieldInfos.java @@ -29,9 +29,6 @@ import java.util.TreeMap; import java.util.Map.Entry; import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.SegmentCodecs; // Required for Java 1.5 javadocs -import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -146,17 +143,6 @@ public final class FieldInfos implements Iterable { } /** - * Returns a new {@link FieldInfos} instance with this as the global field - * map - * - * @return a new {@link FieldInfos} instance with this as the global field - * map - */ - public FieldInfos newFieldInfos(SegmentCodecsBuilder segmentCodecsBuilder) { - return new FieldInfos(this, segmentCodecsBuilder); - } - - /** * Returns true iff the last committed version differs from the * current version, otherwise false * @@ -198,7 +184,6 @@ public final class FieldInfos implements Iterable { private final SortedMap byNumber = new TreeMap(); private final HashMap byName = new HashMap(); private final FieldNumberBiMap globalFieldNumbers; - private final SegmentCodecsBuilder segmentCodecsBuilder; // First used in 2.9; prior to 2.9 there was no format header public static final int FORMAT_START = -2; @@ -230,16 +215,15 @@ public final class FieldInfos implements Iterable { /** * Creates a new {@link FieldInfos} instance with a private - * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} and a default {@link SegmentCodecsBuilder} - * initialized with {@link CodecProvider#getDefault()}. + * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} *

* Note: this ctor should not be used during indexing use * {@link FieldInfos#FieldInfos(FieldInfos)} or - * {@link FieldInfos#FieldInfos(FieldNumberBiMap,org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder)} + * {@link FieldInfos#FieldInfos(FieldNumberBiMap)} * instead. */ public FieldInfos() { - this(new FieldNumberBiMap(), SegmentCodecsBuilder.create(CodecProvider.getDefault())); + this(new FieldNumberBiMap()); } /** @@ -249,7 +233,7 @@ public final class FieldInfos implements Iterable { * @see #isReadOnly() */ FieldInfos(FieldInfos other) { - this(other.globalFieldNumbers, other.segmentCodecsBuilder); + this(other.globalFieldNumbers); } /** @@ -257,9 +241,8 @@ public final class FieldInfos implements Iterable { * If the {@link FieldNumberBiMap} is null this instance will be read-only. * @see #isReadOnly() */ - FieldInfos(FieldNumberBiMap globalFieldNumbers, SegmentCodecsBuilder segmentCodecsBuilder) { + FieldInfos(FieldNumberBiMap globalFieldNumbers) { this.globalFieldNumbers = globalFieldNumbers; - this.segmentCodecsBuilder = segmentCodecsBuilder; } /** @@ -273,7 +256,7 @@ public final class FieldInfos implements Iterable { * @throws IOException */ public FieldInfos(Directory d, String name) throws IOException { - this((FieldNumberBiMap)null, null); // use null here to make this FIs Read-Only + this((FieldNumberBiMap)null); // use null here to make this FIs Read-Only final IndexInput input = d.openInput(name, IOContext.READONCE); try { read(input, name); @@ -309,7 +292,7 @@ public final class FieldInfos implements Iterable { */ @Override synchronized public Object clone() { - FieldInfos fis = new FieldInfos(globalFieldNumbers, segmentCodecsBuilder); + FieldInfos fis = new FieldInfos(globalFieldNumbers); fis.format = format; fis.hasFreq = hasFreq; fis.hasProx = hasProx; @@ -468,7 +451,6 @@ public final class FieldInfos implements Iterable { if (globalFieldNumbers == null) { throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos"); } - assert segmentCodecsBuilder != null : "SegmentCodecsBuilder is set to null but FieldInfos is not read-only"; FieldInfo fi = fieldInfo(name); if (fi == null) { final int fieldNumber = nextFieldNumber(name, preferredFieldNumber); @@ -477,9 +459,6 @@ public final class FieldInfos implements Iterable { fi.update(isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions); fi.setDocValues(docValues); } - if ((fi.isIndexed || fi.hasDocValues()) && fi.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) { - segmentCodecsBuilder.tryAddAndSet(fi); - } version++; return fi; } @@ -569,22 +548,6 @@ public final class FieldInfos implements Iterable { } return false; } - - /** - * Builds the {@link SegmentCodecs} mapping for this {@link FieldInfos} instance. - * @param clearBuilder true iff the internal {@link SegmentCodecsBuilder} must be cleared otherwise false - */ - public SegmentCodecs buildSegmentCodecs(boolean clearBuilder) { - if (globalFieldNumbers == null) { - throw new IllegalStateException("FieldInfos are read-only no SegmentCodecs available"); - } - assert segmentCodecsBuilder != null; - final SegmentCodecs segmentCodecs = segmentCodecsBuilder.build(); - if (clearBuilder) { - segmentCodecsBuilder.clear(); - } - return segmentCodecs; - } public void write(Directory d, String name) throws IOException { IndexOutput output = d.createOutput(name, IOContext.READONCE); @@ -628,7 +591,6 @@ public final class FieldInfos implements Iterable { bits |= OMIT_POSITIONS; output.writeString(fi.name); output.writeInt(fi.number); - output.writeInt(fi.getCodecId()); output.writeByte(bits); final byte b; @@ -698,9 +660,7 @@ public final class FieldInfos implements Iterable { for (int i = 0; i < size; i++) { String name = input.readString(); - // if this is a previous format codec 0 will be preflex! final int fieldNumber = format <= FORMAT_FLEX? input.readInt():i; - final int codecId = format <= FORMAT_FLEX? input.readInt():0; byte bits = input.readByte(); boolean isIndexed = (bits & IS_INDEXED) != 0; boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0; @@ -781,8 +741,7 @@ public final class FieldInfos implements Iterable { throw new IllegalStateException("unhandled indexValues type " + b); } } - final FieldInfo addInternal = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType); - addInternal.setCodecId(codecId); + addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType); } if (input.getFilePointer() != input.length()) { @@ -804,7 +763,7 @@ public final class FieldInfos implements Iterable { if (isReadOnly()) { return this; } - final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null, null); + final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null); for (FieldInfo fieldInfo : this) { FieldInfo clone = (FieldInfo) (fieldInfo).clone(); roFis.putInternal(clone); @@ -814,5 +773,14 @@ public final class FieldInfos implements Iterable { } return roFis; } - + + public boolean anyDocValuesFields() { + for (FieldInfo fi : this) { + if (fi.hasDocValues()) { + return true; + } + } + + return false; + } } diff --git a/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java b/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java index 203580d..758123c 100644 --- a/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java @@ -57,7 +57,7 @@ final class FreqProxTermsWriter extends TermsHashConsumer { // Sort by field name CollectionUtil.quickSort(allFields); - final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state); + final FieldsConsumer consumer = state.codec.postingsFormat().fieldsConsumer(state); boolean success = false; diff --git a/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java b/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java index 1d5587a..2c3be7d 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java @@ -18,7 +18,6 @@ package org.apache.lucene.index; */ import java.io.FileNotFoundException; -import java.io.FilenameFilter; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; @@ -29,7 +28,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NoSuchDirectoryException; import org.apache.lucene.util.CollectionUtil; @@ -122,8 +120,6 @@ final class IndexFileDeleter { infoStream.println("IFD [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message); } - private final FilenameFilter indexFilenameFilter; - // called only from assert private boolean locked() { return writer == null || Thread.holdsLock(writer); @@ -138,7 +134,7 @@ final class IndexFileDeleter { * @throws IOException if there is a low-level IO error */ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, - PrintStream infoStream, CodecProvider codecs, IndexWriter writer) throws CorruptIndexException, IOException { + PrintStream infoStream, IndexWriter writer) throws CorruptIndexException, IOException { this.infoStream = infoStream; this.writer = writer; @@ -154,7 +150,6 @@ final class IndexFileDeleter { // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.getGeneration(); - indexFilenameFilter = new IndexFileNameFilter(codecs); CommitPoint currentCommitPoint = null; String[] files = null; @@ -167,7 +162,7 @@ final class IndexFileDeleter { for (String fileName : files) { - if ((indexFilenameFilter.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) { + if ((IndexFileNameFilter.INSTANCE.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: getRefCount(fileName); @@ -180,9 +175,9 @@ final class IndexFileDeleter { if (infoStream != null) { message("init: load commit \"" + fileName + "\""); } - SegmentInfos sis = new SegmentInfos(codecs); + SegmentInfos sis = new SegmentInfos(); try { - sis.read(directory, fileName, codecs); + sis.read(directory, fileName); } catch (FileNotFoundException e) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth @@ -253,9 +248,9 @@ final class IndexFileDeleter { // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: - SegmentInfos sis = new SegmentInfos(codecs); + SegmentInfos sis = new SegmentInfos(); try { - sis.read(directory, currentSegmentsFile, codecs); + sis.read(directory, currentSegmentsFile); } catch (IOException e) { throw new CorruptIndexException("failed to locate current segments_N file"); } @@ -373,7 +368,7 @@ final class IndexFileDeleter { for(int i=0;iThis does not accept *-write.lock files. * * @lucene.internal */ public class IndexFileNameFilter implements FilenameFilter { - private final HashSet extensions; - - public IndexFileNameFilter(CodecProvider codecs) { - extensions = new HashSet(); - for (String ext : IndexFileNames.INDEX_EXTENSIONS) { - extensions.add(ext); - } - if (codecs != null) { - for(String ext : codecs.getAllExtensions()) { - extensions.add(ext); - } - } + public static final FilenameFilter INSTANCE = new IndexFileNameFilter(); + + private IndexFileNameFilter() { } + // Approximate match for files that seem to be Lucene + // index files. This can easily over-match, ie if some + // app names a file _foo_bar.go: + private final Pattern luceneFilePattern = Pattern.compile("^_[a-z0-9]+(_[a-z0-9]+)?\\.[a-z0-9]+$"); + /* (non-Javadoc) * @see java.io.FilenameFilter#accept(java.io.File, java.lang.String) */ public boolean accept(File dir, String name) { - int i = name.lastIndexOf('.'); - if (i != -1) { - String extension = name.substring(1+i); - if (extensions.contains(extension)) { - return true; - } else if (extension.startsWith("f") && - extension.matches("f\\d+")) { - return true; - } else if (extension.startsWith("s") && - extension.matches("s\\d+")) { - return true; - } + if (name.lastIndexOf('.') != -1) { + // Has an extension + return luceneFilePattern.matcher(name).matches(); } else { - if (name.startsWith(IndexFileNames.SEGMENTS)) return true; + // No extension -- only segments_N file; + return name.startsWith(IndexFileNames.SEGMENTS); } - return false; } } diff --git a/lucene/src/java/org/apache/lucene/index/IndexFileNames.java b/lucene/src/java/org/apache/lucene/index/IndexFileNames.java index e8d4ef6..0b3719e 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexFileNames.java +++ b/lucene/src/java/org/apache/lucene/index/IndexFileNames.java @@ -19,7 +19,9 @@ package org.apache.lucene.index; import java.util.regex.Pattern; -import org.apache.lucene.index.codecs.Codec; // for javadocs +import org.apache.lucene.index.codecs.PostingsFormat; // for javadocs + +// TODO: put all files under codec and remove all the static extensions here /** * This class contains useful constants representing filenames and extensions @@ -31,7 +33,7 @@ import org.apache.lucene.index.codecs.Codec; // for javadocs * {@link #segmentFileName(String, String, String) segmentFileName}). * *

NOTE: extensions used by codecs are not - * listed here. You must interact with the {@link Codec} + * listed here. You must interact with the {@link PostingsFormat} * directly. * * @lucene.internal @@ -188,20 +190,20 @@ public final class IndexFileNames { * NOTE: .<ext> is added to the result file name only if * ext is not empty. *

- * NOTE: _<name> is added to the result file name only if - * name is not empty. + * NOTE: _<segmentSuffix> is added to the result file name only if + * it's not the empty string *

* NOTE: all custom files should be named using this method, or * otherwise some structures may fail to handle them properly (such as if they * are added to compound files). */ - public static String segmentFileName(String segmentName, String name, String ext) { - if (ext.length() > 0 || name.length() > 0) { + public static String segmentFileName(String segmentName, String segmentSuffix, String ext) { + if (ext.length() > 0 || segmentSuffix.length() > 0) { assert !ext.startsWith("."); - StringBuilder sb = new StringBuilder(segmentName.length() + 2 + name.length() + ext.length()); + StringBuilder sb = new StringBuilder(segmentName.length() + 2 + segmentSuffix.length() + ext.length()); sb.append(segmentName); - if (name.length() > 0) { - sb.append('_').append(name); + if (segmentSuffix.length() > 0) { + sb.append('_').append(segmentSuffix); } if (ext.length() > 0) { sb.append('.').append(ext); @@ -212,11 +214,6 @@ public final class IndexFileNames { } } - /** Sugar for passing "" + name instead */ - public static String segmentFileName(String segmentName, int name, String ext) { - return segmentFileName(segmentName, ""+name, ext); - } - /** * Returns true if the given filename ends with the given extension. One * should provide a pure extension, without '.'. diff --git a/lucene/src/java/org/apache/lucene/index/IndexReader.java b/lucene/src/java/org/apache/lucene/index/IndexReader.java index 0961853..ddd82b5 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/IndexReader.java @@ -28,8 +28,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.document.Document; import org.apache.lucene.document.DocumentStoredFieldVisitor; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.values.IndexDocValues; import org.apache.lucene.search.FieldCache; // javadocs @@ -312,7 +311,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory) throws CorruptIndexException, IOException { - return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR); } /** Returns an IndexReader reading the index in the given @@ -326,9 +325,9 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } - + /** * Open a near real time IndexReader from the {@link org.apache.lucene.index.IndexWriter}. * @@ -363,7 +362,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } /** Expert: returns an IndexReader reading the index in @@ -381,7 +380,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } /** Expert: returns an IndexReader reading the index in @@ -409,7 +408,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, null); + return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor); } /** Expert: returns an IndexReader reading the index in @@ -429,7 +428,7 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } /** Expert: returns an IndexReader reading the index in @@ -462,78 +461,11 @@ public abstract class IndexReader implements Cloneable,Closeable { * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, null); - } - - /** Expert: returns an IndexReader reading the index in - * the given Directory, with a custom {@link - * IndexDeletionPolicy}, and specified {@link CodecProvider}. - * You should pass readOnly=true, since it gives much - * better concurrent performance, unless you intend to do - * write operations (delete documents or change norms) - * with the reader. - * @param directory the index directory - * @param deletionPolicy a custom deletion policy (only used - * if you use this reader to perform deletes or to set - * norms); see {@link IndexWriter} for details. - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader - * @param termInfosIndexDivisor Subsamples which indexed - * terms are loaded into RAM. This has the same effect as {@link - * IndexWriterConfig#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1. Set this - * to -1 to skip loading the terms index entirely. - * @param codecs CodecProvider to use when opening index - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, codecs); + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor); } - /** Expert: returns an IndexReader reading the index in - * the given Directory, using a specific commit and with - * a custom {@link IndexDeletionPolicy} and specified - * {@link CodecProvider}. You should pass readOnly=true, since - * it gives much better concurrent performance, unless - * you intend to do write operations (delete documents or - * change norms) with the reader. - - * @param commit the specific {@link IndexCommit} to open; - * see {@link IndexReader#listCommits} to list all commits - * in a directory - * @param deletionPolicy a custom deletion policy (only used - * if you use this reader to perform deletes or to set - * norms); see {@link IndexWriter} for details. - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader - * @param termInfosIndexDivisor Subsamples which indexed - * terms are loaded into RAM. This has the same effect as {@link - * IndexWriterConfig#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1. Set this - * to -1 to skip loading the terms index entirely. - * @param codecs CodecProvider to use when opening index - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs); - } - - private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor, - CodecProvider codecs) throws CorruptIndexException, IOException { - if (codecs == null) { - codecs = CodecProvider.getDefault(); - } - return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs); + private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { + return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor); } /** @@ -767,20 +699,6 @@ public abstract class IndexReader implements Cloneable,Closeable { } }.run()).longValue(); } - - /** - * Reads version number from segments files. The version number is - * initialized with a timestamp and then increased by one for each change of - * the index. - * - * @param directory where the index resides. - * @return version number. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static long getCurrentVersion(Directory directory) throws CorruptIndexException, IOException { - return getCurrentVersion(directory, CodecProvider.getDefault()); - } /** * Reads version number from segments files. The version number is @@ -788,34 +706,14 @@ public abstract class IndexReader implements Cloneable,Closeable { * the index. * * @param directory where the index resides. - * @param codecs the {@link CodecProvider} holding all {@link Codec}s required to open the index * @return version number. * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static long getCurrentVersion(Directory directory, CodecProvider codecs) throws CorruptIndexException, IOException { - return SegmentInfos.readCurrentVersion(directory, codecs); - } - - /** - * Reads commitUserData, previously passed to {@link - * IndexWriter#commit(Map)}, from current index - * segments file. This will return null if {@link - * IndexWriter#commit(Map)} has never been called for - * this index. - * - * @param directory where the index resides. - * @return commit userData. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - * - * @see #getCommitUserData() - */ - public static Map getCommitUserData(Directory directory) throws CorruptIndexException, IOException { - return getCommitUserData(directory, CodecProvider.getDefault()); + public static long getCurrentVersion(Directory directory) throws CorruptIndexException, IOException { + return SegmentInfos.readCurrentVersion(directory); } - /** * Reads commitUserData, previously passed to {@link * IndexWriter#commit(Map)}, from current index @@ -824,15 +722,14 @@ public abstract class IndexReader implements Cloneable,Closeable { * this index. * * @param directory where the index resides. - * @param codecs the {@link CodecProvider} provider holding all {@link Codec}s required to open the index * @return commit userData. * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error * * @see #getCommitUserData() */ - public static Map getCommitUserData(Directory directory, CodecProvider codecs) throws CorruptIndexException, IOException { - return SegmentInfos.readCurrentUserData(directory, codecs); + public static Map getCommitUserData(Directory directory) throws CorruptIndexException, IOException { + return SegmentInfos.readCurrentUserData(directory); } /** @@ -985,22 +882,6 @@ public abstract class IndexReader implements Cloneable,Closeable { } } - /** - * Returns true if an index exists at the specified directory. - * @param directory the directory to check for an index - * @param codecProvider provides a CodecProvider in case the index uses non-core codecs - * @return true if an index exists; false otherwise - * @throws IOException if there is a problem with accessing the index - */ - public static boolean indexExists(Directory directory, CodecProvider codecProvider) throws IOException { - try { - new SegmentInfos().read(directory, codecProvider); - return true; - } catch (IOException ioe) { - return false; - } - } - /** Returns the number of documents in this index. */ public abstract int numDocs(); diff --git a/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java b/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java index 87a656a..34d5348 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java +++ b/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java @@ -114,12 +114,12 @@ public final class IndexUpgrader { } public void upgrade() throws IOException { - if (!IndexReader.indexExists(dir, iwc.getCodecProvider())) { + if (!IndexReader.indexExists(dir)) { throw new IndexNotFoundException(dir.toString()); } if (!deletePriorCommits) { - final Collection commits = DirectoryReader.listCommits(dir, iwc.getCodecProvider()); + final Collection commits = DirectoryReader.listCommits(dir); if (commits.size() > 1) { throw new IllegalArgumentException("This tool was invoked to not delete prior commit points, but the following commits were found: " + commits); } diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index 0f5b284..56c4571 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -39,8 +39,7 @@ import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor; -import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.Query; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.CompoundFileDirectory; @@ -375,7 +374,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // just like we do when loading segments_N synchronized(this) { maybeApplyDeletes(applyAllDeletes); - r = new DirectoryReader(this, segmentInfos, codecs, applyAllDeletes); + r = new DirectoryReader(this, segmentInfos, applyAllDeletes); if (infoStream != null) { message("return reader version=" + r.getVersion() + " reader=" + r); } @@ -802,7 +801,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { infoStream.println("IW " + messageID + " [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message); } - CodecProvider codecs; + final Codec codec; // for writing new segments /** * Constructs a new IndexWriter per the settings given in conf. @@ -837,7 +836,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { mergePolicy = conf.getMergePolicy(); mergePolicy.setIndexWriter(this); mergeScheduler = conf.getMergeScheduler(); - codecs = conf.getCodecProvider(); + codec = conf.getCodec(); bufferedDeletesStream = new BufferedDeletesStream(messageID); bufferedDeletesStream.setInfoStream(infoStream); @@ -862,7 +861,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // If index is too old, reading the segments will throw // IndexFormatTooOldException. - segmentInfos = new SegmentInfos(codecs); + segmentInfos = new SegmentInfos(); try { if (create) { // Try to read first. This is to allow create @@ -870,7 +869,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // searching. In this case we write the next // segments_N file with no segments: try { - segmentInfos.read(directory, codecs); + segmentInfos.read(directory); segmentInfos.clear(); } catch (IOException e) { // Likely this means it's a fresh directory @@ -881,7 +880,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { changeCount++; segmentInfos.changed(); } else { - segmentInfos.read(directory, codecs); + segmentInfos.read(directory); IndexCommit commit = conf.getIndexCommit(); if (commit != null) { @@ -892,8 +891,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // points. if (commit.getDirectory() != directory) throw new IllegalArgumentException("IndexCommit's directory doesn't match my directory"); - SegmentInfos oldInfos = new SegmentInfos(codecs); - oldInfos.read(directory, commit.getSegmentsFileName(), codecs); + SegmentInfos oldInfos = new SegmentInfos(); + oldInfos.read(directory, commit.getSegmentsFileName()); segmentInfos.replace(oldInfos); changeCount++; segmentInfos.changed(); @@ -906,7 +905,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // start with previous field numbers, but new FieldInfos globalFieldNumberMap = segmentInfos.getOrLoadGlobalFieldNumberMap(directory); - docWriter = new DocumentsWriter(config, directory, this, globalFieldNumberMap, bufferedDeletesStream); + docWriter = new DocumentsWriter(codec, config, directory, this, globalFieldNumberMap, bufferedDeletesStream); docWriter.setInfoStream(infoStream); // Default deleter (for backwards compatibility) is @@ -914,8 +913,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { synchronized(this) { deleter = new IndexFileDeleter(directory, conf.getIndexDeletionPolicy(), - segmentInfos, infoStream, - codecs, this); + segmentInfos, infoStream, this); } if (deleter.startingCommitDeleted) { @@ -2149,6 +2147,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { */ public synchronized void deleteAll() throws IOException { ensureOpen(); + boolean success = false; try { // Abort any running merges @@ -2170,10 +2169,11 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Mark that the index has changed ++changeCount; segmentInfos.changed(); + success = true; } catch (OutOfMemoryError oom) { handleOOM(oom, "deleteAll"); } finally { - if (infoStream != null) { + if (!success && infoStream != null) { message("hit exception during deleteAll"); } } @@ -2476,8 +2476,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { if (infoStream != null) { message("addIndexes: process directory " + dir); } - SegmentInfos sis = new SegmentInfos(codecs); // read infos from dir - sis.read(dir, codecs); + SegmentInfos sis = new SegmentInfos(); // read infos from dir + sis.read(dir); final Set dsFilesCopied = new HashSet(); final Map dsNames = new HashMap(); for (SegmentInfo info : sis) { @@ -2567,7 +2567,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // abortable so that IW.close(false) is able to stop it SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, null, payloadProcessorProvider, - globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)), context); + new FieldInfos(globalFieldNumberMap), codec, context); for (IndexReader reader : readers) // add new indexes merger.add(reader); @@ -2575,7 +2575,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { final FieldInfos fieldInfos = merger.fieldInfos(); SegmentInfo info = new SegmentInfo(mergedName, docCount, directory, - false, merger.getSegmentCodecs(), + false, merger.getCodec(), fieldInfos); setDiagnostics(info, "addIndexes(IndexReader...)"); @@ -2591,7 +2591,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Now create the compound file if needed if (useCompoundFile) { - merger.createCompoundFile(mergedName + ".cfs", info, context); + merger.createCompoundFile(IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_EXTENSION), info, context); // delete new non cfs files directly: they were never // registered with IFD @@ -2916,7 +2916,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { try { if (infoStream != null) message("commit: pendingCommit != null"); - pendingCommit.finishCommit(directory); + pendingCommit.finishCommit(directory, codec); if (infoStream != null) message("commit: wrote segments file \"" + pendingCommit.getCurrentSegmentFileName() + "\""); lastCommitChangeCount = pendingCommitChangeCount; @@ -3459,7 +3459,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Bind a new segment name here so even with // ConcurrentMergePolicy we keep deterministic segment // names. - merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs))); + merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, new FieldInfos(globalFieldNumberMap)); // Lock order: IW -> BD final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments); @@ -3633,7 +3633,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { IOContext context = new IOContext(merge.getMergeInfo()); SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge, - payloadProcessorProvider, merge.info.getFieldInfos(), context); + payloadProcessorProvider, merge.info.getFieldInfos(), codec, context); if (infoStream != null) { message("merging " + merge.segString(directory) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors()); @@ -3679,10 +3679,10 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { mergedDocCount = merge.info.docCount = merger.merge(); // Record which codec was used to write the segment - merge.info.setSegmentCodecs(merger.getSegmentCodecs()); + merge.info.setCodec(merger.getCodec()); if (infoStream != null) { - message("merge segmentCodecs=" + merger.getSegmentCodecs()); + message("merge codecs=" + merger.getCodec()); message("merge store matchedCount=" + merger.getMatchedSubReaderCount() + " vs " + merge.readers.size()); } anyNonBulkMerges |= merger.getAnyNonBulkMerges(); @@ -3975,7 +3975,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Exception here means nothing is prepared // (this method unwinds everything it did on // an exception) - toSync.prepareCommit(directory); + toSync.prepareCommit(directory, codec); pendingCommitSet = true; pendingCommit = toSync; diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java b/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java index 326a1d8..af56141 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java @@ -20,7 +20,7 @@ package org.apache.lucene.index; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.util.Version; @@ -121,7 +121,7 @@ public final class IndexWriterConfig implements Cloneable { private volatile int maxBufferedDocs; private volatile IndexingChain indexingChain; private volatile IndexReaderWarmer mergedSegmentWarmer; - private volatile CodecProvider codecProvider; + private volatile Codec codec; private volatile MergePolicy mergePolicy; private volatile DocumentsWriterPerThreadPool indexerThreadPool; private volatile boolean readerPooling; @@ -158,7 +158,7 @@ public final class IndexWriterConfig implements Cloneable { maxBufferedDocs = DEFAULT_MAX_BUFFERED_DOCS; indexingChain = DocumentsWriterPerThread.defaultIndexingChain; mergedSegmentWarmer = null; - codecProvider = CodecProvider.getDefault(); + codec = Codec.getDefault(); if (matchVersion.onOrAfter(Version.LUCENE_32)) { mergePolicy = new TieredMergePolicy(); } else { @@ -521,17 +521,17 @@ public final class IndexWriterConfig implements Cloneable { return this; } - /** Set the CodecProvider. See {@link CodecProvider}. + /** Set the Codec. See {@link Codec}. * *

Only takes effect when IndexWriter is first created. */ - public IndexWriterConfig setCodecProvider(CodecProvider codecProvider) { - this.codecProvider = codecProvider; + public IndexWriterConfig setCodec(Codec codec) { + this.codec = codec; return this; } - /** Returns the current merged segment warmer. See {@link IndexReaderWarmer}. */ - public CodecProvider getCodecProvider() { - return codecProvider; + /** Returns the current Codec. See {@link Codec}. */ + public Codec getCodec() { + return codec; } @@ -694,7 +694,7 @@ public final class IndexWriterConfig implements Cloneable { sb.append("ramBufferSizeMB=").append(ramBufferSizeMB).append("\n"); sb.append("maxBufferedDocs=").append(maxBufferedDocs).append("\n"); sb.append("mergedSegmentWarmer=").append(mergedSegmentWarmer).append("\n"); - sb.append("codecProvider=").append(codecProvider).append("\n"); + sb.append("codec=").append(codec).append("\n"); sb.append("mergePolicy=").append(mergePolicy).append("\n"); sb.append("indexerThreadPool=").append(indexerThreadPool).append("\n"); sb.append("readerPooling=").append(readerPooling).append("\n"); diff --git a/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java b/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java index d5c1abb..8821bee 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java +++ b/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java @@ -19,7 +19,6 @@ package org.apache.lucene.index; import org.apache.lucene.index.FieldInfo.IndexOptions; -/** @lucene.experimental */ public interface IndexableFieldType { /** True if this field should be indexed (inverted) */ diff --git a/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java b/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java index 84574e6..cfa1ab0 100644 --- a/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java +++ b/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java @@ -34,41 +34,37 @@ public class PerDocWriteState { public final String segmentName; public final FieldInfos fieldInfos; public final Counter bytesUsed; - public final SegmentCodecs segmentCodecs; - public final int codecId; + public final String segmentSuffix; public final IOContext context; - PerDocWriteState(PrintStream infoStream, Directory directory, + public PerDocWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos, Counter bytesUsed, - int codecId, IOContext context) { + String segmentSuffix, IOContext context) { this.infoStream = infoStream; this.directory = directory; this.segmentName = segmentName; this.fieldInfos = fieldInfos; - this.segmentCodecs = fieldInfos.buildSegmentCodecs(false); - this.codecId = codecId; + this.segmentSuffix = segmentSuffix; this.bytesUsed = bytesUsed; this.context = context; } - PerDocWriteState(SegmentWriteState state) { + public PerDocWriteState(SegmentWriteState state) { infoStream = state.infoStream; directory = state.directory; - segmentCodecs = state.segmentCodecs; segmentName = state.segmentName; fieldInfos = state.fieldInfos; - codecId = state.codecId; + segmentSuffix = state.segmentSuffix; bytesUsed = Counter.newCounter(); context = state.context; } - PerDocWriteState(PerDocWriteState state, int codecId) { + public PerDocWriteState(PerDocWriteState state, String segmentSuffix) { this.infoStream = state.infoStream; this.directory = state.directory; this.segmentName = state.segmentName; this.fieldInfos = state.fieldInfos; - this.segmentCodecs = state.segmentCodecs; - this.codecId = codecId; + this.segmentSuffix = segmentSuffix; this.bytesUsed = state.bytesUsed; this.context = state.context; } diff --git a/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java b/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java deleted file mode 100644 index 9e950d1..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java b/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java deleted file mode 100644 index 68c4aa3..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java b/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java index 3e6e307..f5f77c6 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FieldsReader; import org.apache.lucene.index.codecs.PerDocValues; @@ -68,7 +69,7 @@ final class SegmentCoreReaders { } segment = si.name; - final SegmentCodecs segmentCodecs = si.getSegmentCodecs(); + final Codec codec = si.getCodec(); this.context = context; this.dir = dir; @@ -85,12 +86,12 @@ final class SegmentCoreReaders { fieldInfos = si.getFieldInfos(); this.termsIndexDivisor = termsIndexDivisor; - final Codec codec = segmentCodecs.codec(); + final PostingsFormat format = codec.postingsFormat(); final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, context, termsIndexDivisor); // Ask codec for its Fields - fields = codec.fieldsProducer(segmentReadState); + fields = format.fieldsProducer(segmentReadState); assert fields != null; - perDocProducer = codec.docsProducer(segmentReadState); + perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState); success = true; } finally { if (!success) { @@ -165,7 +166,7 @@ final class SegmentCoreReaders { } final String storesSegment = si.getDocStoreSegment(); - fieldsReaderOrig = si.getSegmentCodecs().provider.fieldsReader(storeDir, storesSegment, fieldInfos, context, + fieldsReaderOrig = si.getCodec().fieldsFormat().fieldsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount); // Verify two sources of "maxDoc" agree: diff --git a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java index 005672e..e4bf7fc 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java @@ -28,7 +28,6 @@ import java.util.Map.Entry; import java.util.Set; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; @@ -97,7 +96,7 @@ public final class SegmentInfo implements Cloneable { private FieldInfos fieldInfos; - private SegmentCodecs segmentCodecs; + private Codec codec; private Map diagnostics; @@ -116,7 +115,7 @@ public final class SegmentInfo implements Cloneable { private long fieldInfosVersion; public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, - SegmentCodecs segmentCodecs, FieldInfos fieldInfos) { + Codec codec, FieldInfos fieldInfos) { this.name = name; this.docCount = docCount; this.dir = dir; @@ -124,7 +123,7 @@ public final class SegmentInfo implements Cloneable { this.isCompoundFile = isCompoundFile; this.docStoreOffset = -1; this.docStoreSegment = name; - this.segmentCodecs = segmentCodecs; + this.codec = codec; delCount = 0; version = Constants.LUCENE_MAIN_VERSION; this.fieldInfos = fieldInfos; @@ -156,7 +155,7 @@ public final class SegmentInfo implements Cloneable { } isCompoundFile = src.isCompoundFile; delCount = src.delCount; - segmentCodecs = src.segmentCodecs; + codec = src.codec; } void setDiagnostics(Map diagnostics) { @@ -177,7 +176,7 @@ public final class SegmentInfo implements Cloneable { * @param format format of the segments info file * @param input input handle to read segment info from */ - public SegmentInfo(Directory dir, int format, IndexInput input, CodecProvider codecs) throws IOException { + public SegmentInfo(Directory dir, int format, IndexInput input) throws IOException { this.dir = dir; if (format <= DefaultSegmentInfosWriter.FORMAT_3_1) { version = input.readString(); @@ -221,13 +220,13 @@ public final class SegmentInfo implements Cloneable { hasProx = input.readByte(); + // System.out.println(Thread.currentThread().getName() + ": si.read hasProx=" + hasProx + " seg=" + name); + // note: if the codec is not available: Codec.forName will throw an exception. if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { - segmentCodecs = new SegmentCodecs(codecs, input); + codec = Codec.forName(input.readString()); } else { - // codec ID on FieldInfo is 0 so it will simply use the first codec available - // TODO what todo if preflex is not available in the provider? register it or fail? - segmentCodecs = new SegmentCodecs(codecs, new Codec[] { codecs.lookup("PreFlex")}); + codec = Codec.forName("Lucene3x"); } diagnostics = input.readStringStringMap(); @@ -350,7 +349,7 @@ public final class SegmentInfo implements Cloneable { @Override public Object clone() { - final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, segmentCodecs, + final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, codec, fieldInfos == null ? null : (FieldInfos) fieldInfos.clone()); si.docStoreOffset = docStoreOffset; si.docStoreSegment = docStoreSegment; @@ -573,7 +572,7 @@ public final class SegmentInfo implements Cloneable { output.writeByte((byte) (isCompoundFile ? YES : NO)); output.writeInt(delCount); output.writeByte((byte) (hasProx)); - segmentCodecs.write(output); + output.writeString(codec.getName()); output.writeStringStringMap(diagnostics); output.writeByte((byte) (hasVectors)); } @@ -583,16 +582,16 @@ public final class SegmentInfo implements Cloneable { } /** Can only be called once. */ - public void setSegmentCodecs(SegmentCodecs segmentCodecs) { - assert this.segmentCodecs == null; - if (segmentCodecs == null) { + public void setCodec(Codec codec) { + assert this.codec == null; + if (codec == null) { throw new IllegalArgumentException("segmentCodecs must be non-null"); } - this.segmentCodecs = segmentCodecs; + this.codec = codec; } - SegmentCodecs getSegmentCodecs() { - return segmentCodecs; + Codec getCodec() { + return codec; } private void addIfExists(Set files, String fileName) throws IOException { @@ -628,7 +627,7 @@ public final class SegmentInfo implements Cloneable { for(String ext : IndexFileNames.NON_STORE_INDEX_EXTENSIONS) { addIfExists(fileSet, IndexFileNames.segmentFileName(name, "", ext)); } - segmentCodecs.files(dir, this, fileSet); + codec.files(dir, this, fileSet); } if (docStoreOffset != -1) { diff --git a/lucene/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/src/java/org/apache/lucene/index/SegmentInfos.java index f9867b8..1040674 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentInfos.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentInfos.java @@ -32,10 +32,11 @@ import java.util.Map; import java.util.Set; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.index.codecs.SegmentInfosReader; import org.apache.lucene.index.codecs.SegmentInfosWriter; +import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -83,8 +84,6 @@ public final class SegmentInfos implements Cloneable, Iterable { // there was an IOException that had interrupted a commit public Map userData = Collections.emptyMap(); // Opaque Map that user can specify during IndexWriter.commit - - private CodecProvider codecs; private int format; @@ -95,20 +94,14 @@ public final class SegmentInfos implements Cloneable, Iterable { private transient List cachedUnmodifiableList; private transient Set cachedUnmodifiableSet; + private Codec codecFormat; + /** * If non-null, information about loading segments_N files * will be printed here. @see #setInfoStream. */ private static PrintStream infoStream = null; - public SegmentInfos() { - this(CodecProvider.getDefault()); - } - - public SegmentInfos(CodecProvider codecs) { - this.codecs = codecs; - } - public void setFormat(int format) { this.format = format; } @@ -197,7 +190,7 @@ public final class SegmentInfos implements Cloneable, Iterable { * since this file might belong to more than one segment (global map) and * could otherwise easily be confused with a per-segment file. */ - return IndexFileNames.segmentFileName(""+ version, "", IndexFileNames.GLOBAL_FIELD_NUM_MAP_EXTENSION); + return IndexFileNames.segmentFileName("_"+ version, "", IndexFileNames.GLOBAL_FIELD_NUM_MAP_EXTENSION); } /** @@ -241,9 +234,7 @@ public final class SegmentInfos implements Cloneable, Iterable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public final void read(Directory directory, String segmentFileName, - CodecProvider codecs) throws CorruptIndexException, IOException { - this.codecs = codecs; + public final void read(Directory directory, String segmentFileName) throws CorruptIndexException, IOException { boolean success = false; // Clear any previous segments: @@ -253,12 +244,40 @@ public final class SegmentInfos implements Cloneable, Iterable { lastGeneration = generation; + // TODO: scary to have default impl reopen the file... but to make it a bit more flexible, + // maybe we could use a plain indexinput here... could default impl rewind/wrap with checksumII, + // and any checksumming is then up to implementation? + ChecksumIndexInput input = null; try { - SegmentInfosReader infosReader = codecs.getSegmentInfosReader(); - infosReader.read(directory, segmentFileName, codecs, this, IOContext.READ); + input = new ChecksumIndexInput(directory.openInput(segmentFileName, IOContext.READ)); + final int format = input.readInt(); + setFormat(format); + + // check that it is a format we can understand + if (format > DefaultSegmentInfosWriter.FORMAT_MINIMUM) + throw new IndexFormatTooOldException(segmentFileName, format, + DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); + if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT) + throw new IndexFormatTooNewException(segmentFileName, format, + DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); + + if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { + codecFormat = Codec.forName(input.readString()); + } else { + codecFormat = Codec.forName("Lucene3x"); + } + SegmentInfosReader infosReader = codecFormat.segmentInfosFormat().getSegmentInfosReader(); + infosReader.read(directory, segmentFileName, input, this, IOContext.READ); + final long checksumNow = input.getChecksum(); + final long checksumThen = input.readLong(); + if (checksumNow != checksumThen) + throw new CorruptIndexException("checksum mismatch in segments file"); success = true; } finally { + if (input != null) { + input.close(); + } if (!success) { // Clear any segment infos we had loaded so we // have a clean slate on retry: @@ -267,25 +286,14 @@ public final class SegmentInfos implements Cloneable, Iterable { } } - /** - * This version of read uses the retry logic (for lock-less - * commits) to find the right segments file to load. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ public final void read(Directory directory) throws CorruptIndexException, IOException { - read(directory, CodecProvider.getDefault()); - } - - public final void read(Directory directory, final CodecProvider codecs) throws CorruptIndexException, IOException { generation = lastGeneration = -1; - this.codecs = codecs; new FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - read(directory, segmentFileName, codecs); + read(directory, segmentFileName); return null; } }.run(); @@ -297,7 +305,7 @@ public final class SegmentInfos implements Cloneable, Iterable { // before finishCommit is called IndexOutput pendingSegnOutput; - private void write(Directory directory) throws IOException { + private void write(Directory directory, Codec codec) throws IOException { String segmentFileName = getNextSegmentFileName(); final String globalFieldMapFile; @@ -322,8 +330,8 @@ public final class SegmentInfos implements Cloneable, Iterable { boolean success = false; try { - SegmentInfosWriter infosWriter = codecs.getSegmentInfosWriter(); - segnOutput = infosWriter.writeInfos(directory, segmentFileName, this, IOContext.DEFAULT); + SegmentInfosWriter infosWriter = codec.segmentInfosFormat().getSegmentInfosWriter(); + segnOutput = infosWriter.writeInfos(directory, segmentFileName, codec.getName(), this, IOContext.DEFAULT); infosWriter.prepareCommit(segnOutput); pendingSegnOutput = segnOutput; success = true; @@ -380,7 +388,7 @@ public final class SegmentInfos implements Cloneable, Iterable { sis.cachedUnmodifiableList = null; sis.cachedUnmodifiableSet = null; for(final SegmentInfo info : this) { - assert info.getSegmentCodecs() != null; + assert info.getCodec() != null; // dont directly access segments, use add method!!! sis.add((SegmentInfo) info.clone()); } @@ -409,7 +417,7 @@ public final class SegmentInfos implements Cloneable, Iterable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static long readCurrentVersion(Directory directory, final CodecProvider codecs) + public static long readCurrentVersion(Directory directory) throws CorruptIndexException, IOException { // Fully read the segments file: this ensures that it's @@ -417,8 +425,8 @@ public final class SegmentInfos implements Cloneable, Iterable { // IndexWriter.prepareCommit has been called (but not // yet commit), then the reader will still see itself as // current: - SegmentInfos sis = new SegmentInfos(codecs); - sis.read(directory, codecs); + SegmentInfos sis = new SegmentInfos(); + sis.read(directory); return sis.version; } @@ -427,10 +435,10 @@ public final class SegmentInfos implements Cloneable, Iterable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static Map readCurrentUserData(Directory directory, CodecProvider codecs) + public static Map readCurrentUserData(Directory directory) throws CorruptIndexException, IOException { - SegmentInfos sis = new SegmentInfos(codecs); - sis.read(directory, codecs); + SegmentInfos sis = new SegmentInfos(); + sis.read(directory); return sis.getUserData(); } @@ -808,10 +816,10 @@ public final class SegmentInfos implements Cloneable, Iterable { * method if changes have been made to this {@link SegmentInfos} instance *

**/ - final void prepareCommit(Directory dir) throws IOException { + final void prepareCommit(Directory dir, Codec codec) throws IOException { if (pendingSegnOutput != null) throw new IllegalStateException("prepareCommit was already called"); - write(dir); + write(dir, codec); } private final long writeGlobalFieldMap(FieldNumberBiMap map, Directory dir, String name) throws IOException { @@ -882,12 +890,12 @@ public final class SegmentInfos implements Cloneable, Iterable { return files; } - final void finishCommit(Directory dir) throws IOException { + final void finishCommit(Directory dir, Codec codec) throws IOException { if (pendingSegnOutput == null) throw new IllegalStateException("prepareCommit was not called"); boolean success = false; try { - SegmentInfosWriter infosWriter = codecs.getSegmentInfosWriter(); + SegmentInfosWriter infosWriter = codec.segmentInfosFormat().getSegmentInfosWriter(); infosWriter.finishCommit(pendingSegnOutput); pendingSegnOutput = null; success = true; @@ -958,9 +966,9 @@ public final class SegmentInfos implements Cloneable, Iterable { * method if changes have been made to this {@link SegmentInfos} instance *

**/ - final void commit(Directory dir) throws IOException { - prepareCommit(dir); - finishCommit(dir); + final void commit(Directory dir, Codec codec) throws IOException { + prepareCommit(dir, codec); + finishCommit(dir, codec); } public String toString(Directory directory) { @@ -1106,7 +1114,7 @@ public final class SegmentInfos implements Cloneable, Iterable { if (cloneChildren) { final List list = new ArrayList(size()); for(final SegmentInfo info : this) { - assert info.getSegmentCodecs() != null; + assert info.getCodec() != null; list.add((SegmentInfo) info.clone()); } return list; @@ -1120,6 +1128,14 @@ public final class SegmentInfos implements Cloneable, Iterable { this.addAll(infos); } + /** + * Returns the codec used to decode this SegmentInfos from disk + * @lucene.internal + */ + Codec codecFormat() { + return codecFormat; + } + /** Returns an unmodifiable {@link Iterator} of contained segments in order. */ // @Override (comment out until Java 6) public Iterator iterator() { diff --git a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java index a53d4b3..2e775b1 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java @@ -73,7 +73,7 @@ final class SegmentMerger { private IOContext context; - SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, IOContext context) { + SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, Codec codec, IOContext context) { this.payloadProcessorProvider = payloadProcessorProvider; directory = dir; segment = name; @@ -89,6 +89,7 @@ final class SegmentMerger { }; } this.termIndexInterval = termIndexInterval; + this.codec = codec; this.context = context; } @@ -254,12 +255,11 @@ final class SegmentMerger { fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.DOC_VALUES), false); } } - final SegmentCodecs codecInfo = fieldInfos.buildSegmentCodecs(false); int docCount = 0; setMatchingSegmentReaders(); - final FieldsWriter fieldsWriter = codecInfo.provider.fieldsWriter(directory, segment, context); + final FieldsWriter fieldsWriter = codec.fieldsFormat().fieldsWriter(directory, segment, context); try { int idx = 0; for (MergeState.IndexReaderAndLiveDocs reader : readers) { @@ -293,7 +293,7 @@ final class SegmentMerger { // entering the index. See LUCENE-1282 for // details. throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption"); - segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo, null, context); + segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codec, null, context); return docCount; } @@ -493,9 +493,9 @@ final class SegmentMerger { } } - SegmentCodecs getSegmentCodecs() { + Codec getCodec() { assert segmentWriteState != null; - return segmentWriteState.segmentCodecs; + return segmentWriteState.codec; } private final void mergeTerms() throws CorruptIndexException, IOException { @@ -566,8 +566,8 @@ final class SegmentMerger { mergeState.dirPayloadProcessor[i] = payloadProcessorProvider.getDirProcessor(reader.reader.directory()); } } - codec = segmentWriteState.segmentCodecs.codec(); - final FieldsConsumer consumer = codec.fieldsConsumer(segmentWriteState); + + final FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(segmentWriteState); boolean success = false; try { consumer.merge(mergeState, @@ -584,7 +584,7 @@ final class SegmentMerger { } private void mergePerDoc() throws IOException { - final PerDocConsumer docsConsumer = codec + final PerDocConsumer docsConsumer = codec.docValuesFormat() .docsConsumer(new PerDocWriteState(segmentWriteState)); // TODO: remove this check when 3.x indexes are no longer supported // (3.x indexes don't have docvalues) diff --git a/lucene/src/java/org/apache/lucene/index/SegmentReadState.java b/lucene/src/java/org/apache/lucene/index/SegmentReadState.java index e7e717f..b59d572 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentReadState.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentReadState.java @@ -35,11 +35,11 @@ public class SegmentReadState { // that must do so), then it should negate this value to // get the app's terms divisor: public int termsIndexDivisor; - public final int codecId; + public final String segmentSuffix; public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, IOContext context, int termsIndexDivisor) { - this(dir, info, fieldInfos, context, termsIndexDivisor, -1); + this(dir, info, fieldInfos, context, termsIndexDivisor, ""); } public SegmentReadState(Directory dir, @@ -47,12 +47,22 @@ public class SegmentReadState { FieldInfos fieldInfos, IOContext context, int termsIndexDivisor, - int codecId) { + String segmentSuffix) { this.dir = dir; this.segmentInfo = info; this.fieldInfos = fieldInfos; this.context = context; this.termsIndexDivisor = termsIndexDivisor; - this.codecId = codecId; + this.segmentSuffix = segmentSuffix; } -} \ No newline at end of file + + public SegmentReadState(SegmentReadState other, + String newSegmentSuffix) { + this.dir = other.dir; + this.segmentInfo = other.segmentInfo; + this.fieldInfos = other.fieldInfos; + this.context = other.context; + this.termsIndexDivisor = other.termsIndexDivisor; + this.segmentSuffix = newSegmentSuffix; + } +} diff --git a/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java b/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java index 270c084..b092fca 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java @@ -19,6 +19,7 @@ package org.apache.lucene.index; import java.io.PrintStream; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.BitVector; @@ -43,8 +44,8 @@ public class SegmentWriteState { // Lazily created: public BitVector liveDocs; - final SegmentCodecs segmentCodecs; - public final int codecId; + public final Codec codec; + public final String segmentSuffix; /** Expert: The fraction of terms in the "dictionary" which should be stored * in RAM. Smaller values use more memory, but make searching slightly @@ -56,7 +57,7 @@ public class SegmentWriteState { public final IOContext context; public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos, - int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, BufferedDeletes segDeletes, IOContext context) { + int numDocs, int termIndexInterval, Codec codec, BufferedDeletes segDeletes, IOContext context) { this.infoStream = infoStream; this.segDeletes = segDeletes; this.directory = directory; @@ -64,24 +65,24 @@ public class SegmentWriteState { this.fieldInfos = fieldInfos; this.numDocs = numDocs; this.termIndexInterval = termIndexInterval; - this.segmentCodecs = segmentCodecs; - codecId = -1; + this.codec = codec; + segmentSuffix = ""; this.context = context; } /** - * Create a shallow {@link SegmentWriteState} copy final a codec ID + * Create a shallow {@link SegmentWriteState} copy final a format ID */ - SegmentWriteState(SegmentWriteState state, int codecId) { + public SegmentWriteState(SegmentWriteState state, String segmentSuffix) { infoStream = state.infoStream; directory = state.directory; segmentName = state.segmentName; fieldInfos = state.fieldInfos; numDocs = state.numDocs; termIndexInterval = state.termIndexInterval; - segmentCodecs = state.segmentCodecs; context = state.context; - this.codecId = codecId; + codec = state.codec; + this.segmentSuffix = segmentSuffix; segDeletes = state.segDeletes; } } diff --git a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java index 08a3414..2dd2b79 100644 --- a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java @@ -19,7 +19,7 @@ package org.apache.lucene.index; import java.io.IOException; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldsWriter; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.ArrayUtil; @@ -35,12 +35,12 @@ final class StoredFieldsWriter { int freeCount; final DocumentsWriterPerThread.DocState docState; - final CodecProvider codecProvider; + final Codec codec; public StoredFieldsWriter(DocumentsWriterPerThread docWriter) { this.docWriter = docWriter; this.docState = docWriter.docState; - this.codecProvider = docWriter.codecProvider; + this.codec = docWriter.codec; } private int numStoredFields; @@ -81,7 +81,7 @@ final class StoredFieldsWriter { private synchronized void initFieldsWriter(IOContext context) throws IOException { if (fieldsWriter == null) { - fieldsWriter = codecProvider.fieldsWriter(docWriter.directory, docWriter.getSegment(), context); + fieldsWriter = codec.fieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegment(), context); lastDocID = 0; } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java index 6534109..5497c69 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java @@ -109,14 +109,14 @@ public class BlockTermsReader extends FieldsProducer { // private String segment; public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext context, - int termsCacheSize, int codecId) + int termsCacheSize, String segmentSuffix) throws IOException { this.postingsReader = postingsReader; termsCache = new DoubleBarrelLRUCache(termsCacheSize); // this.segment = segment; - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTermsWriter.TERMS_EXTENSION), + in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTermsWriter.TERMS_EXTENSION), context); boolean success = false; @@ -194,12 +194,8 @@ public class BlockTermsReader extends FieldsProducer { } } - public static void files(Directory dir, SegmentInfo segmentInfo, int id, Collection files) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, BlockTermsWriter.TERMS_EXTENSION)); - } - - public static void getExtensions(Collection extensions) { - extensions.add(BlockTermsWriter.TERMS_EXTENSION); + public static void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTermsWriter.TERMS_EXTENSION)); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java index a07b251..af93d73 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java @@ -71,7 +71,7 @@ public class BlockTermsWriter extends FieldsConsumer { public BlockTermsWriter(TermsIndexWriterBase termsIndexWriter, SegmentWriteState state, PostingsWriterBase postingsWriter) throws IOException { - final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_EXTENSION); this.termsIndexWriter = termsIndexWriter; out = state.directory.createOutput(termsFileName, state.context); boolean success = false; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java index aa04ae2..b409246 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java @@ -110,13 +110,13 @@ public class BlockTreeTermsReader extends FieldsProducer { public BlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext ioContext, - int codecId, int indexDivisor) + String segmentSuffix, int indexDivisor) throws IOException { this.postingsReader = postingsReader; this.segment = segment; - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTreeTermsWriter.TERMS_EXTENSION), + in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTreeTermsWriter.TERMS_EXTENSION), ioContext); boolean success = false; @@ -125,7 +125,7 @@ public class BlockTreeTermsReader extends FieldsProducer { try { readHeader(in); if (indexDivisor != -1) { - indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION), + indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION), ioContext); readIndexHeader(indexIn); } @@ -206,14 +206,9 @@ public class BlockTreeTermsReader extends FieldsProducer { } } - public static void files(Directory dir, SegmentInfo segmentInfo, int codecID, Collection files) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, BlockTreeTermsWriter.TERMS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION)); - } - - public static void getExtensions(Collection extensions) { - extensions.add(BlockTreeTermsWriter.TERMS_EXTENSION); - extensions.add(BlockTreeTermsWriter.TERMS_INDEX_EXTENSION); + public static void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTreeTermsWriter.TERMS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION)); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java index bee12eb..59d96d9 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java @@ -145,7 +145,7 @@ public class BlockTreeTermsWriter extends FieldsConsumer { throw new IllegalArgumentException("maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock); } - final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_EXTENSION); out = state.directory.createOutput(termsFileName, state.context); boolean success = false; IndexOutput indexOut = null; @@ -157,7 +157,7 @@ public class BlockTreeTermsWriter extends FieldsConsumer { //DEBUG = state.segmentName.equals("_4a"); - final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); + final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_INDEX_EXTENSION); indexOut = state.directory.createOutput(termsIndexFileName, state.context); writeIndexHeader(indexOut); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/Codec.java b/lucene/src/java/org/apache/lucene/index/codecs/Codec.java dissimilarity index 62% index a5ab3cb..e25ca6e 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/Codec.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/Codec.java @@ -?,? +1,88 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.util.NamedSPILoader; +import org.apache.lucene.store.Directory; + +/** + * Encodes/decodes an inverted index segment + */ +public abstract class Codec implements NamedSPILoader.NamedSPI { + + private static final NamedSPILoader loader = + new NamedSPILoader(Codec.class); + + private final String name; + + public Codec(String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + postingsFormat().files(dir, info, "", files); + //TODO: not yet fieldsFormat().files(dir, info, files); + docValuesFormat().files(dir, info, files); + } + + /** Encodes/decodes postings */ + public abstract PostingsFormat postingsFormat(); + + /** Encodes/decodes docvalues */ + public abstract DocValuesFormat docValuesFormat(); + + /** Encodes/decodes stored fields, term vectors, fieldinfos */ + public abstract FieldsFormat fieldsFormat(); + + public abstract SegmentInfosFormat segmentInfosFormat(); + + /** looks up a codec by name */ + public static Codec forName(String name) { + return loader.lookup(name); + } + + /** returns a list of all available codec names */ + public static Set availableCodecs() { + return loader.availableServices(); + } + + private static Codec defaultCodec = Codec.forName("Lucene40"); + + // TODO: should we use this, or maybe a system property is better? + public static Codec getDefault() { + return defaultCodec; + } + + public static void setDefault(Codec codec) { + defaultCodec = codec; + } + + @Override + public String toString() { + return name; + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java b/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java deleted file mode 100644 index 086fd7b..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java b/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java deleted file mode 100644 index 74d547b..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java index 114b029..56de92f 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java @@ -25,7 +25,6 @@ import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.codecs.DocValuesWriterBase; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; @@ -34,42 +33,45 @@ import org.apache.lucene.store.Directory; * @lucene.experimental */ public class DefaultDocValuesConsumer extends DocValuesWriterBase { - private final Directory directory; + private final Directory mainDirectory; + private Directory directory; + + final static String DOC_VALUES_SEGMENT_SUFFIX = "dv"; public DefaultDocValuesConsumer(PerDocWriteState state) throws IOException { super(state); + mainDirectory = state.directory; //TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files? - this.directory = new CompoundFileDirectory(state.directory, - IndexFileNames.segmentFileName(state.segmentName, state.codecId, - IndexFileNames.COMPOUND_FILE_EXTENSION), state.context, true); } @Override - protected Directory getDirectory() { + protected Directory getDirectory() throws IOException { + // lazy init + if (directory == null) { + directory = new CompoundFileDirectory(mainDirectory, + IndexFileNames.segmentFileName(segmentName, DOC_VALUES_SEGMENT_SUFFIX, + IndexFileNames.COMPOUND_FILE_EXTENSION), context, true); + } return directory; } @Override public void close() throws IOException { - this.directory.close(); + if (directory != null) { + directory.close(); + } } - @SuppressWarnings("fallthrough") - public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { + public static void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException { FieldInfos fieldInfos = segmentInfo.getFieldInfos(); for (FieldInfo fieldInfo : fieldInfos) { - if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); - assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); - assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); - return; + if (fieldInfo.hasDocValues()) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_EXTENSION)); + break; } } } - - public static void getExtensions(Set extensions) { - extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION); - extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION); - } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java new file mode 100644 index 0000000..e35563a --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java @@ -0,0 +1,44 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.PerDocWriteState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.store.Directory; + +public class DefaultDocValuesFormat extends DocValuesFormat { + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new DefaultDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new DefaultDocValuesProducer(state); + } + + @Override + public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + DefaultDocValuesConsumer.files(dir, info, files); + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java index 6a903a0..d1fee75 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java @@ -26,7 +26,6 @@ import java.util.TreeMap; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.DocValuesReaderBase; import org.apache.lucene.index.values.IndexDocValues; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; @@ -37,7 +36,7 @@ import org.apache.lucene.util.IOUtils; * @lucene.experimental */ public class DefaultDocValuesProducer extends DocValuesReaderBase { - protected final TreeMap docValues; + protected final TreeMap docValues; private final Directory cfs; /** @@ -45,10 +44,16 @@ public class DefaultDocValuesProducer extends DocValuesReaderBase { * {@link IndexDocValues} instances for this segment and codec. */ public DefaultDocValuesProducer(SegmentReadState state) throws IOException { - cfs = new CompoundFileDirectory(state.dir, - IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), - state.context, false); - docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.codecId, state.context); + if (state.fieldInfos.anyDocValuesFields()) { + cfs = new CompoundFileDirectory(state.dir, + IndexFileNames.segmentFileName(state.segmentInfo.name, + DefaultDocValuesConsumer.DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_EXTENSION), + state.context, false); + docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.context); + } else { + cfs = null; + docValues = new TreeMap(); + } } @Override @@ -58,8 +63,12 @@ public class DefaultDocValuesProducer extends DocValuesReaderBase { @Override protected void closeInternal(Collection closeables) throws IOException { - final ArrayList list = new ArrayList(closeables); - list.add(cfs); - IOUtils.close(list); + if (cfs != null) { + final ArrayList list = new ArrayList(closeables); + list.add(cfs); + IOUtils.close(list); + } else { + IOUtils.close(closeables); + } } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java new file mode 100644 index 0000000..aaaa169 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java @@ -0,0 +1,48 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +/** @lucene.experimental */ +public class DefaultFieldsFormat extends FieldsFormat { + + @Override + public FieldsReader fieldsReader(Directory directory, String segment, + FieldInfos fn, IOContext context, int docStoreOffset, int size) + throws IOException { + return new DefaultFieldsReader(directory, segment, fn, context, docStoreOffset, size); + } + + @Override + public FieldsWriter fieldsWriter(Directory directory, String segment, + IOContext context) throws IOException { + return new DefaultFieldsWriter(directory, segment, context); + } + + @Override + public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + // TODO! + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java index cf0bbb2..eda8b0b 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java @@ -82,7 +82,7 @@ public final class DefaultFieldsReader extends FieldsReader implements Cloneable /** Verifies that the code version which wrote the segment is supported. */ public static void checkCodeVersion(Directory dir, String segment) throws IOException { - final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); + final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_INDEX_EXTENSION); IndexInput idxStream = dir.openInput(indexStreamFN, IOContext.DEFAULT); try { @@ -121,8 +121,8 @@ public final class DefaultFieldsReader extends FieldsReader implements Cloneable try { fieldInfos = fn; - cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); - final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); + cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_EXTENSION), context); + final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_INDEX_EXTENSION); cloneableIndexStream = d.openInput(indexStreamFN, context); format = cloneableIndexStream.readInt(); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java index 7d4503b..ec0fa6b 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java @@ -61,6 +61,10 @@ public final class DefaultFieldsWriter extends FieldsWriter { // when removing support for old versions, leave the last supported version here static final int FORMAT_MINIMUM = FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS; + // TODO: remove from IndexFileNames + public static final String FIELDS_EXTENSION = IndexFileNames.FIELDS_EXTENSION; + public static final String FIELDS_INDEX_EXTENSION = IndexFileNames.FIELDS_INDEX_EXTENSION; + // If null - we were supplied with streams, if notnull - we manage them ourselves private Directory directory; private String segment; @@ -73,8 +77,8 @@ public final class DefaultFieldsWriter extends FieldsWriter { boolean success = false; try { - fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); - indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION), context); + fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context); + indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION), context); fieldsStream.writeInt(FORMAT_CURRENT); indexStream.writeInt(FORMAT_CURRENT); @@ -129,11 +133,11 @@ public final class DefaultFieldsWriter extends FieldsWriter { } catch (IOException ignored) { } try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION)); + directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION)); } catch (IOException ignored) { } try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); + directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION)); } catch (IOException ignored) { } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosFormat.java new file mode 100644 index 0000000..96bdbc1 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosFormat.java @@ -0,0 +1,36 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @lucene.experimental + */ +public class DefaultSegmentInfosFormat extends SegmentInfosFormat { + private final SegmentInfosReader reader = new DefaultSegmentInfosReader(); + private final SegmentInfosWriter writer = new DefaultSegmentInfosWriter(); + + @Override + public SegmentInfosReader getSegmentInfosReader() { + return reader; + } + + @Override + public SegmentInfosWriter getSegmentInfosWriter() { + return writer; + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java dissimilarity index 70% index b8f1bee..6518f62 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java @@ -?,? +1,93 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.ChecksumIndexInput; +import org.apache.lucene.store.CompoundFileDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +/** + * Default implementation of {@link SegmentInfosReader}. + * @lucene.experimental + */ +public class DefaultSegmentInfosReader extends SegmentInfosReader { + + // TODO: shove all backwards code to preflex! + // this is a little tricky, because of IR.commit(), two options: + // 1. PreFlex writes 4.x SIS format, but reads both 3.x and 4.x + // (and maybe RW always only writes the 3.x one? for that to work well, + // we have to move .fnx file to codec too, not too bad but more work). + // or we just have crappier RW testing like today. + // 2. PreFlex writes 3.x SIS format, and only reads 3.x + // (in this case we have to move .fnx file to codec as well) + @Override + public void read(Directory directory, String segmentsFileName, ChecksumIndexInput input, SegmentInfos infos, IOContext context) throws IOException { + infos.version = input.readLong(); // read version + infos.counter = input.readInt(); // read counter + final int format = infos.getFormat(); + if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { + infos.setGlobalFieldMapVersion(input.readLong()); + } + for (int i = input.readInt(); i > 0; i--) { // read segmentInfos + SegmentInfo si = new SegmentInfo(directory, format, input); + if (si.getVersion() == null) { + // Could be a 3.0 - try to open the doc stores - if it fails, it's a + // 2.x segment, and an IndexFormatTooOldException will be thrown, + // which is what we want. + Directory dir = directory; + if (si.getDocStoreOffset() != -1) { + if (si.getDocStoreIsCompoundFile()) { + dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName( + si.getDocStoreSegment(), "", + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false); + } + } else if (si.getUseCompoundFile()) { + dir = new CompoundFileDirectory(dir,IndexFileNames.segmentFileName( + si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); + } + + try { + DefaultFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment()); + } finally { + // If we opened the directory, close it + if (dir != directory) dir.close(); + } + + // Above call succeeded, so it's a 3.0 segment. Upgrade it so the next + // time the segment is read, its version won't be null and we won't + // need to open FieldsReader every time for each such segment. + si.setVersion("3.0"); + } else if (si.getVersion().equals("2.x")) { + // If it's a 3x index touched by 3.1+ code, then segments record their + // version, whether they are 2.x ones or not. We detect that and throw + // appropriate exception. + throw new IndexFormatTooOldException(si.name, si.getVersion()); + } + infos.add(si); + } + + infos.userData = input.readStringStringMap(); + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java index e029262..cf69f56 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java @@ -56,12 +56,13 @@ public class DefaultSegmentInfosWriter extends SegmentInfosWriter { public static final int FORMAT_MINIMUM = FORMAT_DIAGNOSTICS; @Override - public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos, IOContext context) + public IndexOutput writeInfos(Directory dir, String segmentFileName, String codecID, SegmentInfos infos, IOContext context) throws IOException { IndexOutput out = createOutput(dir, segmentFileName, new IOContext(new FlushInfo(infos.size(), infos.totalDocCount()))); boolean success = false; try { out.writeInt(FORMAT_CURRENT); // write FORMAT + out.writeString(codecID); // write codecID out.writeLong(infos.version); out.writeInt(infos.counter); // write counter out.writeLong(infos.getGlobalFieldMapVersion()); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java new file mode 100644 index 0000000..2bfc466 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java @@ -0,0 +1,32 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.PerDocWriteState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.store.Directory; + +public abstract class DocValuesFormat { + public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException; + public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException; + public abstract void files(Directory dir, SegmentInfo info, Set files) throws IOException; +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java index 568ee4e..e545d4a 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java @@ -65,19 +65,19 @@ public abstract class DocValuesReaderBase extends PerDocValues { // Only opens files... doesn't actually load any values protected TreeMap load(FieldInfos fieldInfos, - String segment, int docCount, Directory dir, int codecId, IOContext context) + String segment, int docCount, Directory dir, IOContext context) throws IOException { TreeMap values = new TreeMap(); boolean success = false; try { for (FieldInfo fieldInfo : fieldInfos) { - if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) { + if (fieldInfo.hasDocValues()) { final String field = fieldInfo.name; // TODO can we have a compound file per segment and codec for // docvalues? final String id = DefaultDocValuesConsumer.docValuesId(segment, - codecId, fieldInfo.number); + fieldInfo.number); values.put(field, loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), context)); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java index e328704..619b124 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java @@ -33,19 +33,19 @@ import org.apache.lucene.util.Counter; * @lucene.experimental */ public abstract class DocValuesWriterBase extends PerDocConsumer { - private final String segmentName; - private final int codecId; + protected final String segmentName; + protected final String segmentSuffix; private final Counter bytesUsed; - private final IOContext context; + protected final IOContext context; protected DocValuesWriterBase(PerDocWriteState state) { this.segmentName = state.segmentName; - this.codecId = state.codecId; + this.segmentSuffix = state.segmentSuffix; this.bytesUsed = state.bytesUsed; this.context = state.context; } - protected abstract Directory getDirectory(); + protected abstract Directory getDirectory() throws IOException; @Override public void close() throws IOException { @@ -54,12 +54,12 @@ public abstract class DocValuesWriterBase extends PerDocConsumer { @Override public DocValuesConsumer addValuesField(FieldInfo field) throws IOException { return Writer.create(field.getDocValues(), - docValuesId(segmentName, codecId, field.number), + docValuesId(segmentName, field.number), getDirectory(), getComparator(), bytesUsed, context); } - public static String docValuesId(String segmentsName, int codecID, int fieldId) { - return segmentsName + "_" + codecID + "-" + fieldId; + public static String docValuesId(String segmentsName, int fieldId) { + return segmentsName + "_" + fieldId; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java new file mode 100644 index 0000000..ed4f413 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java @@ -0,0 +1,35 @@ +package org.apache.lucene.index.codecs; + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Controls the format of stored fields/termvectors/... + */ +public abstract class FieldsFormat { + public abstract FieldsReader fieldsReader(Directory directory, String segment, FieldInfos fn, IOContext context, int docStoreOffset, int size) throws IOException; + public abstract FieldsWriter fieldsWriter(Directory directory, String segment, IOContext context) throws IOException; + public abstract void files(Directory dir, SegmentInfo info, Set files) throws IOException; +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java b/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java index acb08ca..60f4bf5 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java @@ -34,23 +34,4 @@ import org.apache.lucene.index.Terms; public abstract class FieldsProducer extends Fields implements Closeable { public abstract void close() throws IOException; - - public static final FieldsProducer EMPTY = new FieldsProducer() { - - @Override - public Terms terms(String field) throws IOException { - return null; - } - - @Override - public FieldsEnum iterator() throws IOException { - return FieldsEnum.EMPTY; - } - - @Override - public void close() throws IOException { - - } - }; - } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java index 6ce2eeb..062179b 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java @@ -68,14 +68,14 @@ public class FixedGapTermsIndexReader extends TermsIndexReaderBase { // start of the field info data protected long dirOffset; - public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, int codecId, IOContext context) + public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, String segmentSuffix, IOContext context) throws IOException { this.termComp = termComp; assert indexDivisor == -1 || indexDivisor > 0; - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); + in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); boolean success = false; @@ -387,17 +387,8 @@ public class FixedGapTermsIndexReader extends TermsIndexReaderBase { } } - public static void files(Directory dir, SegmentInfo info, int id, Collection files) { - files.add(IndexFileNames.segmentFileName(info.name, id, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); - } - - public static void getIndexExtensions(Collection extensions) { - extensions.add(FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION); - } - - @Override - public void getExtensions(Collection extensions) { - getIndexExtensions(extensions); + public static void files(Directory dir, SegmentInfo info, String segmentSuffix, Collection files) { + files.add(IndexFileNames.segmentFileName(info.name, segmentSuffix, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java index 4d424a6..58ae9e8 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java @@ -56,7 +56,7 @@ public class FixedGapTermsIndexWriter extends TermsIndexWriterBase { private final FieldInfos fieldInfos; // unread public FixedGapTermsIndexWriter(SegmentWriteState state) throws IOException { - final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); + final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_INDEX_EXTENSION); termIndexInterval = state.termIndexInterval; out = state.directory.createOutput(indexFileName, state.context); boolean success = false; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java b/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java index 869a975..11de4d4 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java @@ -29,7 +29,7 @@ import org.apache.lucene.index.values.ValueType; * this convert field values into a Codec specific format during indexing. *

* The {@link PerDocConsumer} API is accessible through the - * {@link Codec} - API providing per field consumers and producers for inverted + * {@link PostingsFormat} - API providing per field consumers and producers for inverted * data (terms, postings) as well as per-document data. * * @lucene.experimental diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java b/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java index 5c2b721..3420c70 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java @@ -28,7 +28,7 @@ import org.apache.lucene.index.values.IndexDocValues; * {@link PerDocConsumer} counterpart. *

* The {@link PerDocValues} API is accessible through the - * {@link Codec} - API providing per field consumers and producers for inverted + * {@link PostingsFormat} - API providing per field consumers and producers for inverted * data (terms, postings) as well as per-document data. * * @lucene.experimental diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PostingsBaseFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/PostingsBaseFormat.java new file mode 100644 index 0000000..fabb8dc --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/PostingsBaseFormat.java @@ -0,0 +1,55 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.store.Directory; + +/** + * Provides a {@link PostingsReaderBase} and {@link + * PostingsWriterBase}. + * + * @lucene.experimental */ + +// TODO: find a better name; this defines the API that the +// terms dict impls use to talk to a postings impl. +// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer + +// can we clean this up and do this some other way? +// refactor some of these classes and use covariant return? +public abstract class PostingsBaseFormat { + + /** Unique name that's used to retrieve this codec when + * reading the index */ + public final String name; + + protected PostingsBaseFormat(String name) { + this.name = name; + } + + public abstract PostingsReaderBase postingsReaderBase(SegmentReadState state) throws IOException; + + public abstract PostingsWriterBase postingsWriterBase(SegmentWriteState state) throws IOException; + + public abstract void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException; +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java new file mode 100644 index 0000000..407092d --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java @@ -0,0 +1,84 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.util.NamedSPILoader; + +import org.apache.lucene.store.Directory; + +/** @lucene.experimental */ +public abstract class PostingsFormat implements NamedSPILoader.NamedSPI { + + private static final NamedSPILoader loader = + new NamedSPILoader(PostingsFormat.class); + + public static final PostingsFormat[] EMPTY = new PostingsFormat[0]; + /** Unique name that's used to retrieve this format when + * reading the index. + */ + private final String name; + + protected PostingsFormat(String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + /** Writes a new segment */ + public abstract FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException; + + /** Reads a segment. NOTE: by the time this call + * returns, it must hold open any files it will need to + * use; else, those files may be deleted. */ + public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException; + + /** + * Gathers files associated with this segment + * + * @param dir the {@link Directory} this segment was written to + * @param segmentInfo the {@link SegmentInfo} for this segment + * @param segmentSuffix the format's suffix within this segment + * @param files the of files to add the codec files to. + */ + public abstract void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException; + + @Override + public String toString() { + return "PostingsFormat(name=" + name + ")"; + } + + /** looks up a format by name */ + public static PostingsFormat forName(String name) { + return loader.lookup(name); + } + + /** returns a list of all available format names */ + public static Set availablePostingsFormats() { + return loader.availableServices(); + } + +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java b/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java index 05aec94..54253b3 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java @@ -36,6 +36,9 @@ import org.apache.lucene.util.Bits; * time. * @lucene.experimental */ +// TODO: find a better name; this defines the API that the +// terms dict impls use to talk to a postings impl. +// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer public abstract class PostingsReaderBase implements Closeable { public abstract void init(IndexInput termsIn) throws IOException; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java b/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java index 5e6ea14..b456615 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java @@ -27,6 +27,9 @@ import org.apache.lucene.index.FieldInfo; * @lucene.experimental */ +// TODO: find a better name; this defines the API that the +// terms dict impls use to talk to a postings impl. +// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer public abstract class PostingsWriterBase extends PostingsConsumer implements Closeable { public abstract void start(IndexOutput termsOut) throws IOException; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosFormat.java new file mode 100644 index 0000000..1632ce1 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosFormat.java @@ -0,0 +1,37 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Expert: Controls the format of the segments file. + * Note, this isn't a per-segment file, if you change the format, other versions + * of lucene won't be able to read it, yackedy schmackedy + * + * @lucene.experimental + */ +// TODO: would be great to handle this situation better. +// ideally a custom implementation could implement two-phase commit differently, +// (e.g. atomic rename), and ideally all versions of lucene could still read it. +// but this is just reflecting reality as it is today... +// +// also, perhaps the name should change (to cover all global files like .fnx?) +// then again, maybe we can just remove that file... +public abstract class SegmentInfosFormat { + public abstract SegmentInfosReader getSegmentInfosReader(); + public abstract SegmentInfosWriter getSegmentInfosWriter(); +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java index 69b6804..e3f8a7c 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java @@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs; import java.io.IOException; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -33,9 +34,9 @@ public abstract class SegmentInfosReader { * Read {@link SegmentInfos} data from a directory. * @param directory directory to read from * @param segmentsFileName name of the "segments_N" file - * @param codecs current codecs + * @param header input of "segments_N" file after reading preamble * @param infos empty instance to be populated with data * @throws IOException */ - public abstract void read(Directory directory, String segmentsFileName, CodecProvider codecs, SegmentInfos infos, IOContext context) throws IOException; + public abstract void read(Directory directory, String segmentsFileName, ChecksumIndexInput header, SegmentInfos infos, IOContext context) throws IOException; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java index 9c79edf..d392b87 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java @@ -42,13 +42,13 @@ public abstract class SegmentInfosWriter { * phase commit" operations as described above. * @throws IOException */ - public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, SegmentInfos infos, IOContext context) throws IOException; + public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, String codecID, SegmentInfos infos, IOContext context) throws IOException; /** * First phase of the two-phase commit - ensure that all output can be * successfully written out. * @param out an instance of {@link IndexOutput} returned from a previous - * call to {@link #writeInfos(Directory, String, SegmentInfos, IOContext)}. + * call to {@link #writeInfos(Directory, String, String, SegmentInfos, IOContext)}. * @throws IOException */ public abstract void prepareCommit(IndexOutput out) throws IOException; @@ -57,7 +57,7 @@ public abstract class SegmentInfosWriter { * Second phase of the two-phase commit. In this step the output should be * finalized and closed. * @param out an instance of {@link IndexOutput} returned from a previous - * call to {@link #writeInfos(Directory, String, SegmentInfos, IOContext)}. + * call to {@link #writeInfos(Directory, String, String, SegmentInfos, IOContext)}. * @throws IOException */ public abstract void finishCommit(IndexOutput out) throws IOException; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexReaderBase.java b/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexReaderBase.java index 9123ff3..d66999a 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexReaderBase.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/TermsIndexReaderBase.java @@ -45,8 +45,6 @@ public abstract class TermsIndexReaderBase implements Closeable { public abstract void close() throws IOException; - public abstract void getExtensions(Collection extensions); - public abstract boolean supportsOrd(); public abstract int getDivisor(); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java index f9d1552..de80e6d 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java @@ -57,9 +57,9 @@ public class VariableGapTermsIndexReader extends TermsIndexReaderBase { protected long dirOffset; final String segment; - public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, int codecId, IOContext context) + public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, String segmentSuffix, IOContext context) throws IOException { - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true)); + in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true)); this.segment = segment; boolean success = false; assert indexDivisor == -1 || indexDivisor > 0; @@ -215,17 +215,8 @@ public class VariableGapTermsIndexReader extends TermsIndexReaderBase { } } - public static void files(Directory dir, SegmentInfo info, int id, Collection files) { - files.add(IndexFileNames.segmentFileName(info.name, id, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); - } - - public static void getIndexExtensions(Collection extensions) { - extensions.add(VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION); - } - - @Override - public void getExtensions(Collection extensions) { - getIndexExtensions(extensions); + public static void files(Directory dir, SegmentInfo info, String segmentSuffix, Collection files) { + files.add(IndexFileNames.segmentFileName(info.name, segmentSuffix, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java index 221451f..9964609 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java @@ -158,7 +158,7 @@ public class VariableGapTermsIndexWriter extends TermsIndexWriterBase { // in the extremes. public VariableGapTermsIndexWriter(SegmentWriteState state, IndexTermSelector policy) throws IOException { - final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); + final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_INDEX_EXTENSION); out = state.directory.createOutput(indexFileName, state.context); boolean success = false; try { diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java new file mode 100644 index 0000000..ac4e8ef --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java @@ -0,0 +1,90 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.PerDocWriteState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultFieldsFormat; +import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; +import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldsFormat; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.SegmentInfosFormat; +import org.apache.lucene.store.Directory; + +/** + * Supports the Lucene 3.x index format (readonly) + */ +public class Lucene3xCodec extends Codec { + public Lucene3xCodec() { + super("Lucene3x"); + } + + private final PostingsFormat postingsFormat = new Lucene3xPostingsFormat(); + + // TODO: this should really be a different impl + private final FieldsFormat fieldsFormat = new DefaultFieldsFormat(); + + // TODO: this should really be a different impl + // also if we want preflex to *really* be read-only it should throw exception for the writer? + // this way IR.commit fails on delete/undelete/setNorm/etc ? + private final SegmentInfosFormat infosFormat = new DefaultSegmentInfosFormat(); + + // 3.x doesn't support docvalues + private final DocValuesFormat docValuesFormat = new DocValuesFormat() { + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return null; + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return null; + } + + @Override + public void files(Directory dir, SegmentInfo info, Set files) throws IOException {} + }; + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + @Override + public FieldsFormat fieldsFormat() { + return fieldsFormat; + } + + @Override + public SegmentInfosFormat segmentInfosFormat() { + return infosFormat; + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java similarity index 97% rename from lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java index e68555e..194eb3c 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.preflex; +package org.apache.lucene.index.codecs.lucene3x; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -51,7 +51,7 @@ import org.apache.lucene.util.UnicodeUtil; * @deprecated (4.0) */ @Deprecated -public class PreFlexFields extends FieldsProducer { +public class Lucene3xFields extends FieldsProducer { private static final boolean DEBUG_SURROGATES = false; @@ -68,7 +68,7 @@ public class PreFlexFields extends FieldsProducer { private final IOContext context; private Directory cfsReader; - public PreFlexFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, IOContext context, int indexDivisor) + public Lucene3xFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, IOContext context, int indexDivisor) throws IOException { si = info; @@ -95,7 +95,7 @@ public class PreFlexFields extends FieldsProducer { // make sure that all index files have been read or are kept open // so that if an index update removes them we'll still have them - freqStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.FREQ_EXTENSION), context); + freqStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.FREQ_EXTENSION), context); boolean anyProx = false; for (FieldInfo fi : fieldInfos) { if (fi.isIndexed) { @@ -108,7 +108,7 @@ public class PreFlexFields extends FieldsProducer { } if (anyProx) { - proxStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.PROX_EXTENSION), context); + proxStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.PROX_EXTENSION), context); } else { proxStream = null; } @@ -136,16 +136,16 @@ public class PreFlexFields extends FieldsProducer { } static void files(Directory dir, SegmentInfo info, Collection files) throws IOException { - files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.TERMS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.TERMS_INDEX_EXTENSION)); - files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.FREQ_EXTENSION)); + files.add(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.TERMS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION)); + files.add(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.FREQ_EXTENSION)); if (info.getHasProx()) { // LUCENE-1739: for certain versions of 2.9-dev, // hasProx would be incorrectly computed during // indexing as true, and then stored into the segments // file, when it should have been false. So we do the // extra check, here: - final String prx = IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.PROX_EXTENSION); + final String prx = IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.PROX_EXTENSION); if (dir.fileExists(prx)) { files.add(prx); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xPostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xPostingsFormat.java new file mode 100644 index 0000000..23da306 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xPostingsFormat.java @@ -0,0 +1,73 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Set; +import java.io.IOException; + +import org.apache.lucene.store.Directory; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; + +/** Codec that reads the pre-flex-indexing postings + * format. It does not provide a writer because newly + * written segments should use StandardCodec. + * + * @deprecated (4.0) This is only used to read indexes created + * before 4.0. + * @lucene.experimental + */ +@Deprecated +public class Lucene3xPostingsFormat extends PostingsFormat { + + /** Extension of terms file */ + public static final String TERMS_EXTENSION = "tis"; + + /** Extension of terms index file */ + public static final String TERMS_INDEX_EXTENSION = "tii"; + + /** Extension of freq postings file */ + public static final String FREQ_EXTENSION = "frq"; + + /** Extension of prox postings file */ + public static final String PROX_EXTENSION = "prx"; + + public Lucene3xPostingsFormat() { + super("Lucene3x"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + throw new IllegalArgumentException("this codec can only be used for reading"); + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + return new Lucene3xFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor); + } + + @Override + public void files(Directory dir, SegmentInfo info, String segmentSuffix, Set files) throws IOException { + // preflex fields have no segmentSuffix - we ignore it here + Lucene3xFields.files(dir, info, files); + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java similarity index 98% rename from lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java index b140f61..5177b7d 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.preflex; +package org.apache.lucene.index.codecs.lucene3x; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -23,7 +23,7 @@ import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.standard.DefaultSkipListReader; +import org.apache.lucene.index.codecs.lucene40.DefaultSkipListReader; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.Bits; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermEnum.java similarity index 99% rename from lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermEnum.java index 179e946..1bb854f 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermEnum.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.preflex; +package org.apache.lucene.index.codecs.lucene3x; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermPositions.java similarity index 99% rename from lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermPositions.java index 882e784..5f6921c 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermPositions.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.preflex; +package org.apache.lucene.index.codecs.lucene3x; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermBuffer.java similarity index 98% rename from lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermBuffer.java index 9a51206..24c2089 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermBuffer.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.preflex; +package org.apache.lucene.index.codecs.lucene3x; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfo.java similarity index 97% rename from lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfo.java index 8f91569..23009cb 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfo.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.preflex; +package org.apache.lucene.index.codecs.lucene3x; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReader.java similarity index 97% rename from lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReader.java index c5192d7..ad6e2a0 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReader.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.preflex; +package org.apache.lucene.index.codecs.lucene3x; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -109,7 +109,7 @@ public final class TermInfosReader { segment = seg; fieldInfos = fis; - origEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_EXTENSION), + origEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), context), fieldInfos, false); size = origEnum.size; @@ -118,7 +118,7 @@ public final class TermInfosReader { // Load terms index totalIndexInterval = origEnum.indexInterval * indexDivisor; - final String indexFileName = IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION); + final String indexFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(indexFileName, context), fieldInfos, true); @@ -228,8 +228,8 @@ public final class TermInfosReader { // optimize sequential access: first try scanning cached enum w/o seeking if (enumerator.term() != null // term is at or past current - && ((enumerator.prev() != null && compareAsUTF16(term, enumerator.prev())> 0) - || compareAsUTF16(term, enumerator.term()) >= 0)) { + && ((enumerator.prev() != null && compareAsUTF16(term, enumerator.prev())> 0) + || compareAsUTF16(term, enumerator.term()) >= 0)) { int enumOffset = (int)(enumerator.position/totalIndexInterval)+1; if (indexLength == enumOffset // but before end of block || index.compareTo(term, enumOffset) < 0) { diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java similarity index 99% rename from lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java index d384ff9..6dd0398 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.preflex; +package org.apache.lucene.index.codecs.lucene3x; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/package.html b/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/package.html similarity index 100% rename from lucene/src/java/org/apache/lucene/index/codecs/preflex/package.html rename to lucene/src/java/org/apache/lucene/index/codecs/lucene3x/package.html diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListReader.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListReader.java similarity index 98% rename from lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListReader.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListReader.java index 32f52c8..17906d1 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListReader.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.standard; +package org.apache.lucene.index.codecs.lucene40; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListWriter.java similarity index 99% rename from lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListWriter.java index c2435c8..eb993ad 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListWriter.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.standard; +package org.apache.lucene.index.codecs.lucene40; /** * Licensed to the Apache Software Foundation (ASF) under one or more diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java new file mode 100644 index 0000000..668ffce --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java @@ -0,0 +1,85 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesFormat; +import org.apache.lucene.index.codecs.DefaultFieldsFormat; +import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; +import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldsFormat; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.SegmentInfosFormat; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; + +/** + * Implements the Lucene 4.0 index format, with configurable per-field postings formats + * and using {@link DefaultFieldsFormat} for stored fields and {@link + * DefaultDocValuesFormat} for doc values. + * + * @lucene.experimental + */ +// NOTE: if we make largish changes in a minor release, easier to just make Lucene42Codec or whatever +// if they are backwards compatible or smallish we can probably do the backwards in the postingsreader +// (it writes a minor version, etc). +public class Lucene40Codec extends Codec { + private final FieldsFormat fieldsFormat = new DefaultFieldsFormat(); + private final DocValuesFormat docValuesFormat = new DefaultDocValuesFormat(); + private final SegmentInfosFormat infosFormat = new DefaultSegmentInfosFormat(); + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return Lucene40Codec.this.getPostingsFormatForField(field); + } + }; + + public Lucene40Codec() { + super("Lucene40"); + } + + @Override + public FieldsFormat fieldsFormat() { + return fieldsFormat; + } + + @Override + public DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public SegmentInfosFormat segmentInfosFormat() { + return infosFormat; + } + + /** Returns the postings format that should be used for writing + * new segments of field. + * + * The default implementation always returns "Lucene40" + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene40"); +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsBaseFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsBaseFormat.java new file mode 100644 index 0000000..edd4a53 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsBaseFormat.java @@ -0,0 +1,58 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.PostingsBaseFormat; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.store.Directory; + +/** + * Provides a {@link PostingsReaderBase} and {@link + * PostingsWriterBase}. + * + * @lucene.experimental */ + +// TODO: should these also be named / looked up via SPI? +public final class Lucene40PostingsBaseFormat extends PostingsBaseFormat { + + public Lucene40PostingsBaseFormat() { + super("Lucene40"); + } + + @Override + public PostingsReaderBase postingsReaderBase(SegmentReadState state) throws IOException { + return new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + } + + @Override + public PostingsWriterBase postingsWriterBase(SegmentWriteState state) throws IOException { + return new Lucene40PostingsWriter(state); + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java new file mode 100644 index 0000000..9104acb --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java @@ -0,0 +1,118 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.BlockTreeTermsReader; +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.store.Directory; + +/** Default codec. + * @lucene.experimental */ + +// TODO: this class could be created by wrapping +// BlockTreeTermsDict around Lucene40PostingsBaseFormat; ie +// we should not duplicate the code from that class here: +public class Lucene40PostingsFormat extends PostingsFormat { + + private final int minBlockSize; + private final int maxBlockSize; + + public Lucene40PostingsFormat() { + this(BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); + } + + public Lucene40PostingsFormat(int minBlockSize, int maxBlockSize) { + super("Lucene40"); + this.minBlockSize = minBlockSize; + assert minBlockSize > 1; + this.maxBlockSize = maxBlockSize; + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docs = new Lucene40PostingsWriter(state); + + // TODO: should we make the terms index more easily + // pluggable? Ie so that this codec would record which + // index impl was used, and switch on loading? + // Or... you must make a new Codec for this? + boolean success = false; + try { + FieldsConsumer ret = new BlockTreeTermsWriter(state, docs, minBlockSize, maxBlockSize); + success = true; + return ret; + } finally { + if (!success) { + docs.close(); + } + } + } + + public final static int TERMS_CACHE_SIZE = 1024; + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase postings = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + + boolean success = false; + try { + FieldsProducer ret = new BlockTreeTermsReader( + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postings, + state.context, + state.segmentSuffix, + state.termsIndexDivisor); + success = true; + return ret; + } finally { + if (!success) { + postings.close(); + } + } + } + + /** Extension of freq postings file */ + static final String FREQ_EXTENSION = "frq"; + + /** Extension of prox postings file */ + static final String PROX_EXTENSION = "prx"; + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); + } + + @Override + public String toString() { + return getName() + "(minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")"; + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java similarity index 97% rename from lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java index 488c81f..5283c56 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.standard; +package org.apache.lucene.index.codecs.lucene40; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -42,7 +42,7 @@ import org.apache.lucene.util.CodecUtil; * postings format. * @lucene.experimental */ -public class StandardPostingsReader extends PostingsReaderBase { +public class Lucene40PostingsReader extends PostingsReaderBase { private final IndexInput freqIn; private final IndexInput proxIn; @@ -54,14 +54,14 @@ public class StandardPostingsReader extends PostingsReaderBase { // private String segment; - public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext ioContext, int codecId) throws IOException { - freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.FREQ_EXTENSION), + public Lucene40PostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext ioContext, String segmentSuffix) throws IOException { + freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION), ioContext); // this.segment = segmentInfo.name; if (segmentInfo.getHasProx()) { boolean success = false; try { - proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.PROX_EXTENSION), + proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION), ioContext); success = true; } finally { @@ -74,10 +74,10 @@ public class StandardPostingsReader extends PostingsReaderBase { } } - public static void files(Directory dir, SegmentInfo segmentInfo, int codecID, Collection files) throws IOException { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, StandardCodec.FREQ_EXTENSION)); + public static void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Collection files) throws IOException { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION)); if (segmentInfo.getHasProx()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, StandardCodec.PROX_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION)); } } @@ -85,8 +85,8 @@ public class StandardPostingsReader extends PostingsReaderBase { public void init(IndexInput termsIn) throws IOException { // Make sure we are talking to the matching past writer - CodecUtil.checkHeader(termsIn, StandardPostingsWriter.CODEC, - StandardPostingsWriter.VERSION_START, StandardPostingsWriter.VERSION_START); + CodecUtil.checkHeader(termsIn, Lucene40PostingsWriter.CODEC, + Lucene40PostingsWriter.VERSION_START, Lucene40PostingsWriter.VERSION_START); skipInterval = termsIn.readInt(); maxSkipLevels = termsIn.readInt(); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java similarity index 96% rename from lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java rename to lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java index 1a31693..ef93fc5 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.standard; +package org.apache.lucene.index.codecs.lucene40; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -38,8 +38,8 @@ import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CodecUtil; /** @lucene.experimental */ -public final class StandardPostingsWriter extends PostingsWriterBase { - final static String CODEC = "StandardPostingsWriter"; +public final class Lucene40PostingsWriter extends PostingsWriterBase { + final static String CODEC = "Lucene40PostingsWriter"; //private static boolean DEBUG = BlockTreeTermsWriter.DEBUG; @@ -81,21 +81,21 @@ public final class StandardPostingsWriter extends PostingsWriterBase { // private String segment; - public StandardPostingsWriter(SegmentWriteState state) throws IOException { + public Lucene40PostingsWriter(SegmentWriteState state) throws IOException { this(state, DEFAULT_SKIP_INTERVAL); } - public StandardPostingsWriter(SegmentWriteState state, int skipInterval) throws IOException { + public Lucene40PostingsWriter(SegmentWriteState state, int skipInterval) throws IOException { super(); this.skipInterval = skipInterval; this.skipMinimum = skipInterval; /* set to the same for now */ // this.segment = state.segmentName; - String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.FREQ_EXTENSION); + String fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION); freqOut = state.directory.createOutput(fileName, state.context); if (state.fieldInfos.hasProx()) { // At least one field does not omit TF, so create the // prox file - fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION); + fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION); proxOut = state.directory.createOutput(fileName, state.context); } else { // Every field omits TF so we will write no prox file diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/package.html b/lucene/src/java/org/apache/lucene/index/codecs/lucene40/package.html similarity index 100% rename from lucene/src/java/org/apache/lucene/index/codecs/standard/package.html rename to lucene/src/java/org/apache/lucene/index/codecs/lucene40/package.html diff --git a/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java b/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java similarity index 95% rename from lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java rename to lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java index 3eaa8ee..0b6c070 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java @@ -31,19 +31,14 @@ import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsConsumer; import org.apache.lucene.index.codecs.TermStats; import org.apache.lucene.index.codecs.TermsConsumer; @@ -81,9 +76,12 @@ import org.apache.lucene.util.fst.FST; * * @lucene.experimental */ -public class MemoryCodec extends Codec { +// TODO: Maybe name this 'Cached' or something to reflect +// the reality that it is actually written to disk, but +// loads itself in ram? +public class MemoryPostingsFormat extends PostingsFormat { - public MemoryCodec() { + public MemoryPostingsFormat() { super("Memory"); } @@ -245,7 +243,7 @@ public class MemoryCodec extends Codec { @Override public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - final String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, EXTENSION); + final String fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, EXTENSION); final IndexOutput out = state.directory.createOutput(fileName, state.context); return new FieldsConsumer() { @@ -737,7 +735,7 @@ public class MemoryCodec extends Codec { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, EXTENSION); + final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION); final IndexInput in = state.dir.openInput(fileName, IOContext.READONCE); final SortedMap fields = new TreeMap(); @@ -792,24 +790,7 @@ public class MemoryCodec extends Codec { } @Override - public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, EXTENSION)); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); - } - - @Override - public void getExtensions(Set extensions) { - extensions.add(EXTENSION); - DefaultDocValuesConsumer.getExtensions(extensions); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION)); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java new file mode 100644 index 0000000..98bb641 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java @@ -0,0 +1,341 @@ +package org.apache.lucene.index.codecs.perfield; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.TermsConsumer; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.util.IOUtils; + +/** + * Enables per field format support. + * + * @lucene.experimental + */ + +public abstract class PerFieldPostingsFormat extends PostingsFormat { + + public static final String PER_FIELD_EXTENSION = "per"; + public static final String PER_FIELD_NAME = "PerField40"; + + public static final int VERSION_START = 0; + public static final int VERSION_LATEST = VERSION_START; + + public PerFieldPostingsFormat() { + super(PER_FIELD_NAME); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) + throws IOException { + return new FieldsWriter(state); + } + + // NOTE: not private to avoid $accessN at runtime!! + static class FieldsConsumerAndID implements Closeable { + final FieldsConsumer fieldsConsumer; + final String segmentSuffix; + + public FieldsConsumerAndID(FieldsConsumer fieldsConsumer, String segmentSuffix) { + this.fieldsConsumer = fieldsConsumer; + this.segmentSuffix = segmentSuffix; + } + + @Override + public void close() throws IOException { + fieldsConsumer.close(); + } + }; + + private class FieldsWriter extends FieldsConsumer { + + private final Map formats = new IdentityHashMap(); + + /** Records all fields we wrote. */ + private final Map fieldToFormat = new HashMap(); + + private final SegmentWriteState segmentWriteState; + + public FieldsWriter(SegmentWriteState state) throws IOException { + segmentWriteState = state; + } + + @Override + public TermsConsumer addField(FieldInfo field) throws IOException { + final PostingsFormat format = getPostingsFormatForField(field.name); + if (format == null) { + throw new IllegalStateException("invalid null PostingsFormat for field=\"" + field.name + "\""); + } + + assert !fieldToFormat.containsKey(field.name); + fieldToFormat.put(field.name, format); + + FieldsConsumerAndID consumerAndId = formats.get(format); + if (consumerAndId == null) { + // First time we are seeing this format; assign + // next id and init it: + final String segmentSuffix = getFullSegmentSuffix(field.name, + segmentWriteState.segmentSuffix, + ""+formats.size()); + consumerAndId = new FieldsConsumerAndID(format.fieldsConsumer(new SegmentWriteState(segmentWriteState, segmentSuffix)), + segmentSuffix); + formats.put(format, consumerAndId); + } + + return consumerAndId.fieldsConsumer.addField(field); + } + + @Override + public void close() throws IOException { + + // Close all subs + IOUtils.close(formats.values()); + + // Write _X.per: maps field name -> format name and + // format name -> format id + final String mapFileName = IndexFileNames.segmentFileName(segmentWriteState.segmentName, segmentWriteState.segmentSuffix, PER_FIELD_EXTENSION); + final IndexOutput out = segmentWriteState.directory.createOutput(mapFileName, segmentWriteState.context); + boolean success = false; + try { + CodecUtil.writeHeader(out, PER_FIELD_NAME, VERSION_LATEST); + + // format name -> int id + out.writeVInt(formats.size()); + for(Map.Entry ent : formats.entrySet()) { + out.writeString(ent.getValue().segmentSuffix); + out.writeString(ent.getKey().getName()); + } + + // field name -> format name + out.writeVInt(fieldToFormat.size()); + for(Map.Entry ent : fieldToFormat.entrySet()) { + out.writeString(ent.getKey()); + out.writeString(ent.getValue().getName()); + } + + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + } else { + IOUtils.close(out); + } + } + } + } + + static String getFullSegmentSuffix(String fieldName, String outerSegmentSuffix, String segmentSuffix) { + if (outerSegmentSuffix.length() == 0) { + return segmentSuffix; + } else { + // TODO: support embedding; I think it should work but + // we need a test confirm to confirm + // return outerSegmentSuffix + "_" + segmentSuffix; + throw new IllegalStateException("cannot embed PerFieldPostingsFormat inside itself (field \"" + fieldName + "\" returned PerFieldPostingsFormat)"); + } + } + + private class FieldsReader extends FieldsProducer { + + private final Map fields = new TreeMap(); + private final Map formats = new IdentityHashMap(); + + public FieldsReader(final SegmentReadState readState) throws IOException { + + // Read _X.per and init each format: + boolean success = false; + try { + new VisitPerFieldFile(readState.dir, readState.segmentInfo.name, readState.segmentSuffix) { + @Override + protected void visitOneFormat(String segmentSuffix, PostingsFormat postingsFormat) throws IOException { + formats.put(postingsFormat, postingsFormat.fieldsProducer(new SegmentReadState(readState, segmentSuffix))); + } + + @Override + protected void visitOneField(String fieldName, PostingsFormat postingsFormat) throws IOException { + assert formats.containsKey(postingsFormat); + fields.put(fieldName, formats.get(postingsFormat)); + } + }; + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(formats.values()); + } + } + } + + private final class FieldsIterator extends FieldsEnum { + private final Iterator it; + private String current; + + public FieldsIterator() { + it = fields.keySet().iterator(); + } + + @Override + public String next() { + if (it.hasNext()) { + current = it.next(); + } else { + current = null; + } + + return current; + } + + @Override + public TermsEnum terms() throws IOException { + final Terms terms = fields.get(current).terms(current); + if (terms != null) { + return terms.iterator(); + } else { + return TermsEnum.EMPTY; + } + } + } + + @Override + public FieldsEnum iterator() throws IOException { + return new FieldsIterator(); + } + + @Override + public Terms terms(String field) throws IOException { + FieldsProducer fieldsProducer = fields.get(field); + return fieldsProducer == null ? null : fieldsProducer.terms(field); + } + + @Override + public void close() throws IOException { + IOUtils.close(formats.values()); + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) + throws IOException { + return new FieldsReader(state); + } + + private abstract class VisitPerFieldFile { + public VisitPerFieldFile(Directory dir, String segmentName, String outerSegmentSuffix) throws IOException { + final String mapFileName = IndexFileNames.segmentFileName(segmentName, outerSegmentSuffix, PER_FIELD_EXTENSION); + final IndexInput in = dir.openInput(mapFileName, IOContext.READONCE); + boolean success = false; + try { + CodecUtil.checkHeader(in, PER_FIELD_NAME, VERSION_START, VERSION_LATEST); + + // Read format name -> format id + final int formatCount = in.readVInt(); + for(int formatIDX=0;formatIDX " + postingsFormat); + if (postingsFormat == null) { + throw new IllegalStateException("unable to lookup PostingsFormat for name=\"" + formatName + "\": got null"); + } + + // Better be defined, because it was defined + // during indexing: + visitOneFormat(segmentSuffix, postingsFormat); + } + + // Read field name -> format name + final int fieldCount = in.readVInt(); + for(int fieldIDX=0;fieldIDX files) + throws IOException { + + final String mapFileName = IndexFileNames.segmentFileName(info.name, segmentSuffix, PER_FIELD_EXTENSION); + files.add(mapFileName); + + try { + new VisitPerFieldFile(dir, info.name, segmentSuffix) { + @Override + protected void visitOneFormat(String segmentSuffix, PostingsFormat format) throws IOException { + format.files(dir, info, segmentSuffix, files); + } + + @Override + protected void visitOneField(String field, PostingsFormat format) { + } + }; + } catch (FileNotFoundException fnfe) { + // TODO: this is somewhat shady... if we can't open + // the .per file then most likely someone is calling + // .files() after this segment was deleted, so, they + // wouldn't be able to do anything with the files even + // if we could return them, so we don't add any files + // in this case. + } + } + + // NOTE: only called during writing; for reading we read + // all we need from the index (ie we save the field -> + // format mapping) + public abstract PostingsFormat getPostingsFormatForField(String field); +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java deleted file mode 100644 index 257c3ea..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/Pulsing40PostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/Pulsing40PostingsFormat.java new file mode 100644 index 0000000..6d20af0 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/Pulsing40PostingsFormat.java @@ -0,0 +1,42 @@ +package org.apache.lucene.index.codecs.pulsing; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsBaseFormat; + +/** + * @lucene.experimental + */ +public class Pulsing40PostingsFormat extends PulsingPostingsFormat { + + /** Inlines docFreq=1 terms, otherwise uses the normal "Lucene40" format. */ + public Pulsing40PostingsFormat() { + this(1); + } + + /** Inlines docFreq=freqCutoff terms, otherwise uses the normal "Lucene40" format. */ + public Pulsing40PostingsFormat(int freqCutoff) { + this(freqCutoff, BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); + } + + /** Inlines docFreq=freqCutoff terms, otherwise uses the normal "Lucene40" format. */ + public Pulsing40PostingsFormat(int freqCutoff, int minBlockSize, int maxBlockSize) { + super("Pulsing40", new Lucene40PostingsBaseFormat(), freqCutoff, minBlockSize, maxBlockSize); + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java deleted file mode 100644 index 04d1a14..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java new file mode 100644 index 0000000..0c83f20 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java @@ -0,0 +1,122 @@ +package org.apache.lucene.index.codecs.pulsing; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.BlockTreeTermsReader; +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; +import org.apache.lucene.index.codecs.PostingsBaseFormat; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.store.Directory; + +/** This postings format "inlines" the postings for terms that have + * low docFreq. It wraps another postings format, which is used for + * writing the non-inlined terms. + * + * @lucene.experimental */ + +public abstract class PulsingPostingsFormat extends PostingsFormat { + + private final int freqCutoff; + private final int minBlockSize; + private final int maxBlockSize; + private final PostingsBaseFormat wrappedPostingsBaseFormat; + + public PulsingPostingsFormat(String name, PostingsBaseFormat wrappedPostingsBaseFormat, int freqCutoff) { + this(name, wrappedPostingsBaseFormat, freqCutoff, BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); + } + + /** Terms with freq <= freqCutoff are inlined into terms + * dict. */ + public PulsingPostingsFormat(String name, PostingsBaseFormat wrappedPostingsBaseFormat, int freqCutoff, int minBlockSize, int maxBlockSize) { + super(name); + this.freqCutoff = freqCutoff; + this.minBlockSize = minBlockSize; + assert minBlockSize > 1; + this.maxBlockSize = maxBlockSize; + this.wrappedPostingsBaseFormat = wrappedPostingsBaseFormat; + } + + @Override + public String toString() { + return getName() + "(freqCutoff=" + freqCutoff + " minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")"; + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docsWriter = wrappedPostingsBaseFormat.postingsWriterBase(state); + + // Terms that have <= freqCutoff number of docs are + // "pulsed" (inlined): + PostingsWriterBase pulsingWriter = new PulsingPostingsWriter(freqCutoff, docsWriter); + + // Terms dict + boolean success = false; + try { + FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, minBlockSize, maxBlockSize); + success = true; + return ret; + } finally { + if (!success) { + pulsingWriter.close(); + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + + PostingsReaderBase docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state); + PostingsReaderBase pulsingReader = new PulsingPostingsReader(docsReader); + + boolean success = false; + try { + FieldsProducer ret = new BlockTreeTermsReader( + state.dir, state.fieldInfos, state.segmentInfo.name, + pulsingReader, + state.context, + state.segmentSuffix, + state.termsIndexDivisor); + success = true; + return ret; + } finally { + if (!success) { + pulsingReader.close(); + } + } + } + + public int getFreqCutoff() { + return freqCutoff; + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + wrappedPostingsBaseFormat.files(dir, segmentInfo, segmentSuffix, files); + BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java index 6083d29..8a387a2 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java @@ -47,12 +47,12 @@ public class SepDocValuesConsumer extends DocValuesWriterBase { } @SuppressWarnings("fallthrough") - public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, + public static void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException { FieldInfos fieldInfos = segmentInfo.getFieldInfos(); for (FieldInfo fieldInfo : fieldInfos) { - if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) { - String filename = docValuesId(segmentInfo.name, codecId, fieldInfo.number); + if (fieldInfo.hasDocValues()) { + String filename = docValuesId(segmentInfo.name, fieldInfo.number); switch (fieldInfo.getDocValues()) { case BYTES_FIXED_DEREF: case BYTES_VAR_DEREF: @@ -83,9 +83,4 @@ public class SepDocValuesConsumer extends DocValuesWriterBase { } } } - - public static void getExtensions(Set extensions) { - extensions.add(Writer.DATA_EXTENSION); - extensions.add(Writer.INDEX_EXTENSION); - } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java index 71f42a7..88d9f1f 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java @@ -39,7 +39,7 @@ public class SepDocValuesProducer extends DocValuesReaderBase { * {@link IndexDocValues} instances for this segment and codec. */ public SepDocValuesProducer(SegmentReadState state) throws IOException { - docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, state.dir, state.codecId, state.context); + docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, state.dir, state.context); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java index 50f87f2..678d3c8 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java @@ -60,23 +60,23 @@ public class SepPostingsReader extends PostingsReaderBase { int maxSkipLevels; int skipMinimum; - public SepPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext context, IntStreamFactory intFactory, int codecId) throws IOException { + public SepPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext context, IntStreamFactory intFactory, String segmentSuffix) throws IOException { boolean success = false; try { - final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.DOC_EXTENSION); + final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.DOC_EXTENSION); docIn = intFactory.openInput(dir, docFileName, context); - skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.SKIP_EXTENSION), context); + skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.SKIP_EXTENSION), context); if (segmentInfo.getFieldInfos().hasFreq()) { - freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.FREQ_EXTENSION), context); + freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.FREQ_EXTENSION), context); } else { freqIn = null; } if (segmentInfo.getHasProx()) { - posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.POS_EXTENSION), context); - payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.PAYLOAD_EXTENSION), context); + posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.POS_EXTENSION), context); + payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.PAYLOAD_EXTENSION), context); } else { posIn = null; payloadIn = null; @@ -89,17 +89,17 @@ public class SepPostingsReader extends PostingsReaderBase { } } - public static void files(SegmentInfo segmentInfo, int codecId, Collection files) throws IOException { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.DOC_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.SKIP_EXTENSION)); + public static void files(SegmentInfo segmentInfo, String segmentSuffix, Collection files) throws IOException { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.DOC_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.SKIP_EXTENSION)); if (segmentInfo.getFieldInfos().hasFreq()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.FREQ_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.FREQ_EXTENSION)); } if (segmentInfo.getHasProx()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.POS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.PAYLOAD_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.POS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.PAYLOAD_EXTENSION)); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java index 05a5639..7031e5c 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java @@ -116,27 +116,27 @@ public final class SepPostingsWriter extends PostingsWriterBase { try { this.skipInterval = skipInterval; this.skipMinimum = skipInterval; /* set to the same for now */ - final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, DOC_EXTENSION); + final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, DOC_EXTENSION); docOut = factory.createOutput(state.directory, docFileName, state.context); docIndex = docOut.index(); if (state.fieldInfos.hasFreq()) { - final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, FREQ_EXTENSION); + final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, FREQ_EXTENSION); freqOut = factory.createOutput(state.directory, frqFileName, state.context); freqIndex = freqOut.index(); } if (state.fieldInfos.hasProx()) { - final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, POS_EXTENSION); + final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, POS_EXTENSION); posOut = factory.createOutput(state.directory, posFileName, state.context); posIndex = posOut.index(); // TODO: -- only if at least one field stores payloads? - final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, PAYLOAD_EXTENSION); + final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, PAYLOAD_EXTENSION); payloadOut = state.directory.createOutput(payloadFileName, state.context); } - final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SKIP_EXTENSION); + final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, SKIP_EXTENSION); skipOut = state.directory.createOutput(skipFileName, state.context); totalNumDocs = state.numDocs; @@ -391,12 +391,4 @@ public final class SepPostingsWriter extends PostingsWriterBase { public void close() throws IOException { IOUtils.close(docOut, skipOut, freqOut, posOut, payloadOut); } - - public static void getExtensions(Set extensions) { - extensions.add(DOC_EXTENSION); - extensions.add(FREQ_EXTENSION); - extensions.add(SKIP_EXTENSION); - extensions.add(POS_EXTENSION); - extensions.add(PAYLOAD_EXTENSION); - } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java deleted file mode 100644 index 93fac42..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java index 7db26cb..1c415f1 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java @@ -60,7 +60,7 @@ class SimpleTextFieldsReader extends FieldsProducer { final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD; public SimpleTextFieldsReader(SegmentReadState state) throws IOException { - in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, state.codecId), state.context); + in = state.dir.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context); fieldInfos = state.fieldInfos; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java index 9427da6..4deec0c 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java @@ -47,7 +47,7 @@ class SimpleTextFieldsWriter extends FieldsConsumer { final static BytesRef PAYLOAD = new BytesRef(" payload "); public SimpleTextFieldsWriter(SegmentWriteState state) throws IOException { - final String fileName = SimpleTextCodec.getPostingsFileName(state.segmentName, state.codecId); + final String fileName = SimpleTextPostingsFormat.getPostingsFileName(state.segmentName, state.segmentSuffix); out = state.directory.createOutput(fileName, state.context); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java new file mode 100644 index 0000000..6b47bf3 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java @@ -0,0 +1,67 @@ +package org.apache.lucene.index.codecs.simpletext; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.store.Directory; + +/** For debugging, curiosity, transparency only!! Do not + * use this codec in production. + * + *

This codec stores all postings data in a single + * human-readable text file (_N.pst). You can view this in + * any text editor, and even edit it to alter your index. + * + * @lucene.experimental */ +public class SimpleTextPostingsFormat extends PostingsFormat { + + public SimpleTextPostingsFormat() { + super("SimpleText"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + return new SimpleTextFieldsWriter(state); + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + return new SimpleTextFieldsReader(state); + } + + /** Extension of freq postings file */ + static final String POSTINGS_EXTENSION = "pst"; + + static String getPostingsFileName(String segment, String segmentSuffix) { + return IndexFileNames.segmentFileName(segment, segmentSuffix, POSTINGS_EXTENSION); + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + files.add(getPostingsFileName(segmentInfo.name, segmentSuffix)); + } +} diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java deleted file mode 100644 index a8a546a..0000000 diff --git a/lucene/src/java/org/apache/lucene/index/values/Bytes.java b/lucene/src/java/org/apache/lucene/index/values/Bytes.java index 9d942ca..aac08af 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Bytes.java +++ b/lucene/src/java/org/apache/lucene/index/values/Bytes.java @@ -61,6 +61,9 @@ import org.apache.lucene.util.packed.PackedInts; * @lucene.experimental */ public final class Bytes { + + static final String DV_SEGMENT_SUFFIX = "dv"; + // TODO - add bulk copy where possible private Bytes() { /* don't instantiate! */ } @@ -244,7 +247,7 @@ public final class Bytes { if (datOut == null) { boolean success = false; try { - datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", + datOut = dir.createOutput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, DATA_EXTENSION), context); CodecUtil.writeHeader(datOut, codecName, version); success = true; @@ -269,7 +272,7 @@ public final class Bytes { boolean success = false; try { if (idxOut == null) { - idxOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", + idxOut = dir.createOutput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, INDEX_EXTENSION), context); CodecUtil.writeHeader(idxOut, codecName, version); } @@ -307,10 +310,10 @@ public final class Bytes { @Override public void files(Collection files) throws IOException { assert datOut != null; - files.add(IndexFileNames.segmentFileName(id, "", DATA_EXTENSION)); + files.add(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, DATA_EXTENSION)); if (idxOut != null) { // called after flush - so this must be initialized // if needed or present - final String idxFile = IndexFileNames.segmentFileName(id, "", + final String idxFile = IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, INDEX_EXTENSION); files.add(idxFile); } @@ -334,11 +337,11 @@ public final class Bytes { IndexInput indexIn = null; boolean success = false; try { - dataIn = dir.openInput(IndexFileNames.segmentFileName(id, "", + dataIn = dir.openInput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, Writer.DATA_EXTENSION), context); version = CodecUtil.checkHeader(dataIn, codecName, maxVersion, maxVersion); if (doIndex) { - indexIn = dir.openInput(IndexFileNames.segmentFileName(id, "", + indexIn = dir.openInput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, Writer.INDEX_EXTENSION), context); final int version2 = CodecUtil.checkHeader(indexIn, codecName, maxVersion, maxVersion); @@ -494,8 +497,7 @@ public final class Bytes { protected void releaseResources() { hash.close(); - bytesUsed - .addAndGet((-docToEntry.length) * RamUsageEstimator.NUM_BYTES_INT); + bytesUsed.addAndGet((-docToEntry.length) * RamUsageEstimator.NUM_BYTES_INT); docToEntry = null; } diff --git a/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java b/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java index 1e1b9d5..6d00aaa 100644 --- a/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java +++ b/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java @@ -24,8 +24,7 @@ import org.apache.lucene.document.IndexDocValuesField; import org.apache.lucene.index.Fields; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.util.BytesRef; /** @@ -40,13 +39,11 @@ import org.apache.lucene.util.BytesRef; * performance {@link IndexDocValues} should be consumed per-segment just like * IndexReader. *

- * {@link IndexDocValues} are fully integrated into the {@link Codec} API. - * Custom implementations can be exposed on a per field basis via - * {@link CodecProvider}. + * {@link IndexDocValues} are fully integrated into the {@link DocValuesFormat} API. * * @see ValueType for limitations and default implementation documentation * @see IndexDocValuesField for adding values to the index - * @see Codec#docsConsumer(org.apache.lucene.index.PerDocWriteState) for + * @see DocValuesFormat#docsConsumer(org.apache.lucene.index.PerDocWriteState) for * customization * @lucene.experimental */ diff --git a/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java b/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java index 293558e..97470ef 100644 --- a/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java +++ b/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java @@ -156,7 +156,7 @@ class PackedIntValues { protected PackedIntsReader(Directory dir, String id, int numDocs, IOContext context) throws IOException { datIn = dir.openInput( - IndexFileNames.segmentFileName(id, "", Writer.DATA_EXTENSION), + IndexFileNames.segmentFileName(id, Bytes.DV_SEGMENT_SUFFIX, Writer.DATA_EXTENSION), context); this.numDocs = numDocs; boolean success = false; diff --git a/lucene/src/java/org/apache/lucene/index/values/ValueType.java b/lucene/src/java/org/apache/lucene/index/values/ValueType.java index 8e4a133..5a267eb 100644 --- a/lucene/src/java/org/apache/lucene/index/values/ValueType.java +++ b/lucene/src/java/org/apache/lucene/index/values/ValueType.java @@ -17,7 +17,7 @@ package org.apache.lucene.index.values; * limitations under the License. */ -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.util.BytesRef; @@ -27,7 +27,7 @@ import org.apache.lucene.util.packed.PackedInts; * ValueType specifies the {@link IndexDocValues} type for a * certain field. A ValueType only defines the data type for a field * while the actual implementation used to encode and decode the values depends - * on the the {@link Codec#docsConsumer} and {@link Codec#docsProducer} methods. + * on the the {@link DocValuesFormat#docsConsumer} and {@link DocValuesFormat#docsProducer} methods. * * @lucene.experimental */ diff --git a/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java b/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java index c8ed772..cb00e4d 100644 --- a/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java +++ b/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java @@ -106,7 +106,9 @@ public final class CompoundFileDirectory extends Directory { numEntries); for (int i = 0; i < numEntries; i++) { final FileEntry fileEntry = new FileEntry(); - mapping.put(input.readString(), fileEntry); + final String id = input.readString(); + assert !mapping.containsKey(id): "id=" + id + " was written multiple times in the CFS"; + mapping.put(id, fileEntry); fileEntry.offset = input.readLong(); fileEntry.length = input.readLong(); } @@ -170,6 +172,9 @@ public final class CompoundFileDirectory extends Directory { entry = new FileEntry(); entry.offset = offset; + + assert !entries.containsKey(id); + entries.put(id, entry); } @@ -271,7 +276,7 @@ public final class CompoundFileDirectory extends Directory { public long fileLength(String name) throws IOException { ensureOpen(); if (this.writer != null) { - return writer.fileLenght(name); + return writer.fileLength(name); } FileEntry e = entries.get(IndexFileNames.stripSegmentName(name)); if (e == null) @@ -323,4 +328,9 @@ public final class CompoundFileDirectory extends Directory { } }; } + + @Override + public String toString() { + return "CompoundFileDirectory(file=\"" + fileName + "\" in dir=" + directory + ")"; + } } diff --git a/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java b/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java index 4508172..37ae0a5 100644 --- a/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java +++ b/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java @@ -22,9 +22,11 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.Map; import java.util.Queue; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.index.IndexFileNames; @@ -85,6 +87,7 @@ final class CompoundFileWriter implements Closeable{ private final Directory directory; private final Map entries = new HashMap(); + private final Set seenIDs = new HashSet(); // all entries that are written to a sep. file but not yet moved into CFS private final Queue pendingEntries = new LinkedList(); private boolean closed = false; @@ -238,6 +241,9 @@ final class CompoundFileWriter implements Closeable{ final FileEntry entry = new FileEntry(); entry.file = name; entries.put(name, entry); + final String id = IndexFileNames.stripSegmentName(name); + assert !seenIDs.contains(id): "file=\"" + name + "\" maps to id=\"" + id + "\", which was already written"; + seenIDs.add(id); final DirectCFSIndexOutput out; if (outputTaken.compareAndSet(false, true)) { out = new DirectCFSIndexOutput(dataOut, entry, false); @@ -284,7 +290,7 @@ final class CompoundFileWriter implements Closeable{ } } - long fileLenght(String name) throws IOException { + long fileLength(String name) throws IOException { FileEntry fileEntry = entries.get(name); if (fileEntry == null) { throw new FileNotFoundException(name + " does not exist"); diff --git a/lucene/src/java/org/apache/lucene/util/CodecUtil.java b/lucene/src/java/org/apache/lucene/util/CodecUtil.java index f4caa27..f7b2711 100644 --- a/lucene/src/java/org/apache/lucene/util/CodecUtil.java +++ b/lucene/src/java/org/apache/lucene/util/CodecUtil.java @@ -35,7 +35,7 @@ public final class CodecUtil { private final static int CODEC_MAGIC = 0x3fd76c17; - public static DataOutput writeHeader(DataOutput out, String codec, int version) + public static void writeHeader(DataOutput out, String codec, int version) throws IOException { BytesRef bytes = new BytesRef(codec); if (bytes.length != codec.length() || bytes.length >= 128) { @@ -44,8 +44,6 @@ public final class CodecUtil { out.writeInt(CODEC_MAGIC); out.writeString(codec); out.writeInt(version); - - return out; } public static int headerLength(String codec) { diff --git a/lucene/src/java/org/apache/lucene/util/NamedSPILoader.java b/lucene/src/java/org/apache/lucene/util/NamedSPILoader.java new file mode 100644 index 0000000..0f24daa --- /dev/null +++ b/lucene/src/java/org/apache/lucene/util/NamedSPILoader.java @@ -0,0 +1,72 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.Set; +import java.util.ServiceLoader; + +/** + * Helper class for loading named SPIs from classpath (e.g. Codec, PostingsFormat). + * @lucene.internal + */ +public final class NamedSPILoader implements Iterable { + + private final Map services; + private final Class clazz; + + public NamedSPILoader(Class clazz) { + this.clazz = clazz; + final ServiceLoader loader = ServiceLoader.load(clazz); + final LinkedHashMap services = new LinkedHashMap(); + for (final S service : loader) { + final String name = service.getName(); + // only add the first one for each name, later services will be ignored + // this allows to place services before others in classpath to make + // them used instead of others + if (!services.containsKey(name)) { + services.put(name, service); + } + } + this.services = Collections.unmodifiableMap(services); + } + + public S lookup(String name) { + final S service = services.get(name); + if (service != null) return service; + throw new IllegalArgumentException("A SPI class of type "+clazz.getName()+" with name '"+name+"' does not exist. "+ + "You need to add the corresponding JAR file supporting this SPI to your classpath."+ + "The current classpath supports the following names: "+availableServices()); + } + + public Set availableServices() { + return services.keySet(); + } + + public Iterator iterator() { + return services.values().iterator(); + } + + public static interface NamedSPI { + String getName(); + } + +} diff --git a/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec b/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec new file mode 100644 index 0000000..5d8a95e --- /dev/null +++ b/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.lucene.index.codecs.lucene40.Lucene40Codec +org.apache.lucene.index.codecs.lucene3x.Lucene3xCodec diff --git a/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.PostingsFormat b/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.PostingsFormat new file mode 100644 index 0000000..efbc25f --- /dev/null +++ b/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.PostingsFormat @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat +org.apache.lucene.index.codecs.pulsing.Pulsing40PostingsFormat +org.apache.lucene.index.codecs.simpletext.SimpleTextPostingsFormat +org.apache.lucene.index.codecs.memory.MemoryPostingsFormat diff --git a/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java b/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java rename to lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java diff --git a/lucene/src/test-framework/org/apache/lucene/analysis/CollationTestBase.java b/lucene/src/test-framework/java/org/apache/lucene/analysis/CollationTestBase.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/analysis/CollationTestBase.java rename to lucene/src/test-framework/java/org/apache/lucene/analysis/CollationTestBase.java diff --git a/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java b/lucene/src/test-framework/java/org/apache/lucene/analysis/MockAnalyzer.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java rename to lucene/src/test-framework/java/org/apache/lucene/analysis/MockAnalyzer.java diff --git a/lucene/src/test-framework/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java b/lucene/src/test-framework/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java rename to lucene/src/test-framework/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java diff --git a/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java b/lucene/src/test-framework/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java rename to lucene/src/test-framework/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java diff --git a/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenFilter.java b/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenFilter.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/analysis/MockTokenFilter.java rename to lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenFilter.java diff --git a/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java b/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java rename to lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java diff --git a/lucene/src/test-framework/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java b/lucene/src/test-framework/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java rename to lucene/src/test-framework/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java diff --git a/lucene/src/test-framework/org/apache/lucene/analysis/VocabularyAssert.java b/lucene/src/test-framework/java/org/apache/lucene/analysis/VocabularyAssert.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/analysis/VocabularyAssert.java rename to lucene/src/test-framework/java/org/apache/lucene/analysis/VocabularyAssert.java diff --git a/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java b/lucene/src/test-framework/java/org/apache/lucene/index/DocHelper.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/index/DocHelper.java rename to lucene/src/test-framework/java/org/apache/lucene/index/DocHelper.java diff --git a/lucene/src/test-framework/org/apache/lucene/index/MockIndexInput.java b/lucene/src/test-framework/java/org/apache/lucene/index/MockIndexInput.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/index/MockIndexInput.java rename to lucene/src/test-framework/java/org/apache/lucene/index/MockIndexInput.java diff --git a/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java b/lucene/src/test-framework/java/org/apache/lucene/index/MockRandomMergePolicy.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java rename to lucene/src/test-framework/java/org/apache/lucene/index/MockRandomMergePolicy.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java b/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java new file mode 100644 index 0000000..7216390 --- /dev/null +++ b/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java @@ -0,0 +1,103 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Random; + +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.lucene40ords.Lucene40WithOrds; +import org.apache.lucene.index.codecs.memory.MemoryPostingsFormat; +import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mockrandom.MockRandomPostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; +import org.apache.lucene.index.codecs.nestedpulsing.NestedPulsingPostingsFormat; +import org.apache.lucene.index.codecs.pulsing.Pulsing40PostingsFormat; +import org.apache.lucene.index.codecs.simpletext.SimpleTextPostingsFormat; +import org.apache.lucene.util._TestUtil; + +/** + * Codec that assigns per-field random postings formats. + *

+ * The same field/format assignment will happen regardless of order, + * a hash is computed up front that determines the mapping. + * This means fields can be put into things like HashSets and added to + * documents in different orders and the test will still be deterministic + * and reproducable. + */ +public class RandomCodec extends Lucene40Codec { + /** shuffled list of postingsformats to use for new mappings */ + private List formats = new ArrayList(); + /** memorized field->postingsformat mappings */ + private Map previousMappings = new HashMap(); + private final int perFieldSeed; + + @Override + public PostingsFormat getPostingsFormatForField(String name) { + PostingsFormat codec = previousMappings.get(name); + if (codec == null) { + codec = formats.get(Math.abs(perFieldSeed ^ name.hashCode()) % formats.size()); + if (codec instanceof SimpleTextPostingsFormat && perFieldSeed % 5 != 0) { + // make simpletext rarer, choose again + codec = formats.get(Math.abs(perFieldSeed ^ name.toUpperCase(Locale.ENGLISH).hashCode()) % formats.size()); + } + previousMappings.put(name, codec); + // Safety: + assert previousMappings.size() < 10000; + } + return codec; + } + + public RandomCodec(Random random, boolean useNoMemoryExpensiveCodec) { + this.perFieldSeed = random.nextInt(); + // TODO: make it possible to specify min/max iterms per + // block via CL: + int minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); + int maxItemsPerBlock = 2*(Math.max(2, minItemsPerBlock-1)) + random.nextInt(100); + formats.add(new Lucene40PostingsFormat(minItemsPerBlock, maxItemsPerBlock)); + // TODO: make it possible to specify min/max iterms per + // block via CL: + minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); + maxItemsPerBlock = 2*(Math.max(1, minItemsPerBlock-1)) + random.nextInt(100); + formats.add(new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock)); + formats.add(new MockSepPostingsFormat()); + formats.add(new MockFixedIntBlockPostingsFormat(_TestUtil.nextInt(random, 1, 2000))); + formats.add(new MockVariableIntBlockPostingsFormat( _TestUtil.nextInt(random, 1, 127))); + formats.add(new MockRandomPostingsFormat(random)); + formats.add(new NestedPulsingPostingsFormat()); + formats.add(new Lucene40WithOrds()); + if (!useNoMemoryExpensiveCodec) { + formats.add(new SimpleTextPostingsFormat()); + formats.add(new MemoryPostingsFormat()); + } + Collections.shuffle(formats, random); + } + + @Override + public String toString() { + return super.toString() + ": " + previousMappings.toString(); + } +} diff --git a/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java b/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java similarity index 96% rename from lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java rename to lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java index 9e028ce..6abf908 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java +++ b/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java @@ -27,7 +27,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.IndexDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexWriter; // javadoc -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; @@ -53,7 +53,7 @@ public class RandomIndexWriter implements Closeable { private final int fixedBytesLength; private final long docValuesFieldPrefix; private volatile boolean doDocValues; - private CodecProvider codecProvider; + private final Codec codec; // sugar // Randomly calls Thread.yield so we mixup thread scheduling private static final class MockIndexWriter extends IndexWriter { @@ -96,9 +96,10 @@ public class RandomIndexWriter implements Closeable { this.r = r; w = new MockIndexWriter(r, dir, c); flushAt = _TestUtil.nextInt(r, 10, 1000); + codec = w.getConfig().getCodec(); if (LuceneTestCase.VERBOSE) { System.out.println("RIW config=" + w.getConfig()); - System.out.println("codec default=" + w.getConfig().getCodecProvider().getDefaultFieldCodec()); + System.out.println("codec default=" + codec.getName()); w.setInfoStream(System.out); } /* TODO: find some what to make that random... @@ -109,7 +110,6 @@ public class RandomIndexWriter implements Closeable { */ fixedBytesLength = 37; docValuesFieldPrefix = r.nextLong(); - codecProvider = w.getConfig().getCodecProvider(); switchDoDocValues(); } @@ -171,7 +171,7 @@ public class RandomIndexWriter implements Closeable { ValueType[] values = ValueType.values(); ValueType type = values[random.nextInt(values.length)]; String name = "random_" + type.name() + "" + docValuesFieldPrefix; - if ("PreFlex".equals(codecProvider.getFieldCodec(name)) || doc.getField(name) != null) + if ("Lucene3x".equals(codec.getName()) || doc.getField(name) != null) return; IndexDocValuesField docValuesField = new IndexDocValuesField(name); switch (type) { @@ -367,7 +367,7 @@ public class RandomIndexWriter implements Closeable { // If we are writing with PreFlexRW, force a full // IndexReader.open so terms are sorted in codepoint // order during searching: - if (!applyDeletions || !w.codecs.getDefaultFieldCodec().equals("PreFlex") && r.nextBoolean()) { + if (!applyDeletions || !codec.getName().equals("Lucene3x") && r.nextBoolean()) { if (LuceneTestCase.VERBOSE) { System.out.println("RIW.getReader: use NRT reader"); } @@ -382,7 +382,7 @@ public class RandomIndexWriter implements Closeable { w.commit(); switchDoDocValues(); if (r.nextBoolean()) { - return IndexReader.open(w.getDirectory(), new KeepOnlyLastCommitDeletionPolicy(), r.nextBoolean(), _TestUtil.nextInt(r, 1, 10), w.getConfig().getCodecProvider()); + return IndexReader.open(w.getDirectory(), new KeepOnlyLastCommitDeletionPolicy(), r.nextBoolean(), _TestUtil.nextInt(r, 1, 10)); } else { return w.getReader(applyDeletions); } diff --git a/lucene/src/test-framework/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java b/lucene/src/test-framework/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java similarity index 98% rename from lucene/src/test-framework/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java rename to lucene/src/test-framework/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java index 425fe7b..8d3fb5b 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java +++ b/lucene/src/test-framework/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java @@ -35,7 +35,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.Query; @@ -432,12 +431,6 @@ public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCas final long t0 = System.currentTimeMillis(); - final String defaultCodec = CodecProvider.getDefault().getDefaultFieldCodec(); - if (defaultCodec.equals("SimpleText") || defaultCodec.equals("Memory")) { - // no - CodecProvider.getDefault().setDefaultFieldCodec("Standard"); - } - final LineFileDocs docs = new LineFileDocs(random); final File tempDir = _TestUtil.getTempDir(testName); dir = newFSDirectory(tempDir); diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/lucene40ords/Lucene40WithOrds.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/lucene40ords/Lucene40WithOrds.java new file mode 100644 index 0000000..728c361 --- /dev/null +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/lucene40ords/Lucene40WithOrds.java @@ -0,0 +1,145 @@ +package org.apache.lucene.index.codecs.lucene40ords; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.BlockTermsReader; +import org.apache.lucene.index.codecs.BlockTermsWriter; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; +import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.TermsIndexReaderBase; +import org.apache.lucene.index.codecs.TermsIndexWriterBase; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; + +// TODO: we could make separate base class that can wrapp +// any PostingsBaseFormat and make it ord-able... + +public class Lucene40WithOrds extends PostingsFormat { + + public Lucene40WithOrds() { + super("Lucene40WithOrds"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docs = new Lucene40PostingsWriter(state); + + // TODO: should we make the terms index more easily + // pluggable? Ie so that this codec would record which + // index impl was used, and switch on loading? + // Or... you must make a new Codec for this? + TermsIndexWriterBase indexWriter; + boolean success = false; + try { + indexWriter = new FixedGapTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + docs.close(); + } + } + + success = false; + try { + // Must use BlockTermsWriter (not BlockTree) because + // BlockTree doens't support ords (yet)... + FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs); + success = true; + return ret; + } finally { + if (!success) { + try { + docs.close(); + } finally { + indexWriter.close(); + } + } + } + } + + public final static int TERMS_CACHE_SIZE = 1024; + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase postings = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + TermsIndexReaderBase indexReader; + + boolean success = false; + try { + indexReader = new FixedGapTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.segmentSuffix, state.context); + success = true; + } finally { + if (!success) { + postings.close(); + } + } + + success = false; + try { + FieldsProducer ret = new BlockTermsReader(indexReader, + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postings, + state.context, + TERMS_CACHE_SIZE, + state.segmentSuffix); + success = true; + return ret; + } finally { + if (!success) { + try { + postings.close(); + } finally { + indexReader.close(); + } + } + } + } + + /** Extension of freq postings file */ + static final String FREQ_EXTENSION = "frq"; + + /** Extension of prox postings file */ + static final String PROX_EXTENSION = "prx"; + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java new file mode 100644 index 0000000..427ebb2 --- /dev/null +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java @@ -0,0 +1,210 @@ +package org.apache.lucene.index.codecs.mockintblock; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.sep.IntStreamFactory; +import org.apache.lucene.index.codecs.sep.IntIndexInput; +import org.apache.lucene.index.codecs.sep.IntIndexOutput; +import org.apache.lucene.index.codecs.sep.SepPostingsReader; +import org.apache.lucene.index.codecs.sep.SepPostingsWriter; +import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput; +import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; +import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.BlockTermsReader; +import org.apache.lucene.index.codecs.BlockTermsWriter; +import org.apache.lucene.index.codecs.TermsIndexReaderBase; +import org.apache.lucene.index.codecs.TermsIndexWriterBase; +import org.apache.lucene.store.*; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; + +/** + * A silly test codec to verify core support for fixed + * sized int block encoders is working. The int encoder + * used here just writes each block as a series of vInt. + */ + +public class MockFixedIntBlockPostingsFormat extends PostingsFormat { + + private final int blockSize; + + public MockFixedIntBlockPostingsFormat() { + this(1); + } + + public MockFixedIntBlockPostingsFormat(int blockSize) { + super("MockFixedIntBlock"); + this.blockSize = blockSize; + } + + @Override + public String toString() { + return getName() + "(blockSize=" + blockSize + ")"; + } + + // only for testing + public IntStreamFactory getIntFactory() { + return new MockIntFactory(blockSize); + } + + public static class MockIntFactory extends IntStreamFactory { + private final int blockSize; + + public MockIntFactory(int blockSize) { + this.blockSize = blockSize; + } + + @Override + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + return new FixedIntBlockIndexInput(dir.openInput(fileName, context)) { + + @Override + protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException { + return new BlockReader() { + public void seek(long pos) {} + public void readBlock() throws IOException { + for(int i=0;i= 0; + out.writeVInt(buffer[i]); + } + } + }; + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + } + } + } + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new MockIntFactory(blockSize)); + + boolean success = false; + TermsIndexWriterBase indexWriter; + try { + indexWriter = new FixedGapTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + postingsWriter.close(); + } + } + + success = false; + try { + FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, postingsWriter); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, + state.segmentInfo, + state.context, + new MockIntFactory(blockSize), state.segmentSuffix); + + TermsIndexReaderBase indexReader; + boolean success = false; + try { + indexReader = new FixedGapTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator(), state.segmentSuffix, + IOContext.DEFAULT); + success = true; + } finally { + if (!success) { + postingsReader.close(); + } + } + + success = false; + try { + FieldsProducer ret = new BlockTermsReader(indexReader, + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postingsReader, + state.context, + Lucene40PostingsFormat.TERMS_CACHE_SIZE, + state.segmentSuffix); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsReader.close(); + } finally { + indexReader.close(); + } + } + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + SepPostingsReader.files(segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java similarity index 81% rename from lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java rename to lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java index d0a0c0b..6e3cde7 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java @@ -20,27 +20,22 @@ package org.apache.lucene.index.codecs.mockintblock; import java.io.IOException; import java.util.Set; -import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.sep.IntStreamFactory; import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.index.codecs.sep.IntIndexOutput; -import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; -import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; import org.apache.lucene.index.codecs.sep.SepPostingsReader; import org.apache.lucene.index.codecs.sep.SepPostingsWriter; -import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput; import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.BlockTermsReader; @@ -61,17 +56,21 @@ import org.apache.lucene.util.IOUtils; * int is <= 3, else 2*baseBlockSize. */ -public class MockVariableIntBlockCodec extends Codec { +public class MockVariableIntBlockPostingsFormat extends PostingsFormat { private final int baseBlockSize; - public MockVariableIntBlockCodec(int baseBlockSize) { + public MockVariableIntBlockPostingsFormat() { + this(1); + } + + public MockVariableIntBlockPostingsFormat(int baseBlockSize) { super("MockVariableIntBlock"); this.baseBlockSize = baseBlockSize; } @Override public String toString() { - return name + "(baseBlockSize="+ baseBlockSize + ")"; + return getName() + "(baseBlockSize="+ baseBlockSize + ")"; } public static class MockIntFactory extends IntStreamFactory { @@ -184,7 +183,7 @@ public class MockVariableIntBlockCodec extends Codec { PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, state.segmentInfo, state.context, - new MockIntFactory(baseBlockSize), state.codecId); + new MockIntFactory(baseBlockSize), state.segmentSuffix); TermsIndexReaderBase indexReader; boolean success = false; @@ -194,7 +193,7 @@ public class MockVariableIntBlockCodec extends Codec { state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId, state.context); + state.segmentSuffix, state.context); success = true; } finally { if (!success) { @@ -210,8 +209,8 @@ public class MockVariableIntBlockCodec extends Codec { state.segmentInfo.name, postingsReader, state.context, - StandardCodec.TERMS_CACHE_SIZE, - state.codecId); + Lucene40PostingsFormat.TERMS_CACHE_SIZE, + state.segmentSuffix); success = true; return ret; } finally { @@ -226,28 +225,9 @@ public class MockVariableIntBlockCodec extends Codec { } @Override - public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { - SepPostingsReader.files(segmentInfo, codecId, files); - BlockTermsReader.files(dir, segmentInfo, codecId, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); - } - - @Override - public void getExtensions(Set extensions) { - SepPostingsWriter.getExtensions(extensions); - BlockTermsReader.getExtensions(extensions); - FixedGapTermsIndexReader.getIndexExtensions(extensions); - SepDocValuesConsumer.getExtensions(extensions); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new SepDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new SepDocValuesProducer(state); + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + SepPostingsReader.files(segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); } } diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockrandom/MockRandomPostingsFormat.java similarity index 81% rename from lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java rename to lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockrandom/MockRandomPostingsFormat.java index 9cd824c..898229f 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockrandom/MockRandomPostingsFormat.java @@ -26,7 +26,6 @@ import java.util.Set; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentReadState; import org.apache.lucene.index.SegmentWriteState; @@ -34,15 +33,11 @@ import org.apache.lucene.index.codecs.BlockTreeTermsReader; import org.apache.lucene.index.codecs.BlockTreeTermsWriter; import org.apache.lucene.index.codecs.BlockTermsReader; import org.apache.lucene.index.codecs.BlockTermsWriter; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.TermStats; @@ -50,20 +45,18 @@ import org.apache.lucene.index.codecs.TermsIndexReaderBase; import org.apache.lucene.index.codecs.TermsIndexWriterBase; import org.apache.lucene.index.codecs.VariableGapTermsIndexReader; import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter; -import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec; -import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; +import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockPostingsFormat; import org.apache.lucene.index.codecs.mocksep.MockSingleIntFactory; import org.apache.lucene.index.codecs.pulsing.PulsingPostingsReader; import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriter; import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.index.codecs.sep.IntIndexOutput; import org.apache.lucene.index.codecs.sep.IntStreamFactory; -import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; -import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; import org.apache.lucene.index.codecs.sep.SepPostingsReader; import org.apache.lucene.index.codecs.sep.SepPostingsWriter; -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; -import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -76,14 +69,17 @@ import org.apache.lucene.util._TestUtil; * Randomly combines terms index impl w/ postings impls. */ -public class MockRandomCodec extends Codec { - private final boolean useSepDocValues; +public class MockRandomPostingsFormat extends PostingsFormat { private final Random seedRandom; private final String SEED_EXT = "sd"; - public MockRandomCodec(Random random) { + public MockRandomPostingsFormat() { + // just for reading, we are gonna setSeed from the .seed file... right? + this(new Random()); + } + + public MockRandomPostingsFormat(Random random) { super("MockRandom"); - this.useSepDocValues = random.nextBoolean(); this.seedRandom = new Random(random.nextLong()); } @@ -96,9 +92,9 @@ public class MockRandomCodec extends Codec { salt = random.nextInt(); delegates.add(new MockSingleIntFactory()); final int blockSize = _TestUtil.nextInt(random, 1, 2000); - delegates.add(new MockFixedIntBlockCodec.MockIntFactory(blockSize)); + delegates.add(new MockFixedIntBlockPostingsFormat.MockIntFactory(blockSize)); final int baseBlockSize = _TestUtil.nextInt(random, 1, 127); - delegates.add(new MockVariableIntBlockCodec.MockIntFactory(baseBlockSize)); + delegates.add(new MockVariableIntBlockPostingsFormat.MockIntFactory(baseBlockSize)); // TODO: others } @@ -142,10 +138,10 @@ public class MockRandomCodec extends Codec { final long seed = seedRandom.nextLong(); if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: writing to seg=" + state.segmentName + " codecID=" + state.codecId + " seed=" + seed); + System.out.println("MockRandomCodec: writing to seg=" + state.segmentName + " formatID=" + state.segmentSuffix + " seed=" + seed); } - final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SEED_EXT); + final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, SEED_EXT); final IndexOutput out = state.directory.createOutput(seedFileName, state.context); try { out.writeLong(seed); @@ -164,7 +160,7 @@ public class MockRandomCodec extends Codec { if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: writing Standard postings"); } - postingsWriter = new StandardPostingsWriter(state, skipInterval); + postingsWriter = new Lucene40PostingsWriter(state, skipInterval); } if (random.nextBoolean()) { @@ -276,11 +272,11 @@ public class MockRandomCodec extends Codec { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, SEED_EXT); + final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, SEED_EXT); final IndexInput in = state.dir.openInput(seedFileName, state.context); final long seed = in.readLong(); if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " codecID=" + state.codecId + " seed=" + seed); + System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " formatID=" + state.segmentSuffix + " seed=" + seed); } in.close(); @@ -298,12 +294,12 @@ public class MockRandomCodec extends Codec { System.out.println("MockRandomCodec: reading Sep postings"); } postingsReader = new SepPostingsReader(state.dir, state.segmentInfo, - state.context, new MockIntStreamFactory(random), state.codecId); + state.context, new MockIntStreamFactory(random), state.segmentSuffix); } else { if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: reading Standard postings"); } - postingsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); + postingsReader = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); } if (random.nextBoolean()) { @@ -329,7 +325,7 @@ public class MockRandomCodec extends Codec { state.segmentInfo.name, postingsReader, state.context, - state.codecId, + state.segmentSuffix, state.termsIndexDivisor); success = true; } finally { @@ -363,7 +359,7 @@ public class MockRandomCodec extends Codec { state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId, state.context); + state.segmentSuffix, state.context); } else { final int n2 = random.nextInt(3); if (n2 == 1) { @@ -378,7 +374,7 @@ public class MockRandomCodec extends Codec { state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - state.codecId, state.context); + state.segmentSuffix, state.context); } @@ -400,7 +396,7 @@ public class MockRandomCodec extends Codec { postingsReader, state.context, termsCacheSize, - state.codecId); + state.segmentSuffix); success = true; } finally { if (!success) { @@ -417,20 +413,15 @@ public class MockRandomCodec extends Codec { } @Override - public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { - final String seedFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SEED_EXT); + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + final String seedFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SEED_EXT); files.add(seedFileName); - SepPostingsReader.files(segmentInfo, codecId, files); - StandardPostingsReader.files(dir, segmentInfo, codecId, files); - BlockTermsReader.files(dir, segmentInfo, codecId, files); - BlockTreeTermsReader.files(dir, segmentInfo, codecId, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - if (useSepDocValues) { - SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); - } else { - DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files); - } + SepPostingsReader.files(segmentInfo, segmentSuffix, files); + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + VariableGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); // hackish! Iterator it = files.iterator(); while(it.hasNext()) { @@ -441,39 +432,4 @@ public class MockRandomCodec extends Codec { } //System.out.println("MockRandom.files return " + files); } - - @Override - public void getExtensions(Set extensions) { - SepPostingsWriter.getExtensions(extensions); - BlockTermsReader.getExtensions(extensions); - BlockTreeTermsReader.getExtensions(extensions); - FixedGapTermsIndexReader.getIndexExtensions(extensions); - VariableGapTermsIndexReader.getIndexExtensions(extensions); - if (useSepDocValues) { - SepDocValuesConsumer.getExtensions(extensions); - } else { - DefaultDocValuesConsumer.getExtensions(extensions); - } - extensions.add(SEED_EXT); - //System.out.println("MockRandom.getExtensions return " + extensions); - } - - // can we make this more evil? - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - if (useSepDocValues) { - return new SepDocValuesConsumer(state); - } else { - return new DefaultDocValuesConsumer(state); - } - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - if (useSepDocValues) { - return new SepDocValuesProducer(state); - } else { - return new DefaultDocValuesProducer(state); - } - } } diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepDocValuesFormat.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepDocValuesFormat.java new file mode 100644 index 0000000..790bca1 --- /dev/null +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepDocValuesFormat.java @@ -0,0 +1,54 @@ +package org.apache.lucene.index.codecs.mocksep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.PerDocWriteState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; +import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; +import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; +import org.apache.lucene.store.Directory; + +/** + * Separate-file docvalues implementation + * @lucene.experimental + */ +// TODO: we could move this out of src/test ? +public class MockSepDocValuesFormat extends DocValuesFormat { + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new SepDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new SepDocValuesProducer(state); + } + + @Override + public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + SepDocValuesConsumer.files(dir, info, files); + } +} diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepPostingsFormat.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepPostingsFormat.java new file mode 100644 index 0000000..1e8baa1 --- /dev/null +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepPostingsFormat.java @@ -0,0 +1,138 @@ +package org.apache.lucene.index.codecs.mocksep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; +import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.BlockTermsReader; +import org.apache.lucene.index.codecs.BlockTermsWriter; +import org.apache.lucene.index.codecs.TermsIndexReaderBase; +import org.apache.lucene.index.codecs.TermsIndexWriterBase; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.sep.SepPostingsWriter; +import org.apache.lucene.index.codecs.sep.SepPostingsReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; + +/** + * A silly codec that simply writes each file separately as + * single vInts. Don't use this (performance will be poor)! + * This is here just to test the core sep codec + * classes. + */ +public class MockSepPostingsFormat extends PostingsFormat { + + public MockSepPostingsFormat() { + super("MockSep"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + + PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new MockSingleIntFactory()); + + boolean success = false; + TermsIndexWriterBase indexWriter; + try { + indexWriter = new FixedGapTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + postingsWriter.close(); + } + } + + success = false; + try { + FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, postingsWriter); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + + PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, state.segmentInfo, + state.context, new MockSingleIntFactory(), state.segmentSuffix); + + TermsIndexReaderBase indexReader; + boolean success = false; + try { + indexReader = new FixedGapTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.segmentSuffix, state.context); + success = true; + } finally { + if (!success) { + postingsReader.close(); + } + } + + success = false; + try { + FieldsProducer ret = new BlockTermsReader(indexReader, + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postingsReader, + state.context, + Lucene40PostingsFormat.TERMS_CACHE_SIZE, + state.segmentSuffix); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsReader.close(); + } finally { + indexReader.close(); + } + } + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + SepPostingsReader.files(segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java rename to lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java rename to lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java rename to lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/nestedpulsing/NestedPulsingPostingsFormat.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/nestedpulsing/NestedPulsingPostingsFormat.java new file mode 100644 index 0000000..659aa17 --- /dev/null +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/nestedpulsing/NestedPulsingPostingsFormat.java @@ -0,0 +1,99 @@ +package org.apache.lucene.index.codecs.nestedpulsing; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.BlockTreeTermsReader; +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; +import org.apache.lucene.index.codecs.pulsing.PulsingPostingsReader; +import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriter; +import org.apache.lucene.store.Directory; + +/** + * Pulsing(1, Pulsing(2, Lucene40)) + * @lucene.experimental + */ +// TODO: if we create PulsingPostingsBaseFormat then we +// can simplify this? note: I don't like the *BaseFormat +// hierarchy, maybe we can clean that up... +public class NestedPulsingPostingsFormat extends PostingsFormat { + public NestedPulsingPostingsFormat() { + super("NestedPulsing"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docsWriter = new Lucene40PostingsWriter(state); + + PostingsWriterBase pulsingWriterInner = new PulsingPostingsWriter(2, docsWriter); + PostingsWriterBase pulsingWriter = new PulsingPostingsWriter(1, pulsingWriterInner); + + // Terms dict + boolean success = false; + try { + FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, + BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); + success = true; + return ret; + } finally { + if (!success) { + pulsingWriter.close(); + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase docsReader = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + PostingsReaderBase pulsingReaderInner = new PulsingPostingsReader(docsReader); + PostingsReaderBase pulsingReader = new PulsingPostingsReader(pulsingReaderInner); + boolean success = false; + try { + FieldsProducer ret = new BlockTreeTermsReader( + state.dir, state.fieldInfos, state.segmentInfo.name, + pulsingReader, + state.context, + state.segmentSuffix, + state.termsIndexDivisor); + success = true; + return ret; + } finally { + if (!success) { + pulsingReader.close(); + } + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java similarity index 95% rename from lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java rename to lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java index 05bbbfb..ca3c45e 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java @@ -29,9 +29,9 @@ import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.PostingsConsumer; import org.apache.lucene.index.codecs.TermStats; import org.apache.lucene.index.codecs.TermsConsumer; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; -import org.apache.lucene.index.codecs.preflex.TermInfo; -import org.apache.lucene.index.codecs.standard.DefaultSkipListWriter; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; +import org.apache.lucene.index.codecs.lucene3x.TermInfo; +import org.apache.lucene.index.codecs.lucene40.DefaultSkipListWriter; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.IOUtils; @@ -50,12 +50,12 @@ class PreFlexFieldsWriter extends FieldsConsumer { state.fieldInfos, state.termIndexInterval); - final String freqFile = IndexFileNames.segmentFileName(state.segmentName, "", PreFlexCodec.FREQ_EXTENSION); + final String freqFile = IndexFileNames.segmentFileName(state.segmentName, "", Lucene3xPostingsFormat.FREQ_EXTENSION); freqOut = state.directory.createOutput(freqFile, state.context); totalNumDocs = state.numDocs; if (state.fieldInfos.hasProx()) { - final String proxFile = IndexFileNames.segmentFileName(state.segmentName, "", PreFlexCodec.PROX_EXTENSION); + final String proxFile = IndexFileNames.segmentFileName(state.segmentName, "", Lucene3xPostingsFormat.PROX_EXTENSION); proxOut = state.directory.createOutput(proxFile, state.context); } else { proxOut = null; diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java new file mode 100644 index 0000000..ff51b28 --- /dev/null +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java @@ -0,0 +1,39 @@ +package org.apache.lucene.index.codecs.preflexrw; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xCodec; +import org.apache.lucene.util.LuceneTestCase; + +/** + * Writes 3.x-like indexes (not perfect emulation yet) for testing only! + * @lucene.experimental + */ +public class PreFlexRWCodec extends Lucene3xCodec { + private final PostingsFormat postings = new PreFlexRWPostingsFormat(); + + @Override + public PostingsFormat postingsFormat() { + if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) { + return postings; + } else { + return super.postingsFormat(); + } + } +} diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWPostingsFormat.java similarity index 87% rename from lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java rename to lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWPostingsFormat.java index f911ef2..633ea47 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWPostingsFormat.java @@ -21,8 +21,8 @@ import java.io.IOException; import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; -import org.apache.lucene.index.codecs.preflex.PreFlexFields; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xFields; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.util.LuceneTestCase; @@ -32,9 +32,9 @@ import org.apache.lucene.util.LuceneTestCase; * * @lucene.experimental */ -public class PreFlexRWCodec extends PreFlexCodec { +public class PreFlexRWPostingsFormat extends Lucene3xPostingsFormat { - public PreFlexRWCodec() { + public PreFlexRWPostingsFormat() { // NOTE: we impersonate the PreFlex codec so that it can // read the segments we write! } @@ -50,7 +50,7 @@ public class PreFlexRWCodec extends PreFlexCodec { // Whenever IW opens readers, eg for merging, we have to // keep terms order in UTF16: - return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor) { + return new Lucene3xFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor) { @Override protected boolean sortTermsByUnicode() { // We carefully peek into stack track above us: if diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java similarity index 95% rename from lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java rename to lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java index 91e07a1..8ca9be5 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java @@ -23,8 +23,8 @@ import java.io.IOException; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; -import org.apache.lucene.index.codecs.preflex.TermInfo; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; +import org.apache.lucene.index.codecs.lucene3x.TermInfo; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -106,8 +106,8 @@ final class TermInfosWriter implements Closeable { try { directory.deleteFile(IndexFileNames.segmentFileName(segment, "", - (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION - : PreFlexCodec.TERMS_EXTENSION))); + (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION + : Lucene3xPostingsFormat.TERMS_EXTENSION))); } catch (IOException ignored) { } } @@ -125,8 +125,8 @@ final class TermInfosWriter implements Closeable { fieldInfos = fis; isIndex = isi; output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", - (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION - : PreFlexCodec.TERMS_EXTENSION)), IOContext.DEFAULT); + (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION + : Lucene3xPostingsFormat.TERMS_EXTENSION)), IOContext.DEFAULT); boolean success = false; try { output.writeInt(FORMAT_CURRENT); // write format @@ -147,8 +147,8 @@ final class TermInfosWriter implements Closeable { try { directory.deleteFile(IndexFileNames.segmentFileName(segment, "", - (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION - : PreFlexCodec.TERMS_EXTENSION))); + (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION + : Lucene3xPostingsFormat.TERMS_EXTENSION))); } catch (IOException ignored) { } } diff --git a/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java new file mode 100644 index 0000000..f259b54 --- /dev/null +++ b/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java @@ -0,0 +1,575 @@ +package org.apache.lucene.index.codecs.ramonly; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PostingsConsumer; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.TermStats; +import org.apache.lucene.index.codecs.TermsConsumer; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.util.IOUtils; + +/** Stores all postings data in RAM, but writes a small + * token (header + single int) to identify which "slot" the + * index is using in RAM HashMap. + * + * NOTE: this codec sorts terms by reverse-unicode-order! */ + +public class RAMOnlyPostingsFormat extends PostingsFormat { + + // For fun, test that we can override how terms are + // sorted, and basic things still work -- this comparator + // sorts in reversed unicode code point order: + private static final Comparator reverseUnicodeComparator = new Comparator() { + public int compare(BytesRef t1, BytesRef t2) { + byte[] b1 = t1.bytes; + byte[] b2 = t2.bytes; + int b1Stop; + int b1Upto = t1.offset; + int b2Upto = t2.offset; + if (t1.length < t2.length) { + b1Stop = t1.offset + t1.length; + } else { + b1Stop = t1.offset + t2.length; + } + while(b1Upto < b1Stop) { + final int bb1 = b1[b1Upto++] & 0xff; + final int bb2 = b2[b2Upto++] & 0xff; + if (bb1 != bb2) { + //System.out.println("cmp 1=" + t1 + " 2=" + t2 + " return " + (bb2-bb1)); + return bb2 - bb1; + } + } + + // One is prefix of another, or they are equal + return t2.length-t1.length; + } + + @Override + public boolean equals(Object other) { + return this == other; + } + }; + + public RAMOnlyPostingsFormat() { + super("RAMOnly"); + } + + // Postings state: + static class RAMPostings extends FieldsProducer { + final Map fieldToTerms = new TreeMap(); + + @Override + public Terms terms(String field) { + return fieldToTerms.get(field); + } + + @Override + public FieldsEnum iterator() { + return new RAMFieldsEnum(this); + } + + @Override + public void close() { + } + } + + static class RAMField extends Terms { + final String field; + final SortedMap termToDocs = new TreeMap(); + long sumTotalTermFreq; + long sumDocFreq; + int docCount; + + RAMField(String field) { + this.field = field; + } + + @Override + public long getUniqueTermCount() { + return termToDocs.size(); + } + + @Override + public long getSumTotalTermFreq() { + return sumTotalTermFreq; + } + + @Override + public long getSumDocFreq() throws IOException { + return sumDocFreq; + } + + @Override + public int getDocCount() throws IOException { + return docCount; + } + + @Override + public TermsEnum iterator() { + return new RAMTermsEnum(RAMOnlyPostingsFormat.RAMField.this); + } + + @Override + public Comparator getComparator() { + return reverseUnicodeComparator; + } + } + + static class RAMTerm { + final String term; + long totalTermFreq; + final List docs = new ArrayList(); + public RAMTerm(String term) { + this.term = term; + } + } + + static class RAMDoc { + final int docID; + final int[] positions; + byte[][] payloads; + + public RAMDoc(int docID, int freq) { + this.docID = docID; + positions = new int[freq]; + } + } + + // Classes for writing to the postings state + private static class RAMFieldsConsumer extends FieldsConsumer { + + private final RAMPostings postings; + private final RAMTermsConsumer termsConsumer = new RAMTermsConsumer(); + + public RAMFieldsConsumer(RAMPostings postings) { + this.postings = postings; + } + + @Override + public TermsConsumer addField(FieldInfo field) { + RAMField ramField = new RAMField(field.name); + postings.fieldToTerms.put(field.name, ramField); + termsConsumer.reset(ramField); + return termsConsumer; + } + + @Override + public void close() { + // TODO: finalize stuff + } + } + + private static class RAMTermsConsumer extends TermsConsumer { + private RAMField field; + private final RAMPostingsWriterImpl postingsWriter = new RAMPostingsWriterImpl(); + RAMTerm current; + + void reset(RAMField field) { + this.field = field; + } + + @Override + public PostingsConsumer startTerm(BytesRef text) { + final String term = text.utf8ToString(); + current = new RAMTerm(term); + postingsWriter.reset(current); + return postingsWriter; + } + + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public void finishTerm(BytesRef text, TermStats stats) { + assert stats.docFreq > 0; + assert stats.docFreq == current.docs.size(); + current.totalTermFreq = stats.totalTermFreq; + field.termToDocs.put(current.term, current); + } + + @Override + public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) { + field.sumTotalTermFreq = sumTotalTermFreq; + field.sumDocFreq = sumDocFreq; + field.docCount = docCount; + } + } + + public static class RAMPostingsWriterImpl extends PostingsConsumer { + private RAMTerm term; + private RAMDoc current; + private int posUpto = 0; + + public void reset(RAMTerm term) { + this.term = term; + } + + @Override + public void startDoc(int docID, int freq) { + current = new RAMDoc(docID, freq); + term.docs.add(current); + posUpto = 0; + } + + @Override + public void addPosition(int position, BytesRef payload) { + current.positions[posUpto] = position; + if (payload != null && payload.length > 0) { + if (current.payloads == null) { + current.payloads = new byte[current.positions.length][]; + } + byte[] bytes = current.payloads[posUpto] = new byte[payload.length]; + System.arraycopy(payload.bytes, payload.offset, bytes, 0, payload.length); + } + posUpto++; + } + + @Override + public void finishDoc() { + assert posUpto == current.positions.length; + } + } + + // Classes for reading from the postings state + static class RAMFieldsEnum extends FieldsEnum { + private final RAMPostings postings; + private final Iterator it; + private String current; + + public RAMFieldsEnum(RAMPostings postings) { + this.postings = postings; + this.it = postings.fieldToTerms.keySet().iterator(); + } + + @Override + public String next() { + if (it.hasNext()) { + current = it.next(); + } else { + current = null; + } + return current; + } + + @Override + public TermsEnum terms() { + return new RAMTermsEnum(postings.fieldToTerms.get(current)); + } + } + + static class RAMTermsEnum extends TermsEnum { + Iterator it; + String current; + private final RAMField ramField; + + public RAMTermsEnum(RAMField field) { + this.ramField = field; + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public BytesRef next() { + if (it == null) { + if (current == null) { + it = ramField.termToDocs.keySet().iterator(); + } else { + it = ramField.termToDocs.tailMap(current).keySet().iterator(); + } + } + if (it.hasNext()) { + current = it.next(); + return new BytesRef(current); + } else { + return null; + } + } + + @Override + public SeekStatus seekCeil(BytesRef term, boolean useCache) { + current = term.utf8ToString(); + it = null; + if (ramField.termToDocs.containsKey(current)) { + return SeekStatus.FOUND; + } else { + if (current.compareTo(ramField.termToDocs.lastKey()) > 0) { + return SeekStatus.END; + } else { + return SeekStatus.NOT_FOUND; + } + } + } + + @Override + public void seekExact(long ord) { + throw new UnsupportedOperationException(); + } + + @Override + public long ord() { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef term() { + // TODO: reuse BytesRef + return new BytesRef(current); + } + + @Override + public int docFreq() { + return ramField.termToDocs.get(current).docs.size(); + } + + @Override + public long totalTermFreq() { + return ramField.termToDocs.get(current).totalTermFreq; + } + + @Override + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) { + return new RAMDocsEnum(ramField.termToDocs.get(current), liveDocs); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) { + return new RAMDocsAndPositionsEnum(ramField.termToDocs.get(current), liveDocs); + } + } + + private static class RAMDocsEnum extends DocsEnum { + private final RAMTerm ramTerm; + private final Bits liveDocs; + private RAMDoc current; + int upto = -1; + int posUpto = 0; + + public RAMDocsEnum(RAMTerm ramTerm, Bits liveDocs) { + this.ramTerm = ramTerm; + this.liveDocs = liveDocs; + } + + @Override + public int advance(int targetDocID) { + do { + nextDoc(); + } while (upto < ramTerm.docs.size() && current.docID < targetDocID); + return NO_MORE_DOCS; + } + + // TODO: override bulk read, for better perf + @Override + public int nextDoc() { + while(true) { + upto++; + if (upto < ramTerm.docs.size()) { + current = ramTerm.docs.get(upto); + if (liveDocs == null || liveDocs.get(current.docID)) { + posUpto = 0; + return current.docID; + } + } else { + return NO_MORE_DOCS; + } + } + } + + @Override + public int freq() { + return current.positions.length; + } + + @Override + public int docID() { + return current.docID; + } + } + + private static class RAMDocsAndPositionsEnum extends DocsAndPositionsEnum { + private final RAMTerm ramTerm; + private final Bits liveDocs; + private RAMDoc current; + int upto = -1; + int posUpto = 0; + + public RAMDocsAndPositionsEnum(RAMTerm ramTerm, Bits liveDocs) { + this.ramTerm = ramTerm; + this.liveDocs = liveDocs; + } + + @Override + public int advance(int targetDocID) { + do { + nextDoc(); + } while (upto < ramTerm.docs.size() && current.docID < targetDocID); + return NO_MORE_DOCS; + } + + // TODO: override bulk read, for better perf + @Override + public int nextDoc() { + while(true) { + upto++; + if (upto < ramTerm.docs.size()) { + current = ramTerm.docs.get(upto); + if (liveDocs == null || liveDocs.get(current.docID)) { + posUpto = 0; + return current.docID; + } + } else { + return NO_MORE_DOCS; + } + } + } + + @Override + public int freq() { + return current.positions.length; + } + + @Override + public int docID() { + return current.docID; + } + + @Override + public int nextPosition() { + return current.positions[posUpto++]; + } + + @Override + public boolean hasPayload() { + return current.payloads != null && current.payloads[posUpto-1] != null; + } + + @Override + public BytesRef getPayload() { + return new BytesRef(current.payloads[posUpto-1]); + } + } + + // Holds all indexes created, keyed by the ID assigned in fieldsConsumer + private final Map state = new HashMap(); + + private final AtomicInteger nextID = new AtomicInteger(); + + private final String RAM_ONLY_NAME = "RAMOnly"; + private final static int VERSION_START = 0; + private final static int VERSION_LATEST = VERSION_START; + + private static final String ID_EXTENSION = "id"; + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState writeState) throws IOException { + final int id = nextID.getAndIncrement(); + + // TODO -- ok to do this up front instead of + // on close....? should be ok? + // Write our ID: + final String idFileName = IndexFileNames.segmentFileName(writeState.segmentName, writeState.segmentSuffix, ID_EXTENSION); + IndexOutput out = writeState.directory.createOutput(idFileName, writeState.context); + boolean success = false; + try { + CodecUtil.writeHeader(out, RAM_ONLY_NAME, VERSION_LATEST); + out.writeVInt(id); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + } else { + IOUtils.close(out); + } + } + + final RAMPostings postings = new RAMPostings(); + final RAMFieldsConsumer consumer = new RAMFieldsConsumer(postings); + + synchronized(state) { + state.put(id, postings); + } + return consumer; + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState readState) + throws IOException { + + // Load our ID: + final String idFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, ID_EXTENSION); + IndexInput in = readState.dir.openInput(idFileName, readState.context); + boolean success = false; + final int id; + try { + CodecUtil.checkHeader(in, RAM_ONLY_NAME, VERSION_START, VERSION_LATEST); + id = in.readVInt(); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(in); + } else { + IOUtils.close(in); + } + } + + synchronized(state) { + return state.get(id); + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) { + final String idFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, ID_EXTENSION); + files.add(idFileName); + } +} diff --git a/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java b/lucene/src/test-framework/java/org/apache/lucene/search/AssertingIndexSearcher.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java rename to lucene/src/test-framework/java/org/apache/lucene/search/AssertingIndexSearcher.java diff --git a/lucene/src/test-framework/org/apache/lucene/search/CachingWrapperFilterHelper.java b/lucene/src/test-framework/java/org/apache/lucene/search/CachingWrapperFilterHelper.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/search/CachingWrapperFilterHelper.java rename to lucene/src/test-framework/java/org/apache/lucene/search/CachingWrapperFilterHelper.java diff --git a/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java b/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/search/CheckHits.java rename to lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java diff --git a/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java b/lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java rename to lucene/src/test-framework/java/org/apache/lucene/search/QueryUtils.java diff --git a/lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java b/lucene/src/test-framework/java/org/apache/lucene/search/RandomSimilarityProvider.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java rename to lucene/src/test-framework/java/org/apache/lucene/search/RandomSimilarityProvider.java diff --git a/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java similarity index 97% rename from lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java rename to lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java index a4cd957..c4c00fa 100644 --- a/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java @@ -33,7 +33,6 @@ import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.ThrottledIndexOutput; import org.apache.lucene.util._TestUtil; @@ -486,14 +485,8 @@ public class MockDirectoryWrapper extends Directory { if (LuceneTestCase.VERBOSE) { System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex"); } - if (codecProvider != null) { - if (IndexReader.indexExists(this, codecProvider)) { - _TestUtil.checkIndex(this, codecProvider); - } - } else { - if (IndexReader.indexExists(this)) { - _TestUtil.checkIndex(this); - } + if (IndexReader.indexExists(this)) { + _TestUtil.checkIndex(this); } } delegate.close(); @@ -523,13 +516,6 @@ public class MockDirectoryWrapper extends Directory { public synchronized void removeIndexInput(IndexInput in, String name) { removeOpenFile(in, name); } - - private CodecProvider codecProvider; - - // We pass this CodecProvider to checkIndex when dir is closed... - public void setCodecProvider(CodecProvider cp) { - codecProvider = cp; - } boolean open = true; diff --git a/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java b/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexInputWrapper.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java rename to lucene/src/test-framework/java/org/apache/lucene/store/MockIndexInputWrapper.java diff --git a/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java b/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexOutputWrapper.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java rename to lucene/src/test-framework/java/org/apache/lucene/store/MockIndexOutputWrapper.java diff --git a/lucene/src/test-framework/org/apache/lucene/store/MockLockFactoryWrapper.java b/lucene/src/test-framework/java/org/apache/lucene/store/MockLockFactoryWrapper.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/store/MockLockFactoryWrapper.java rename to lucene/src/test-framework/java/org/apache/lucene/store/MockLockFactoryWrapper.java diff --git a/lucene/src/test-framework/org/apache/lucene/store/_TestHelper.java b/lucene/src/test-framework/java/org/apache/lucene/store/_TestHelper.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/store/_TestHelper.java rename to lucene/src/test-framework/java/org/apache/lucene/store/_TestHelper.java diff --git a/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java b/lucene/src/test-framework/java/org/apache/lucene/util/LineFileDocs.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java rename to lucene/src/test-framework/java/org/apache/lucene/util/LineFileDocs.java diff --git a/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitDividingSelector.java b/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitDividingSelector.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitDividingSelector.java rename to lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitDividingSelector.java diff --git a/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitResultFormatter.java b/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitResultFormatter.java similarity index 98% rename from lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitResultFormatter.java rename to lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitResultFormatter.java index c67b9bc..a03f780 100644 --- a/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitResultFormatter.java +++ b/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitResultFormatter.java @@ -66,8 +66,7 @@ public class LuceneJUnitResultFormatter implements JUnitResultFormatter { private static final org.apache.lucene.store.Lock lock; static { - File lockDir = new File( - System.getProperty("tests.lockdir", System.getProperty("java.io.tmpdir")), + File lockDir = new File(System.getProperty("java.io.tmpdir"), "lucene_junit_lock"); lockDir.mkdirs(); if (!lockDir.exists()) { diff --git a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java b/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java similarity index 87% rename from lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java rename to lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java index 826e165..37bb746 100644 --- a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java @@ -40,14 +40,18 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.*; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec; -import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec; -import org.apache.lucene.index.codecs.mocksep.MockSepCodec; -import org.apache.lucene.index.codecs.mockrandom.MockRandomCodec; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xCodec; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; +import org.apache.lucene.index.codecs.mockrandom.MockRandomPostingsFormat; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; -import org.apache.lucene.index.codecs.pulsing.PulsingCodec; +import org.apache.lucene.index.codecs.preflexrw.PreFlexRWPostingsFormat; +import org.apache.lucene.index.codecs.pulsing.PulsingPostingsFormat; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.FieldCache.CacheEntry; @@ -135,10 +139,8 @@ public abstract class LuceneTestCase extends Assert { // by default we randomly pick a different codec for // each test case (non-J4 tests) and each test class (J4 // tests) - /** Gets the codec to run tests with. */ - public static final String TEST_CODEC = System.getProperty("tests.codec", "randomPerField"); - /** Gets the codecprovider to run tests with */ - public static final String TEST_CODECPROVIDER = System.getProperty("tests.codecprovider", "random"); + /** Gets the postingsFormat to run tests with. */ + public static final String TEST_POSTINGSFORMAT = System.getProperty("tests.postingsformat", "random"); /** Gets the locale to run tests with */ public static final String TEST_LOCALE = System.getProperty("tests.locale", "random"); /** Gets the timezone to run tests with */ @@ -168,6 +170,9 @@ public abstract class LuceneTestCase extends Assert { */ public static final int RANDOM_MULTIPLIER = Integer.parseInt(System.getProperty("tests.multiplier", "1")); + /** @lucene.internal */ + public static boolean PREFLEX_IMPERSONATION_IS_ACTIVE; + private int savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount(); private volatile Thread.UncaughtExceptionHandler savedUncaughtExceptionHandler = null; @@ -208,12 +213,8 @@ public abstract class LuceneTestCase extends Assert { } private List uncaughtExceptions = Collections.synchronizedList(new ArrayList()); - // saves default codec: we do this statically as many build indexes in @beforeClass - private static String savedDefaultCodec; - // default codec: not set when we use a per-field provider. - private static Codec codec; - // default codec provider - private static CodecProvider savedCodecProvider; + // default codec + private static Codec savedCodec; private static SimilarityProvider similarityProvider; @@ -224,96 +225,6 @@ public abstract class LuceneTestCase extends Assert { protected static Map stores; - private static final String[] TEST_CODECS = new String[] {"MockSep", "MockFixedIntBlock", "MockVariableIntBlock", "MockRandom"}; - - private static void swapCodec(Codec c, CodecProvider cp) { - Codec prior = null; - try { - prior = cp.lookup(c.name); - } catch (IllegalArgumentException iae) { - } - if (prior != null) { - cp.unregister(prior); - } - cp.register(c); - } - - // returns current default codec - static Codec installTestCodecs(String codec, CodecProvider cp) { - savedDefaultCodec = cp.getDefaultFieldCodec(); - - final boolean codecHasParam; - int codecParam = 0; - if (codec.equals("randomPerField")) { - // lie - codec = "Standard"; - codecHasParam = false; - } else if (codec.equals("random")) { - codec = pickRandomCodec(random); - codecHasParam = false; - } else { - Matcher m = codecWithParam.matcher(codec); - if (m.matches()) { - // codec has a fixed param - codecHasParam = true; - codec = m.group(1); - codecParam = Integer.parseInt(m.group(2)); - } else { - codecHasParam = false; - } - } - - cp.setDefaultFieldCodec(codec); - - if (codec.equals("PreFlex")) { - // If we're running w/ PreFlex codec we must swap in the - // test-only PreFlexRW codec (since core PreFlex can - // only read segments): - swapCodec(new PreFlexRWCodec(), cp); - } - - swapCodec(new MockSepCodec(), cp); - // TODO: make it possible to specify min/max iterms per - // block via CL: - int minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); - int maxItemsPerBlock = 2*(Math.max(2, minItemsPerBlock-1)) + random.nextInt(100); - swapCodec(new PulsingCodec(codecHasParam && "Pulsing".equals(codec) ? codecParam : 1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock), cp); - swapCodec(new MockFixedIntBlockCodec(codecHasParam && "MockFixedIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 2000)), cp); - // baseBlockSize cannot be over 127: - swapCodec(new MockVariableIntBlockCodec(codecHasParam && "MockVariableIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 127)), cp); - swapCodec(new MockRandomCodec(random), cp); - - return cp.lookup(codec); - } - - // returns current PreFlex codec - static void removeTestCodecs(Codec codec, CodecProvider cp) { - if (codec.name.equals("PreFlex")) { - final Codec preFlex = cp.lookup("PreFlex"); - if (preFlex != null) { - cp.unregister(preFlex); - } - cp.register(new PreFlexCodec()); - } - cp.unregister(cp.lookup("MockSep")); - cp.unregister(cp.lookup("MockFixedIntBlock")); - cp.unregister(cp.lookup("MockVariableIntBlock")); - cp.unregister(cp.lookup("MockRandom")); - swapCodec(new PulsingCodec(), cp); - cp.setDefaultFieldCodec(savedDefaultCodec); - } - - // randomly picks from core and test codecs - static String pickRandomCodec(Random rnd) { - int idx = rnd.nextInt(CodecProvider.CORE_CODECS.length + - TEST_CODECS.length); - if (idx < CodecProvider.CORE_CODECS.length) { - return CodecProvider.CORE_CODECS[idx]; - } else { - return TEST_CODECS[idx - CodecProvider.CORE_CODECS.length]; - } - } - /** @deprecated (4.0) until we fix no-fork problems in solr tests */ @Deprecated static List testClassesRun = new ArrayList(); @@ -342,40 +253,44 @@ public abstract class LuceneTestCase extends Assert { System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockDirectoryFactory"); } - savedCodecProvider = CodecProvider.getDefault(); - if ("random".equals(TEST_CODECPROVIDER)) { - if ("randomPerField".equals(TEST_CODEC)) { - if (random.nextInt(4) == 0) { // preflex-only setup - codec = installTestCodecs("PreFlex", CodecProvider.getDefault()); - } else { // per-field setup - CodecProvider.setDefault(new RandomCodecProvider(random, useNoMemoryExpensiveCodec)); - codec = installTestCodecs(TEST_CODEC, CodecProvider.getDefault()); - } - } else { // ordinary setup - codec = installTestCodecs(TEST_CODEC, CodecProvider.getDefault()); + // if verbose: print some debugging stuff about which codecs are loaded + if (VERBOSE) { + Set codecs = Codec.availableCodecs(); + for (String codec : codecs) { + System.out.println("Loaded codec: '" + codec + "': " + Codec.forName(codec).getClass().getName()); + } + + Set postingsFormats = PostingsFormat.availablePostingsFormats(); + for (String postingsFormat : postingsFormats) { + System.out.println("Loaded postingsFormat: '" + postingsFormat + "': " + PostingsFormat.forName(postingsFormat).getClass().getName()); } + } + + PREFLEX_IMPERSONATION_IS_ACTIVE = false; + savedCodec = Codec.getDefault(); + final Codec codec; + if ("Lucene3x".equals(TEST_POSTINGSFORMAT) || ("random".equals(TEST_POSTINGSFORMAT) && random.nextInt(4) == 0)) { // preflex-only setup + codec = new PreFlexRWCodec(); + PREFLEX_IMPERSONATION_IS_ACTIVE = true; + } else if ("random".equals(TEST_POSTINGSFORMAT)) { + codec = new RandomCodec(random, useNoMemoryExpensiveCodec); } else { - // someone specified their own codecprovider by class - try { - Class cpClazz = Class.forName(TEST_CODECPROVIDER).asSubclass(CodecProvider.class); - CodecProvider cp = cpClazz.newInstance(); - String codecName; - if (TEST_CODEC.startsWith("random")) { // TODO: somehow do random per-field?! - Set codecSet = cp.listAll(); - String availableCodecs[] = codecSet.toArray(new String[codecSet.size()]); - codecName = availableCodecs[random.nextInt(availableCodecs.length)]; - } else { - codecName = TEST_CODEC; - } + codec = new Lucene40Codec() { + private final PostingsFormat format = PostingsFormat.forName(TEST_POSTINGSFORMAT); - codec = cp.lookup(codecName); - cp.setDefaultFieldCodec(codecName); - CodecProvider.setDefault(cp); - } catch (Exception e) { - System.err.println("Could not instantiate CodecProvider: " + TEST_CODECPROVIDER); - throw new RuntimeException(e); - } + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return format; + } + + @Override + public String toString() { + return super.toString() + ": " + format.toString(); + } + }; } + + Codec.setDefault(codec); savedLocale = Locale.getDefault(); @@ -400,11 +315,6 @@ public abstract class LuceneTestCase extends Assert { TimeZone.setDefault(timeZone); similarityProvider = new RandomSimilarityProvider(random); testsFailed = false; - - // verify assertions are enabled (do last, for smooth cleanup) - if (!Boolean.parseBoolean(System.getProperty("tests.asserts.gracious", "false"))) { - assertTrue("assertions are not enabled!", assertionsEnabled()); - } } @AfterClass @@ -430,7 +340,8 @@ public abstract class LuceneTestCase extends Assert { } } - String codecDescription = uninstallCodecsAfterClass(); + String codecDescription = Codec.getDefault().toString(); + Codec.setDefault(savedCodec); Locale.setDefault(savedLocale); TimeZone.setDefault(savedTimeZone); System.clearProperty("solr.solr.home"); @@ -495,24 +406,6 @@ public abstract class LuceneTestCase extends Assert { + "total=" + Runtime.getRuntime().totalMemory()); } - /** uninstalls test codecs, returns description of the codec used for debugging */ - private static String uninstallCodecsAfterClass() { - String codecDescription; - CodecProvider cp = CodecProvider.getDefault(); - - if ("randomPerField".equals(TEST_CODEC) && cp instanceof RandomCodecProvider) { - codecDescription = cp.toString(); - } else { - codecDescription = codec.toString(); - } - - if ("random".equals(TEST_CODECPROVIDER) && CodecProvider.getDefault() == savedCodecProvider) - removeTestCodecs(codec, CodecProvider.getDefault()); - CodecProvider.setDefault(savedCodecProvider); - - return codecDescription; - } - /** check that directories and their resources were closed */ private static void checkResourcesAfterClass() { for (MockDirectoryWrapper d : stores.keySet()) { @@ -645,12 +538,12 @@ public abstract class LuceneTestCase extends Assert { } if (useNoMemoryExpensiveCodec) { - final String defCodec = CodecProvider.getDefault().getDefaultFieldCodec(); + String defFormat = _TestUtil.getPostingsFormat("thisCodeMakesAbsolutelyNoSenseCanWeDeleteIt"); // Stupid: assumeFalse in setUp() does not print any information, because // TestWatchman does not watch test during setUp() - getName() is also not defined... // => print info directly and use assume without message: - if ("SimpleText".equals(defCodec) || "Memory".equals(defCodec)) { - System.err.println("NOTE: A test method in " + getClass().getSimpleName() + " was ignored, as it uses too much memory with " + defCodec + "."); + if ("SimpleText".equals(defFormat) || "Memory".equals(defFormat)) { + System.err.println("NOTE: A test method in " + getClass().getSimpleName() + " was ignored, as it uses too much memory with " + defFormat + "."); Assume.assumeTrue(false); } } @@ -1380,7 +1273,7 @@ public abstract class LuceneTestCase extends Assert { // extra params that were overridden needed to reproduce the command private static String reproduceWithExtraParams() { StringBuilder sb = new StringBuilder(); - if (!TEST_CODEC.equals("randomPerField")) sb.append(" -Dtests.codec=").append(TEST_CODEC); + if (!TEST_POSTINGSFORMAT.equals("random")) sb.append(" -Dtests.postingsformat=").append(TEST_POSTINGSFORMAT); if (!TEST_LOCALE.equals("random")) sb.append(" -Dtests.locale=").append(TEST_LOCALE); if (!TEST_TIMEZONE.equals("random")) sb.append(" -Dtests.timezone=").append(TEST_TIMEZONE); if (!TEST_DIRECTORY.equals("random")) sb.append(" -Dtests.directory=").append(TEST_DIRECTORY); @@ -1447,15 +1340,4 @@ public abstract class LuceneTestCase extends Assert { @Ignore("just a hack") public final void alwaysIgnoredTestMethod() {} - - /** check if assertions are enabled */ - private static boolean assertionsEnabled() { - try { - assert Boolean.FALSE.booleanValue(); - return false; // should never get here - } catch (AssertionError e) { - return true; - } - } - } diff --git a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java b/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java rename to lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java diff --git a/lucene/src/test-framework/org/apache/lucene/util/SmartRandom.java b/lucene/src/test-framework/java/org/apache/lucene/util/SmartRandom.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/util/SmartRandom.java rename to lucene/src/test-framework/java/org/apache/lucene/util/SmartRandom.java diff --git a/lucene/src/test-framework/org/apache/lucene/util/ThreeLongs.java b/lucene/src/test-framework/java/org/apache/lucene/util/ThreeLongs.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/util/ThreeLongs.java rename to lucene/src/test-framework/java/org/apache/lucene/util/ThreeLongs.java diff --git a/lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java b/lucene/src/test-framework/java/org/apache/lucene/util/ThrottledIndexOutput.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java rename to lucene/src/test-framework/java/org/apache/lucene/util/ThrottledIndexOutput.java diff --git a/lucene/src/test-framework/org/apache/lucene/util/_TestIgnoredException.java b/lucene/src/test-framework/java/org/apache/lucene/util/_TestIgnoredException.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/util/_TestIgnoredException.java rename to lucene/src/test-framework/java/org/apache/lucene/util/_TestIgnoredException.java diff --git a/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java b/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java similarity index 94% rename from lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java rename to lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java index b298aad..c736503 100644 --- a/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java +++ b/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java @@ -45,7 +45,9 @@ import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; import org.apache.lucene.search.FieldDoc; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TopDocs; @@ -144,17 +146,10 @@ public class _TestUtil { * issues are hit, a RuntimeException is thrown; else, * true is returned. */ public static CheckIndex.Status checkIndex(Directory dir) throws IOException { - return checkIndex(dir, CodecProvider.getDefault()); - } - - /** This runs the CheckIndex tool on the index in. If any - * issues are hit, a RuntimeException is thrown; else, - * true is returned. */ - public static CheckIndex.Status checkIndex(Directory dir, CodecProvider codecs) throws IOException { ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); CheckIndex checker = new CheckIndex(dir); checker.setInfoStream(new PrintStream(bos), false); - CheckIndex.Status indexStatus = checker.checkIndex(null, codecs); + CheckIndex.Status indexStatus = checker.checkIndex(null); if (indexStatus == null || indexStatus.clean == false) { System.out.println("CheckIndex failed"); System.out.println(bos.toString()); @@ -351,28 +346,28 @@ public class _TestUtil { return new String(buffer, 0, i); } - public static CodecProvider alwaysCodec(final Codec c) { - CodecProvider p = new CodecProvider() { + + /** Return a Codec that can read any of the + * default codecs and formats, but always writes in the specified + * format. */ + public static Codec alwaysPostingsFormat(final PostingsFormat format) { + return new Lucene40Codec() { @Override - public Codec lookup(String name) { - // can't do this until we fix PreFlexRW to not - //impersonate PreFlex: - if (name.equals(c.name)) { - return c; - } else { - return CodecProvider.getDefault().lookup(name); - } + public PostingsFormat getPostingsFormatForField(String field) { + return format; } }; - p.setDefaultFieldCodec(c.name); - return p; } - /** Return a CodecProvider that can read any of the - * default codecs, but always writes in the specified - * codec. */ - public static CodecProvider alwaysCodec(final String codec) { - return alwaysCodec(CodecProvider.getDefault().lookup(codec)); + // TODO: generalize all 'test-checks-for-crazy-codecs' to + // annotations (LUCENE-3489) + public static String getPostingsFormat(String field) { + PostingsFormat p = Codec.getDefault().postingsFormat(); + if (p instanceof PerFieldPostingsFormat) { + return ((PerFieldPostingsFormat)p).getPostingsFormatForField(field).getName(); + } else { + return p.getName(); + } } public static boolean anyFilesExceptWriteLock(Directory dir) throws IOException { diff --git a/lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java b/lucene/src/test-framework/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java rename to lucene/src/test-framework/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java diff --git a/lucene/src/test-framework/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java b/lucene/src/test-framework/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java rename to lucene/src/test-framework/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java diff --git a/lucene/src/test-framework/org/apache/lucene/util/europarl.lines.txt.gz b/lucene/src/test-framework/java/org/apache/lucene/util/europarl.lines.txt.gz similarity index 100% rename from lucene/src/test-framework/org/apache/lucene/util/europarl.lines.txt.gz rename to lucene/src/test-framework/java/org/apache/lucene/util/europarl.lines.txt.gz diff --git a/lucene/src/test-framework/overview.html b/lucene/src/test-framework/java/overview.html similarity index 100% rename from lucene/src/test-framework/overview.html rename to lucene/src/test-framework/java/overview.html diff --git a/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java b/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java deleted file mode 100644 index b92ce7a..0000000 diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java deleted file mode 100644 index ee471e5..0000000 diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java deleted file mode 100644 index 0b24ed4..0000000 diff --git a/lucene/src/test-framework/resources/META-INF/services/org.apache.lucene.index.codecs.Codec b/lucene/src/test-framework/resources/META-INF/services/org.apache.lucene.index.codecs.Codec new file mode 100644 index 0000000..fdd6f8e --- /dev/null +++ b/lucene/src/test-framework/resources/META-INF/services/org.apache.lucene.index.codecs.Codec @@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec diff --git a/lucene/src/test-framework/resources/META-INF/services/org.apache.lucene.index.codecs.PostingsFormat b/lucene/src/test-framework/resources/META-INF/services/org.apache.lucene.index.codecs.PostingsFormat new file mode 100644 index 0000000..505d010 --- /dev/null +++ b/lucene/src/test-framework/resources/META-INF/services/org.apache.lucene.index.codecs.PostingsFormat @@ -0,0 +1,22 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockPostingsFormat +org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockPostingsFormat +org.apache.lucene.index.codecs.mockrandom.MockRandomPostingsFormat +org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat +org.apache.lucene.index.codecs.nestedpulsing.NestedPulsingPostingsFormat +org.apache.lucene.index.codecs.ramonly.RAMOnlyPostingsFormat +org.apache.lucene.index.codecs.lucene40ords.Lucene40WithOrds diff --git a/lucene/src/test/org/apache/lucene/TestExternalCodecs.java b/lucene/src/test/org/apache/lucene/TestExternalCodecs.java dissimilarity index 77% index b738976..7269691 100644 --- a/lucene/src/test/org/apache/lucene/TestExternalCodecs.java +++ b/lucene/src/test/org/apache/lucene/TestExternalCodecs.java @@ -?,? +1,126 @@ +package org.apache.lucene; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.*; +import java.util.*; + +import org.apache.lucene.analysis.*; +import org.apache.lucene.document.*; +import org.apache.lucene.index.*; +import org.apache.lucene.index.codecs.*; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.search.*; +import org.apache.lucene.store.*; +import org.apache.lucene.util.*; +import org.apache.lucene.util.Bits; + +/* Intentionally outside of oal.index to verify fully + external codecs work fine */ + +public class TestExternalCodecs extends LuceneTestCase { + + private static final class CustomPerFieldCodec extends Lucene40Codec { + + private final PostingsFormat ramFormat = PostingsFormat.forName("RAMOnly"); + private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene40"); + private final PostingsFormat pulsingFormat = PostingsFormat.forName("Pulsing40"); + + @Override + public PostingsFormat getPostingsFormatForField(String field) { + if (field.equals("field2") || field.equals("id")) { + return pulsingFormat; + } else if (field.equals("field1")) { + return defaultFormat; + } else { + return ramFormat; + } + } + } + + // tests storing "id" and "field2" fields as pulsing codec, + // whose term sort is backwards unicode code point, and + // storing "field1" as a custom entirely-in-RAM codec + public void testPerFieldCodec() throws Exception { + + final int NUM_DOCS = atLeast(173); + MockDirectoryWrapper dir = newDirectory(); + dir.setCheckIndexOnClose(false); // we use a custom codec provider + IndexWriter w = new IndexWriter( + dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). + setCodec(new CustomPerFieldCodec()). + setMergePolicy(newLogMergePolicy(3)) + ); + w.setInfoStream(VERBOSE ? System.out : null); + Document doc = new Document(); + // uses default codec: + doc.add(newField("field1", "this field uses the standard codec as the test", TextField.TYPE_UNSTORED)); + // uses pulsing codec: + Field field2 = newField("field2", "this field uses the pulsing codec as the test", TextField.TYPE_UNSTORED); + doc.add(field2); + + Field idField = newField("id", "", StringField.TYPE_UNSTORED); + + doc.add(idField); + for(int i=0;i Integer.MAX_VALUE terms/docs pairs * @lucene.experimental */ +@UseNoMemoryExpensiveCodec public class Test2BPostings extends LuceneTestCase { @Nightly public void test() throws Exception { - - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory")); - assumeFalse("This test is super-slow and very disk-space-consuming with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText")); - MockDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BPostings")); dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER); dir.setCheckIndexOnClose(false); // don't double-checkindex diff --git a/lucene/src/test/org/apache/lucene/index/Test2BTerms.java b/lucene/src/test/org/apache/lucene/index/Test2BTerms.java index c146ba1..03a3031 100644 --- a/lucene/src/test/org/apache/lucene/index/Test2BTerms.java +++ b/lucene/src/test/org/apache/lucene/index/Test2BTerms.java @@ -24,7 +24,8 @@ import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.*; import org.apache.lucene.document.*; import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; + import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -142,7 +143,7 @@ public class Test2BTerms extends LuceneTestCase { @Ignore("Takes ~4 hours to run on a fast machine!! And requires that you don't use PreFlex codec.") public void test2BTerms() throws IOException { - if ("PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())) { + if ("Lucene3x".equals(Codec.getDefault().getName())) { throw new RuntimeException("thist test cannot run with PreFlex codec"); } diff --git a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java index 909609c..3e9f14b 100755 --- a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java +++ b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java @@ -17,9 +17,10 @@ package org.apache.lucene.index; * limitations under the License. */ -import java.io.IOException; import java.io.FileNotFoundException; +import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; @@ -29,11 +30,22 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.mocksep.MockSepCodec; -import org.apache.lucene.index.codecs.pulsing.PulsingCodec; -import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; -import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesFormat; +import org.apache.lucene.index.codecs.DefaultFieldsFormat; +import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; +import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldsFormat; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.SegmentInfosFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsBaseFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.index.codecs.pulsing.Pulsing40PostingsFormat; +import org.apache.lucene.index.codecs.pulsing.PulsingPostingsFormat; +import org.apache.lucene.index.codecs.simpletext.SimpleTextPostingsFormat; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.store.AlreadyClosedException; @@ -975,30 +987,29 @@ public class TestAddIndexes extends LuceneTestCase { } } - public void testSimpleCaseCustomCodecProvider() throws IOException { + public void testSimpleCaseCustomCodec() throws IOException { // main directory Directory dir = newDirectory(); // two auxiliary directories Directory aux = newDirectory(); Directory aux2 = newDirectory(); - CodecProvider provider = new MockCodecProvider(); + Codec codec = new CustomPerFieldCodec(); IndexWriter writer = null; writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodecProvider( - provider)); + new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodec(codec)); // add 100 documents addDocs3(writer, 100); assertEquals(100, writer.maxDoc()); writer.commit(); writer.close(); - _TestUtil.checkIndex(dir, provider); + _TestUtil.checkIndex(dir); writer = newWriter( aux, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.CREATE). - setCodecProvider(provider). + setCodec(codec). setMaxBufferedDocs(10). setMergePolicy(newLogMergePolicy(false)) ); @@ -1012,7 +1023,7 @@ public class TestAddIndexes extends LuceneTestCase { aux2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.CREATE). - setCodecProvider(provider) + setCodec(codec) ); // add 40 documents in compound files addDocs2(writer, 50); @@ -1025,7 +1036,7 @@ public class TestAddIndexes extends LuceneTestCase { dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.APPEND). - setCodecProvider(provider) + setCodec(codec) ); assertEquals(100, writer.maxDoc()); writer.addIndexes(aux, aux2); @@ -1037,19 +1048,24 @@ public class TestAddIndexes extends LuceneTestCase { aux2.close(); } - public static class MockCodecProvider extends CodecProvider { - public MockCodecProvider() { - StandardCodec standardCodec = new StandardCodec(); - SimpleTextCodec simpleTextCodec = new SimpleTextCodec(); - MockSepCodec mockSepCodec = new MockSepCodec(); - register(standardCodec); - register(mockSepCodec); - register(simpleTextCodec); - setFieldCodec("id", simpleTextCodec.name); - setFieldCodec("content", mockSepCodec.name); + private static final class CustomPerFieldCodec extends Lucene40Codec { + private final PostingsFormat simpleTextFormat = PostingsFormat.forName("SimpleText"); + private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene40"); + private final PostingsFormat mockSepFormat = PostingsFormat.forName("MockSep"); + + @Override + public PostingsFormat getPostingsFormatForField(String field) { + if (field.equals("id")) { + return simpleTextFormat; + } else if (field.equals("content")) { + return mockSepFormat; + } else { + return defaultFormat; + } } } + // LUCENE-2790: tests that the non CFS files were deleted by addIndexes public void testNonCFSLeftovers() throws Exception { Directory[] dirs = new Directory[2]; @@ -1066,16 +1082,19 @@ public class TestAddIndexes extends LuceneTestCase { IndexReader[] readers = new IndexReader[] { IndexReader.open(dirs[0]), IndexReader.open(dirs[1]) }; - Directory dir = new RAMDirectory(); + Directory dir = new MockDirectoryWrapper(random, new RAMDirectory()); IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(true); lmp.setNoCFSRatio(1.0); // Force creation of CFS IndexWriter w3 = new IndexWriter(dir, conf); + w3.setInfoStream(VERBOSE ? System.out : null); w3.addIndexes(readers); w3.close(); - // we should now see segments_X, segments.gen,_Y.cfs,_Y.cfe, _Z.fnx - assertEquals("Only one compound segment should exist", 5, dir.listAll().length); + // we should now see segments_X, + // segments.gen,_Y.cfs,_Y.cfe, _Z.fnx + assertEquals("Only one compound segment should exist, but got: " + Arrays.toString(dir.listAll()), 5, dir.listAll().length); + dir.close(); } // LUCENE-3126: tests that if a non-CFS segment is copied, it is converted to @@ -1136,18 +1155,45 @@ public class TestAddIndexes extends LuceneTestCase { src.close(); target.close(); } + + private static class UnRegisteredCodec extends Codec { + public UnRegisteredCodec() { + super("NotRegistered"); + } + + @Override + public PostingsFormat postingsFormat() { + return PostingsFormat.forName("Lucene40"); + } + + @Override + public DocValuesFormat docValuesFormat() { + return new DefaultDocValuesFormat(); + } + + @Override + public FieldsFormat fieldsFormat() { + return new DefaultFieldsFormat(); + } + + @Override + public SegmentInfosFormat segmentInfosFormat() { + return new DefaultSegmentInfosFormat(); + } + } /* * simple test that ensures we getting expected exceptions */ public void testAddIndexMissingCodec() throws IOException { - Directory toAdd = newDirectory(); + MockDirectoryWrapper toAdd = newDirectory(); + // Disable checkIndex, else we get an exception because + // of the unregistered codec: + toAdd.setCheckIndexOnClose(false); { IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); - CodecProvider provider = new CodecProvider(); - provider.register(new StandardCodec()); - conf.setCodecProvider(provider); + conf.setCodec(new UnRegisteredCodec()); IndexWriter w = new IndexWriter(toAdd, conf); Document doc = new Document(); FieldType customType = new FieldType(); @@ -1156,13 +1202,12 @@ public class TestAddIndexes extends LuceneTestCase { w.addDocument(doc); w.close(); } + { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); - CodecProvider provider = new CodecProvider(); - provider.register(new PulsingCodec(1 + random.nextInt(20))); - conf.setCodecProvider(provider); + conf.setCodec(_TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1 + random.nextInt(20)))); IndexWriter w = new IndexWriter(dir, conf); try { w.addIndexes(toAdd); @@ -1177,27 +1222,11 @@ public class TestAddIndexes extends LuceneTestCase { dir.close(); } - { - Directory dir = newDirectory(); - IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)); - CodecProvider provider = new CodecProvider(); - provider.register(new PulsingCodec(1 + random.nextInt(20))); - conf.setCodecProvider(provider); - IndexWriter w = new IndexWriter(dir, conf); + try { IndexReader indexReader = IndexReader.open(toAdd); - try { - w.addIndexes(indexReader); - fail("no such codec"); - } catch (IllegalArgumentException ex) { - // expected - } - indexReader.close(); - w.close(); - IndexReader open = IndexReader.open(dir); - assertEquals(0, open.numDocs()); - open.close(); - dir.close(); + fail("no such codec"); + } catch (IllegalArgumentException ex) { + // expected } toAdd.close(); } diff --git a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java index 8cd628e..f2cde36 100644 --- a/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java +++ b/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java @@ -568,7 +568,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase { "_0_1.s" + contentFieldIndex, "segments_2", "segments.gen", - "1.fnx"}; + "_1.fnx"}; String[] actual = dir.listAll(); Arrays.sort(expected); diff --git a/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java b/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java index d05b8d0..83dc9b1 100644 --- a/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java +++ b/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java @@ -23,7 +23,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; @@ -37,7 +37,7 @@ import org.apache.lucene.util.LuceneTestCase; public class TestBinaryTerms extends LuceneTestCase { public void testBinary() throws IOException { assumeFalse("PreFlex codec cannot work with binary terms!", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + Codec.getDefault().getName().equals("Lucene3x")); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir); diff --git a/lucene/src/test/org/apache/lucene/index/TestCodecs.java b/lucene/src/test/org/apache/lucene/index/TestCodecs.java index 8db99ab..3cce510 100644 --- a/lucene/src/test/org/apache/lucene/index/TestCodecs.java +++ b/lucene/src/test/org/apache/lucene/index/TestCodecs.java @@ -26,14 +26,15 @@ import org.apache.lucene.document.Document; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.PostingsConsumer; import org.apache.lucene.index.codecs.TermStats; import org.apache.lucene.index.codecs.TermsConsumer; -import org.apache.lucene.index.codecs.mocksep.MockSepCodec; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xCodec; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; @@ -64,6 +65,7 @@ import org.junit.BeforeClass; // goes to 1 before next one known to exist // - skipTo(term) // - skipTo(doc) + public class TestCodecs extends LuceneTestCase { private static String[] fieldNames = new String[] {"one", "two", "three", "four"}; @@ -255,9 +257,10 @@ public class TestCodecs extends LuceneTestCase { final Directory dir = newDirectory(); FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone(); this.write(fieldInfos, dir, fields, true); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, clonedFieldInfos.buildSegmentCodecs(false), clonedFieldInfos); + Codec codec = Codec.getDefault(); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, codec, clonedFieldInfos); - final FieldsProducer reader = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); + final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); final FieldsEnum fieldsEnum = reader.iterator(); assertNotNull(fieldsEnum.next()); @@ -307,12 +310,13 @@ public class TestCodecs extends LuceneTestCase { FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone(); this.write(fieldInfos, dir, fields, false); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, clonedFieldInfos.buildSegmentCodecs(false), clonedFieldInfos); + Codec codec = Codec.getDefault(); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, codec, clonedFieldInfos); if (VERBOSE) { System.out.println("TEST: now read postings"); } - final FieldsProducer terms = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); + final FieldsProducer terms = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); final Verify[] threads = new Verify[NUM_TEST_THREADS-1]; for(int i=0;i filesToDelete = merger.createCompoundFile(merged + ".cfs", info, newIOContext(random)); diff --git a/lucene/src/test/org/apache/lucene/index/TestDocCount.java b/lucene/src/test/org/apache/lucene/index/TestDocCount.java index 4c12aa0..bba8e30 100644 --- a/lucene/src/test/org/apache/lucene/index/TestDocCount.java +++ b/lucene/src/test/org/apache/lucene/index/TestDocCount.java @@ -19,7 +19,7 @@ package org.apache.lucene.index; import org.apache.lucene.document.Document; import org.apache.lucene.document.StringField; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.util.FixedBitSet; @@ -32,7 +32,7 @@ import org.apache.lucene.util._TestUtil; public class TestDocCount extends LuceneTestCase { public void testSimple() throws Exception { assumeFalse("PreFlex codec does not support docCount statistic!", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + "Lucene3x".equals(Codec.getDefault().getName())); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir); int numDocs = atLeast(100); diff --git a/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java b/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java index eb3641a..aef44ab 100644 --- a/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java +++ b/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java @@ -34,21 +34,17 @@ import org.apache.lucene.index.DocTermOrds.TermOrdsIterator; import org.apache.lucene.index.codecs.BlockTermsReader; import org.apache.lucene.index.codecs.BlockTermsWriter; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CoreCodecProvider; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.TermsIndexReaderBase; import org.apache.lucene.index.codecs.TermsIndexWriterBase; -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; -import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; @@ -106,130 +102,6 @@ public class TestDocTermOrds extends LuceneTestCase { dir.close(); } - private static class StandardCodecWithOrds extends Codec { - - public StandardCodecWithOrds() { - super("StandardOrds"); - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase docs = new StandardPostingsWriter(state); - - // TODO: should we make the terms index more easily - // pluggable? Ie so that this codec would record which - // index impl was used, and switch on loading? - // Or... you must make a new Codec for this? - TermsIndexWriterBase indexWriter; - boolean success = false; - try { - indexWriter = new FixedGapTermsIndexWriter(state); - success = true; - } finally { - if (!success) { - docs.close(); - } - } - - success = false; - try { - FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs); - success = true; - return ret; - } finally { - if (!success) { - try { - docs.close(); - } finally { - indexWriter.close(); - } - } - } - } - - public final static int TERMS_CACHE_SIZE = 1024; - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); - TermsIndexReaderBase indexReader; - - boolean success = false; - try { - indexReader = new FixedGapTermsIndexReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId, state.context); - success = true; - } finally { - if (!success) { - postings.close(); - } - } - - success = false; - try { - FieldsProducer ret = new BlockTermsReader(indexReader, - state.dir, - state.fieldInfos, - state.segmentInfo.name, - postings, - state.context, - TERMS_CACHE_SIZE, - state.codecId); - success = true; - return ret; - } finally { - if (!success) { - try { - postings.close(); - } finally { - indexReader.close(); - } - } - } - } - - /** Extension of freq postings file */ - static final String FREQ_EXTENSION = "frq"; - - /** Extension of prox postings file */ - static final String PROX_EXTENSION = "prx"; - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, id, files); - BlockTermsReader.files(dir, segmentInfo, id, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, id, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); - } - - @Override - public void getExtensions(Set extensions) { - getStandardExtensions(extensions); - DefaultDocValuesConsumer.getExtensions(extensions); - } - - public static void getStandardExtensions(Set extensions) { - extensions.add(FREQ_EXTENSION); - extensions.add(PROX_EXTENSION); - BlockTermsReader.getExtensions(extensions); - FixedGapTermsIndexReader.getIndexExtensions(extensions); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); - } - } - public void testRandom() throws Exception { MockDirectoryWrapper dir = newDirectory(); @@ -252,13 +124,8 @@ public class TestDocTermOrds extends LuceneTestCase { // Sometimes swap in codec that impls ord(): if (random.nextInt(10) == 7) { // Make sure terms index has ords: - CoreCodecProvider cp = new CoreCodecProvider(); - cp.register(new StandardCodecWithOrds()); - cp.setDefaultFieldCodec("StandardOrds"); - - // So checkIndex on close works - dir.setCodecProvider(cp); - conf.setCodecProvider(cp); + Codec codec = _TestUtil.alwaysPostingsFormat(PostingsFormat.forName("Lucene40WithOrds")); + conf.setCodec(codec); } final RandomIndexWriter w = new RandomIndexWriter(random, dir, conf); @@ -354,14 +221,8 @@ public class TestDocTermOrds extends LuceneTestCase { // Sometimes swap in codec that impls ord(): if (random.nextInt(10) == 7) { - // Make sure terms index has ords: - CoreCodecProvider cp = new CoreCodecProvider(); - cp.register(new StandardCodecWithOrds()); - cp.setDefaultFieldCodec("StandardOrds"); - - // So checkIndex on close works - dir.setCodecProvider(cp); - conf.setCodecProvider(cp); + Codec codec = _TestUtil.alwaysPostingsFormat(PostingsFormat.forName("Lucene40WithOrds")); + conf.setCodec(codec); } final RandomIndexWriter w = new RandomIndexWriter(random, dir, conf); diff --git a/lucene/src/test/org/apache/lucene/index/TestFlex.java b/lucene/src/test/org/apache/lucene/index/TestFlex.java index 7714afb..90e4628 100644 --- a/lucene/src/test/org/apache/lucene/index/TestFlex.java +++ b/lucene/src/test/org/apache/lucene/index/TestFlex.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import org.apache.lucene.store.*; import org.apache.lucene.analysis.*; import org.apache.lucene.document.*; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.util.*; public class TestFlex extends LuceneTestCase { @@ -64,7 +65,7 @@ public class TestFlex extends LuceneTestCase { public void testTermOrd() throws Exception { Directory d = newDirectory(); IndexWriter w = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); Document doc = new Document(); doc.add(newField("f", "a b c", TextField.TYPE_UNSTORED)); w.addDocument(doc); diff --git a/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java b/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java index ad997d1..5bdd9b0 100644 --- a/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java +++ b/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java @@ -22,7 +22,6 @@ import java.util.*; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TopDocs; @@ -36,6 +35,9 @@ public class TestForTooMuchCloning extends LuceneTestCase { // Make sure we don't clone IndexInputs too frequently // during merging: public void test() throws Exception { + // NOTE: if we see a fail on this test with "NestedPulsing" its because its + // reuse isnt perfect (but reasonable). see TestPulsingReuse.testNestedPulsing + // for more details final MockDirectoryWrapper dir = newDirectory(); final TieredMergePolicy tmp = new TieredMergePolicy(); tmp.setMaxMergeAtOnce(2); diff --git a/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java b/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java index a7ac933..54e10fd 100644 --- a/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java +++ b/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java @@ -58,25 +58,25 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { } writer.commit(); Collection files = writer.getIndexFileNames(); - files.remove("1.fnx"); + files.remove("_1.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); writer.commit(); files = writer.getIndexFileNames(); - files.remove("2.fnx"); + files.remove("_2.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } { @@ -89,12 +89,12 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { writer.addDocument(d); writer.close(); Collection files = writer.getIndexFileNames(); - files.remove("2.fnx"); + files.remove("_2.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( @@ -102,7 +102,7 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); dir.close(); } @@ -121,29 +121,29 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } IndexReader reader = IndexReader.open(dir, false); reader.deleteDocument(0); reader.commit(); reader.close(); // make sure this reader can not modify the field map - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); dir.close(); } @@ -162,7 +162,7 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); @@ -170,9 +170,9 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { writer.commit(); writer.commit(); writer.commit(); - assertFNXFiles(dir, "1.fnx", "2.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx"); writer.close(); - assertFNXFiles(dir, "1.fnx", "2.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx"); } { @@ -184,14 +184,14 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { d.add(new BinaryField("f3", new byte[] { 1, 2, 3, 4, 5 })); writer.addDocument(d); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); dir.close(); } } @@ -208,14 +208,14 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); writer.close(); - assertFNXFiles(dir, "1.fnx", "2.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx"); // open first commit List listCommits = IndexReader.listCommits(dir); assertEquals(2, listCommits.size()); @@ -229,18 +229,18 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { writer.addDocument(d); writer.commit(); // now we have 3 files since f3 is not present in the first commit - assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx", "_3.fnx"); writer.close(); - assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx", "_3.fnx"); writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.commit(); listCommits = IndexReader.listCommits(dir); assertEquals(1, listCommits.size()); - assertFNXFiles(dir, "3.fnx"); + assertFNXFiles(dir, "_3.fnx"); writer.close(); - assertFNXFiles(dir, "3.fnx"); + assertFNXFiles(dir, "_3.fnx"); dir.close(); } } @@ -494,7 +494,7 @@ public class TestGlobalFieldNumbers extends LuceneTestCase { assertEquals(1, segmentInfos.getGlobalFieldMapVersion()); assertEquals(1, segmentInfos.getLastGlobalFieldMapVersion()); files = writer.getIndexFileNames(); - assertTrue(files.remove("1.fnx")); + assertTrue(files.remove("_1.fnx")); for (String string : files) { assertFalse(string.endsWith(".fnx")); } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexReader.java b/lucene/src/test/org/apache/lucene/index/TestIndexReader.java index 7d10fa1..024dd49 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexReader.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexReader.java @@ -39,7 +39,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader.FieldOption; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldCache; @@ -88,6 +88,7 @@ public class TestIndexReader extends LuceneTestCase writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode( OpenMode.APPEND).setMaxBufferedDocs(2)); + writer.setInfoStream(VERBOSE ? System.out : null); for(int i=0;i<7;i++) addDocumentWithFields(writer); writer.close(); @@ -947,7 +948,7 @@ public class TestIndexReader extends LuceneTestCase writer.close(); SegmentInfos sis = new SegmentInfos(); - sis.read(d, CodecProvider.getDefault()); + sis.read(d); IndexReader r = IndexReader.open(d, false); IndexCommit c = r.getIndexCommit(); @@ -1231,7 +1232,7 @@ public class TestIndexReader extends LuceneTestCase // LUCENE-1609: don't load terms index public void testNoTermsIndex() throws Throwable { Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); Document doc = new Document(); doc.add(newField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", TextField.TYPE_UNSTORED)); doc.add(newField("number", "0 1 2 3 4 5 6 7 8 9", TextField.TYPE_UNSTORED)); @@ -1251,7 +1252,7 @@ public class TestIndexReader extends LuceneTestCase writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setCodecProvider(_TestUtil.alwaysCodec("Standard")). + setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())). setMergePolicy(newLogMergePolicy(10)) ); writer.addDocument(doc); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java index c58a795..ca85730 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -52,12 +52,17 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.store.NativeFSLockFactory; import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; @@ -1937,4 +1942,22 @@ public class TestIndexWriter extends LuceneTestCase { assert(version3 > version2); d.close(); } + + public void testWhetherDeleteAllDeletesWriteLock() throws Exception { + Directory d = newFSDirectory(_TestUtil.getTempDir("TestIndexWriter.testWhetherDeleteAllDeletesWriteLock")); + // Must use SimpleFSLockFactory... NativeFSLockFactory + // somehow "knows" a lock is held against write.lock + // even if you remove that file: + d.setLockFactory(new SimpleFSLockFactory()); + RandomIndexWriter w1 = new RandomIndexWriter(random, d); + w1.deleteAll(); + try { + new RandomIndexWriter(random, d); + fail("should not be able to create another writer"); + } catch (LockObtainFailedException lofe) { + // expected + } + w1.close(); + d.close(); + } } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java index 02d7c6d..07759d6 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java @@ -28,7 +28,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -169,8 +168,10 @@ public class TestIndexWriterCommit extends LuceneTestCase { // them, the merged result can easily be larger than the // sum because the merged FST may use array encoding for // some arcs (which uses more space): - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory")); - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory")); + + final String idFormat = _TestUtil.getPostingsFormat("id"); + final String contentFormat = _TestUtil.getPostingsFormat("content"); + assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory")); MockDirectoryWrapper dir = newDirectory(); Analyzer analyzer; if (random.nextBoolean()) { diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java index 816977e..85fd1b4 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java @@ -26,6 +26,7 @@ import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.util.LuceneTestCase; @@ -71,6 +72,7 @@ public class TestIndexWriterConfig extends LuceneTestCase { assertEquals(ThreadAffinityDocumentsWriterThreadPool.class, conf.getIndexerThreadPool().getClass()); assertEquals(FlushByRamOrCountsPolicy.class, conf.getFlushPolicy().getClass()); assertEquals(IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB, conf.getRAMPerThreadHardLimitMB()); + assertEquals(Codec.getDefault(), conf.getCodec()); // Sanity check - validate that all getters are covered. Set getters = new HashSet(); getters.add("getAnalyzer"); @@ -88,7 +90,6 @@ public class TestIndexWriterConfig extends LuceneTestCase { getters.add("getMaxBufferedDocs"); getters.add("getIndexingChain"); getters.add("getMergedSegmentWarmer"); - getters.add("getCodecProvider"); getters.add("getMergePolicy"); getters.add("getMaxThreadStates"); getters.add("getReaderPooling"); @@ -96,6 +97,7 @@ public class TestIndexWriterConfig extends LuceneTestCase { getters.add("getReaderTermsIndexDivisor"); getters.add("getFlushPolicy"); getters.add("getRAMPerThreadHardLimitMB"); + getters.add("getCodec"); for (Method m : IndexWriterConfig.class.getDeclaredMethods()) { if (m.getDeclaringClass() == IndexWriterConfig.class && m.getName().startsWith("get")) { diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java index 2e4365e..a2c5b13 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java @@ -31,7 +31,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -897,8 +896,9 @@ public class TestIndexWriterDelete extends LuceneTestCase { } public void testIndexingThenDeleting() throws Exception { - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory")); - assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText")); + final String fieldFormat = _TestUtil.getPostingsFormat("field"); + assumeFalse("This test cannot run with Memory codec", fieldFormat.equals("Memory")); + assumeFalse("This test cannot run with SimpleText codec", fieldFormat.equals("SimpleText")); final Random r = random; Directory dir = newDirectory(); // note this test explicitly disables payloads diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java index 647d97b..5c04b75 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java @@ -23,6 +23,7 @@ import java.io.PrintStream; import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Random; @@ -927,10 +928,10 @@ public class TestIndexWriterExceptions extends LuceneTestCase { } catch (RuntimeException re) { // Expected } - assertTrue(dir.fileExists("1.fnx")); + assertTrue(dir.fileExists("_1.fnx")); assertTrue(failure.failOnCommit && failure.failOnDeleteFile); w.rollback(); - assertFalse(dir.fileExists("1.fnx")); + assertFalse(dir.fileExists("_1.fnx")); assertEquals(0, dir.listAll().length); dir.close(); } diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java index 357a48b..46abe85 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java @@ -24,7 +24,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -154,8 +154,10 @@ public class TestIndexWriterOnDiskFull extends LuceneTestCase { // them, the merged result can easily be larger than the // sum because the merged FST may use array encoding for // some arcs (which uses more space): - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory")); - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory")); + + final String idFormat = _TestUtil.getPostingsFormat("id"); + final String contentFormat = _TestUtil.getPostingsFormat("content"); + assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory")); int START_COUNT = 57; int NUM_DIR = TEST_NIGHTLY ? 50 : 5; diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java index c012cda..ff5e522 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java @@ -30,6 +30,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -988,15 +989,16 @@ public class TestIndexWriterReader extends LuceneTestCase { public void testNoTermsIndex() throws Exception { // Some Codecs don't honor the ReaderTermsIndexDivisor, so skip the test if // they're picked. - HashSet illegalCodecs = new HashSet(); - illegalCodecs.add("PreFlex"); - illegalCodecs.add("SimpleText"); - illegalCodecs.add("Memory"); + assumeFalse("PreFlex codec does not support ReaderTermsIndexDivisor!", + "Lucene3x".equals(Codec.getDefault().getName())); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setReaderTermsIndexDivisor(-1); + // Don't proceed if picked Codec is in the list of illegal ones. - if (illegalCodecs.contains(conf.getCodecProvider().getFieldCodec("f"))) return; + final String format = _TestUtil.getPostingsFormat("f"); + assumeFalse("Format: " + format + " does not support ReaderTermsIndexDivisor!", + (format.equals("SimpleText") || format.equals("Memory"))); Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, conf); @@ -1006,7 +1008,7 @@ public class TestIndexWriterReader extends LuceneTestCase { IndexReader r = IndexReader.open(w, true).getSequentialSubReaders()[0]; try { r.termDocsEnum(null, "f", new BytesRef("val")); - fail("should have failed to seek since terms index was not loaded. Codec used " + conf.getCodecProvider().getFieldCodec("f")); + fail("should have failed to seek since terms index was not loaded."); } catch (IllegalStateException e) { // expected - we didn't load the term index } finally { diff --git a/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java b/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java index 65787b0..1933a4a 100755 --- a/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java +++ b/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java @@ -23,7 +23,6 @@ import java.io.Reader; import org.apache.lucene.analysis.*; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.ScoreDoc; @@ -34,6 +33,7 @@ import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util._TestUtil; /** * Tests lazy skipping on the proximity file. @@ -131,8 +131,10 @@ public class TestLazyProxSkipping extends LuceneTestCase { } public void testLazySkipping() throws IOException { - assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("SimpleText")); - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("Memory")); + final String fieldFormat = _TestUtil.getPostingsFormat(this.field); + assumeFalse("This test cannot run with Memory codec", fieldFormat.equals("Memory")); + assumeFalse("This test cannot run with SimpleText codec", fieldFormat.equals("SimpleText")); + // test whether only the minimum amount of seeks() // are performed performTest(5); diff --git a/lucene/src/test/org/apache/lucene/index/TestLongPostings.java b/lucene/src/test/org/apache/lucene/index/TestLongPostings.java index 034c6f1..84f5f52 100644 --- a/lucene/src/test/org/apache/lucene/index/TestLongPostings.java +++ b/lucene/src/test/org/apache/lucene/index/TestLongPostings.java @@ -29,7 +29,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; diff --git a/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java b/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java index 0a911ec..67d3027 100644 --- a/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java +++ b/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java @@ -25,6 +25,7 @@ import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -68,7 +69,7 @@ public class TestMultiLevelSkipList extends LuceneTestCase { public void testSimpleSkip() throws IOException { Directory dir = new CountingRAMDirectory(new RAMDirectory()); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new PayloadAnalyzer()).setCodecProvider(_TestUtil.alwaysCodec("Standard")).setMergePolicy(newLogMergePolicy())); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new PayloadAnalyzer()).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())).setMergePolicy(newLogMergePolicy())); Term term = new Term("test", "a"); for (int i = 0; i < 5000; i++) { Document d1 = new Document(); diff --git a/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java b/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java deleted file mode 100644 index f55b50f..0000000 diff --git a/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java b/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java index 1e994d1..9590215 100644 --- a/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java +++ b/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java @@ -22,7 +22,6 @@ import org.apache.lucene.analysis.MockTokenizer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.store.AlreadyClosedException; diff --git a/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java b/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java index 248baf0..e0e7c2f 100644 --- a/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java +++ b/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java @@ -19,7 +19,8 @@ package org.apache.lucene.index; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.*; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.memory.MemoryPostingsFormat; import org.apache.lucene.store.*; import org.apache.lucene.util.*; import org.junit.Test; @@ -35,13 +36,12 @@ public class TestRollingUpdates extends LuceneTestCase { dir.setCheckIndexOnClose(false); // we use a custom codec provider final LineFileDocs docs = new LineFileDocs(random); - CodecProvider provider = CodecProvider.getDefault(); //provider.register(new MemoryCodec()); - if ( (!"PreFlex".equals(provider.getDefaultFieldCodec())) && random.nextBoolean()) { - provider.setFieldCodec("docid", "Memory"); + if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random.nextBoolean()) { + Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat())); } - final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(provider)); + final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); w.setInfoStream(VERBOSE ? System.out : null); final int SIZE = atLeast(TEST_NIGHTLY ? 100 : 20); int id = 0; diff --git a/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java b/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java index 0348bd8..fa6c256 100644 --- a/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java +++ b/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java @@ -23,6 +23,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.util.BytesRef; import java.io.IOException; @@ -74,7 +75,7 @@ public class TestSegmentMerger extends LuceneTestCase { } public void testMerge() throws IOException { - SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, null, new FieldInfos(), newIOContext(random)); + SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, null, new FieldInfos(), Codec.getDefault(), newIOContext(random)); merger.add(reader1); merger.add(reader2); int docsMerged = merger.merge(); @@ -82,7 +83,7 @@ public class TestSegmentMerger extends LuceneTestCase { final FieldInfos fieldInfos = merger.fieldInfos(); //Should be able to open a new SegmentReader against the new directory SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, - merger.getSegmentCodecs(), fieldInfos), + merger.getCodec(), fieldInfos), true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); assertTrue(mergedReader != null); assertTrue(mergedReader.numDocs() == 2); @@ -145,7 +146,7 @@ public class TestSegmentMerger extends LuceneTestCase { w.close(); // Assert that SM fails if .del exists - SegmentMerger sm = new SegmentMerger(dir, 1, "a", null, null, null, newIOContext(random)); + SegmentMerger sm = new SegmentMerger(dir, 1, "a", null, null, null, Codec.getDefault(), newIOContext(random)); boolean doFail = false; try { sm.createCompoundFile("b1", w.segmentInfos.info(0), newIOContext(random)); diff --git a/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java b/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java index bd0fefc..c1a393e 100644 --- a/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java +++ b/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java @@ -26,6 +26,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.store.Directory; @@ -74,7 +75,7 @@ public class TestSegmentTermEnum extends LuceneTestCase { public void testPrevTermAtEnd() throws IOException { - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); addDoc(writer, "aaa bbb"); writer.close(); SegmentReader reader = getOnlySegmentReader(IndexReader.open(dir, false)); diff --git a/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java b/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java index 00bcbfb..c945fbc 100644 --- a/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java +++ b/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java @@ -358,11 +358,7 @@ public class TestTermsEnum extends LuceneTestCase { IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); /* - CoreCodecProvider cp = new CoreCodecProvider(); - cp.unregister(cp.lookup("Standard")); - cp.register(new StandardCodec(minTermsInBlock, maxTermsInBlock)); - cp.setDefaultFieldCodec("Standard"); - iwc.setCodecProvider(cp); + iwc.setCodec(new StandardCodec(minTermsInBlock, maxTermsInBlock)); */ final RandomIndexWriter w = new RandomIndexWriter(random, d, iwc); diff --git a/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java b/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java index dad404e..2d3d334 100644 --- a/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java +++ b/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java @@ -29,7 +29,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.CheckHits; import org.apache.lucene.search.IndexSearcher; @@ -57,7 +57,7 @@ public class TestTermsEnum2 extends LuceneTestCase { super.setUp(); // we generate aweful regexps: good for testing. // but for preflex codec, the test can be very slow, so use less iterations. - numIterations = CodecProvider.getDefault().getFieldCodec("field").equals("PreFlex") ? 10 * RANDOM_MULTIPLIER : atLeast(50); + numIterations = Codec.getDefault().getName().equals("Lucene3x") ? 10 * RANDOM_MULTIPLIER : atLeast(50); dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, diff --git a/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java b/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java index dbd01a0..2762aae 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java @@ -27,7 +27,7 @@ public class TestIntBlockCodec extends LuceneTestCase { public void testSimpleIntBlocks() throws Exception { Directory dir = newDirectory(); - IntStreamFactory f = new MockFixedIntBlockCodec(128).getIntFactory(); + IntStreamFactory f = new MockFixedIntBlockPostingsFormat(128).getIntFactory(); IntIndexOutput out = f.createOutput(dir, "test", newIOContext(random)); for(int i=0;i<11777;i++) { @@ -49,7 +49,7 @@ public class TestIntBlockCodec extends LuceneTestCase { public void testEmptySimpleIntBlocks() throws Exception { Directory dir = newDirectory(); - IntStreamFactory f = new MockFixedIntBlockCodec(128).getIntFactory(); + IntStreamFactory f = new MockFixedIntBlockPostingsFormat(128).getIntFactory(); IntIndexOutput out = f.createOutput(dir, "test", newIOContext(random)); // write no ints diff --git a/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestImpersonation.java b/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestImpersonation.java new file mode 100644 index 0000000..25d4331 --- /dev/null +++ b/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestImpersonation.java @@ -0,0 +1,34 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; +import org.apache.lucene.util.LuceneTestCase; + +/** + * Test that the SPI magic is returning "PreFlexRWCodec" for Lucene3x + * + * @lucene.experimental + */ +public class TestImpersonation extends LuceneTestCase { + public void test() throws Exception { + Codec codec = Codec.forName("Lucene3x"); + assertTrue(codec instanceof PreFlexRWCodec); + } +} diff --git a/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java b/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestSurrogates.java similarity index 97% rename from lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java rename to lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestSurrogates.java index 03051da..fdf3a42 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestSurrogates.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.preflex; +package org.apache.lucene.index.codecs.lucene3x; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -27,9 +27,15 @@ import org.apache.lucene.util.*; import java.util.*; import java.io.IOException; +import org.junit.BeforeClass; import org.junit.Test; public class TestSurrogates extends LuceneTestCase { + /** we will manually instantiate preflex-rw here */ + @BeforeClass + public static void beforeClass() { + LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true; + } private static String makeDifficultRandomUnicodeString(Random r) { final int end = r.nextInt(20); @@ -282,7 +288,7 @@ public class TestSurrogates extends LuceneTestCase { RandomIndexWriter w = new RandomIndexWriter(random, dir, newIndexWriterConfig( TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec(new PreFlexRWCodec()))); + new MockAnalyzer(random)).setCodec(new PreFlexRWCodec())); final int numField = _TestUtil.nextInt(random, 2, 5); diff --git a/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java b/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestTermInfosReaderIndex.java similarity index 91% rename from lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java rename to lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestTermInfosReaderIndex.java index d42efa3..b78db06 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestTermInfosReaderIndex.java @@ -1,4 +1,4 @@ -package org.apache.lucene.index.codecs.preflex; +package org.apache.lucene.index.codecs.lucene3x; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -39,9 +39,6 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.index.Term; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.CoreCodecProvider; import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; @@ -49,9 +46,9 @@ import org.apache.lucene.search.TopDocs; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import org.junit.BeforeClass; public class TestTermInfosReaderIndex extends LuceneTestCase { @@ -65,6 +62,12 @@ public class TestTermInfosReaderIndex extends LuceneTestCase { private IndexReader reader; private List sampleTerms; + /** we will manually instantiate preflex-rw here */ + @BeforeClass + public static void beforeClass() { + LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true; + } + @Override public void setUp() throws Exception { super.setUp(); @@ -78,10 +81,10 @@ public class TestTermInfosReaderIndex extends LuceneTestCase { r.close(); FieldInfos fieldInfos = new FieldInfos(directory, IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELD_INFOS_EXTENSION)); - String segmentFileName = IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION); + String segmentFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); long tiiFileLength = directory.fileLength(segmentFileName); IndexInput input = directory.openInput(segmentFileName, newIOContext(random)); - termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_EXTENSION), newIOContext(random)), fieldInfos, false); + termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), newIOContext(random)), fieldInfos, false); int totalIndexInterval = termEnum.indexInterval * indexDivisor; SegmentTermEnum indexEnum = new SegmentTermEnum(input, fieldInfos, true); @@ -164,11 +167,7 @@ public class TestTermInfosReaderIndex extends LuceneTestCase { private int populate(Directory directory) throws CorruptIndexException, LockObtainFailedException, IOException { IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.KEYWORD, false)); - CoreCodecProvider cp = new CoreCodecProvider(); - cp.unregister(cp.lookup("PreFlex")); - cp.register(new PreFlexRWCodec()); - cp.setDefaultFieldCodec("PreFlex"); - config.setCodecProvider(cp); + config.setCodec(new PreFlexRWCodec()); // turn off compound file, this test will open some index files directly. LogMergePolicy mp = newLogMergePolicy(); mp.setUseCompoundFile(false); diff --git a/lucene/src/test/org/apache/lucene/index/codecs/perfield/TestPerFieldPostingsFormat.java b/lucene/src/test/org/apache/lucene/index/codecs/perfield/TestPerFieldPostingsFormat.java new file mode 100644 index 0000000..d178403 --- /dev/null +++ b/lucene/src/test/org/apache/lucene/index/codecs/perfield/TestPerFieldPostingsFormat.java @@ -0,0 +1,272 @@ +package org.apache.lucene.index.codecs.perfield; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.List; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LogDocMergePolicy; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; +import org.apache.lucene.index.codecs.simpletext.SimpleTextPostingsFormat; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; +import org.junit.Test; + +/** + * + * + */ +//TODO: would be better in this test to pull termsenums and instanceof or something? +// this way we can verify PFPF is doing the right thing. +// for now we do termqueries. +public class TestPerFieldPostingsFormat extends LuceneTestCase { + + private IndexWriter newWriter(Directory dir, IndexWriterConfig conf) + throws IOException { + LogDocMergePolicy logByteSizeMergePolicy = new LogDocMergePolicy(); + logByteSizeMergePolicy.setUseCompoundFile(false); // make sure we use plain + // files + conf.setMergePolicy(logByteSizeMergePolicy); + + final IndexWriter writer = new IndexWriter(dir, conf); + writer.setInfoStream(VERBOSE ? System.out : null); + return writer; + } + + private void addDocs(IndexWriter writer, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); + writer.addDocument(doc); + } + } + + private void addDocs2(IndexWriter writer, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(newField("content", "bbb", TextField.TYPE_UNSTORED)); + writer.addDocument(doc); + } + } + + private void addDocs3(IndexWriter writer, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(newField("content", "ccc", TextField.TYPE_UNSTORED)); + doc.add(newField("id", "" + i, StringField.TYPE_STORED)); + writer.addDocument(doc); + } + } + + /* + * Test that heterogeneous index segments are merge successfully + */ + @Test + public void testMergeUnusedPerFieldCodec() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodec(new MockCodec()); + IndexWriter writer = newWriter(dir, iwconf); + addDocs(writer, 10); + writer.commit(); + addDocs3(writer, 10); + writer.commit(); + addDocs2(writer, 10); + writer.commit(); + assertEquals(30, writer.maxDoc()); + _TestUtil.checkIndex(dir); + writer.optimize(); + assertEquals(30, writer.maxDoc()); + writer.close(); + dir.close(); + } + + /* + * Test that heterogeneous index segments are merged sucessfully + */ + // TODO: not sure this test is that great, we should probably peek inside PerFieldPostingsFormat or something?! + @Test + public void testChangeCodecAndMerge() throws IOException { + Directory dir = newDirectory(); + if (VERBOSE) { + System.out.println("TEST: make new index"); + } + IndexWriterConfig iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodec(new MockCodec()); + iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); + //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10); + IndexWriter writer = newWriter(dir, iwconf); + + addDocs(writer, 10); + writer.commit(); + assertQuery(new Term("content", "aaa"), dir, 10); + if (VERBOSE) { + System.out.println("TEST: addDocs3"); + } + addDocs3(writer, 10); + writer.commit(); + writer.close(); + + assertQuery(new Term("content", "ccc"), dir, 10); + assertQuery(new Term("content", "aaa"), dir, 10); + Lucene40Codec codec = (Lucene40Codec)iwconf.getCodec(); + + iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setOpenMode(OpenMode.APPEND).setCodec(codec); + //((LogMergePolicy) iwconf.getMergePolicy()).setUseCompoundFile(false); + //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10); + iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); + + iwconf.setCodec(new MockCodec2()); // uses standard for field content + writer = newWriter(dir, iwconf); + // swap in new codec for currently written segments + if (VERBOSE) { + System.out.println("TEST: add docs w/ Standard codec for content field"); + } + addDocs2(writer, 10); + writer.commit(); + codec = (Lucene40Codec)iwconf.getCodec(); + PostingsFormat origContentCodec = PostingsFormat.forName("MockSep"); + PostingsFormat newContentCodec = PostingsFormat.forName("Lucene40"); + assertEquals(30, writer.maxDoc()); + assertQuery(new Term("content", "bbb"), dir, 10); + assertQuery(new Term("content", "ccc"), dir, 10); //// + assertQuery(new Term("content", "aaa"), dir, 10); + + if (VERBOSE) { + System.out.println("TEST: add more docs w/ new codec"); + } + addDocs2(writer, 10); + writer.commit(); + assertQuery(new Term("content", "ccc"), dir, 10); + assertQuery(new Term("content", "bbb"), dir, 20); + assertQuery(new Term("content", "aaa"), dir, 10); + assertEquals(40, writer.maxDoc()); + + if (VERBOSE) { + System.out.println("TEST: now optimize"); + } + writer.optimize(); + assertEquals(40, writer.maxDoc()); + writer.close(); + assertQuery(new Term("content", "ccc"), dir, 10); + assertQuery(new Term("content", "bbb"), dir, 20); + assertQuery(new Term("content", "aaa"), dir, 10); + + dir.close(); + } + + public void assertQuery(Term t, Directory dir, int num) + throws CorruptIndexException, IOException { + if (VERBOSE) { + System.out.println("\nTEST: assertQuery " + t); + } + IndexReader reader = IndexReader.open(dir, null, true, 1); + IndexSearcher searcher = newSearcher(reader); + TopDocs search = searcher.search(new TermQuery(t), num + 10); + assertEquals(num, search.totalHits); + searcher.close(); + reader.close(); + + } + + public static class MockCodec extends Lucene40Codec { + final PostingsFormat lucene40 = new Lucene40PostingsFormat(); + final PostingsFormat simpleText = new SimpleTextPostingsFormat(); + final PostingsFormat mockSep = new MockSepPostingsFormat(); + + @Override + public PostingsFormat getPostingsFormatForField(String field) { + if (field.equals("id")) { + return simpleText; + } else if (field.equals("content")) { + return mockSep; + } else { + return lucene40; + } + } + } + + public static class MockCodec2 extends Lucene40Codec { + final PostingsFormat lucene40 = new Lucene40PostingsFormat(); + final PostingsFormat simpleText = new SimpleTextPostingsFormat(); + + @Override + public PostingsFormat getPostingsFormatForField(String field) { + if (field.equals("id")) { + return simpleText; + } else { + return lucene40; + } + } + } + + /* + * Test per field codec support - adding fields with random codecs + */ + @Test + public void testStressPerFieldCodec() throws IOException { + Directory dir = newDirectory(random); + final int docsPerRound = 97; + int numRounds = atLeast(1); + for (int i = 0; i < numRounds; i++) { + int num = _TestUtil.nextInt(random, 30, 60); + IndexWriterConfig config = newIndexWriterConfig(random, + TEST_VERSION_CURRENT, new MockAnalyzer(random)); + config.setOpenMode(OpenMode.CREATE_OR_APPEND); + IndexWriter writer = newWriter(dir, config); + for (int j = 0; j < docsPerRound; j++) { + final Document doc = new Document(); + for (int k = 0; k < num; k++) { + FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + customType.setTokenized(random.nextBoolean()); + customType.setOmitNorms(random.nextBoolean()); + Field field = newField("" + k, _TestUtil + .randomRealisticUnicodeString(random, 128), customType); + doc.add(field); + } + writer.addDocument(doc); + } + if (random.nextBoolean()) { + writer.optimize(); + } + writer.commit(); + assertEquals((i + 1) * docsPerRound, writer.maxDoc()); + writer.close(); + } + dir.close(); + } +} diff --git a/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java b/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java index b1c1761..4ce4a0d 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java @@ -35,7 +35,8 @@ import org.apache.lucene.index.MultiFields; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsBaseFormat; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; @@ -51,13 +52,13 @@ import org.junit.Ignore; public class Test10KPulsings extends LuceneTestCase { public void test10kPulsed() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); + Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1)); File f = _TestUtil.getTempDir("10kpulsed"); MockDirectoryWrapper dir = newFSDirectory(f); dir.setCheckIndexOnClose(false); // we do this ourselves explicitly RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document document = new Document(); FieldType ft = new FieldType(TextField.TYPE_STORED); @@ -101,13 +102,14 @@ public class Test10KPulsings extends LuceneTestCase { */ public void test10kNotPulsed() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); + int freqCutoff = _TestUtil.nextInt(random, 1, 10); + Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(freqCutoff)); File f = _TestUtil.getTempDir("10knotpulsed"); MockDirectoryWrapper dir = newFSDirectory(f); dir.setCheckIndexOnClose(false); // we do this ourselves explicitly RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document document = new Document(); FieldType ft = new FieldType(TextField.TYPE_STORED); @@ -123,10 +125,7 @@ public class Test10KPulsings extends LuceneTestCase { NumberFormat df = new DecimalFormat("00000", new DecimalFormatSymbols(Locale.ENGLISH)); - Codec codec = cp.lookup(cp.getFieldCodec("field")); - assertTrue(codec instanceof PulsingCodec); - PulsingCodec pulsing = (PulsingCodec) codec; - final int freq = pulsing.getFreqCutoff() + 1; + final int freq = freqCutoff + 1; for (int i = 0; i < 10050; i++) { StringBuilder sb = new StringBuilder(); diff --git a/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java b/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java index e4dcc84..c1addf5 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java @@ -17,10 +17,8 @@ package org.apache.lucene.index.codecs.pulsing; * limitations under the License. */ -import java.io.IOException; import java.util.IdentityHashMap; import java.util.Map; -import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -30,27 +28,10 @@ import org.apache.lucene.index.CheckIndex; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.BlockTreeTermsReader; -import org.apache.lucene.index.codecs.BlockTreeTermsWriter; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; -import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; +import org.apache.lucene.index.codecs.nestedpulsing.NestedPulsingPostingsFormat; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; @@ -63,10 +44,10 @@ public class TestPulsingReuse extends LuceneTestCase { // TODO: this is a basic test. this thing is complicated, add more public void testSophisticatedReuse() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); + Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1)); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document doc = new Document(); doc.add(new Field("foo", "a b b c c c d e f g g h i i j j k", TextField.TYPE_UNSTORED)); iw.addDocument(doc); @@ -101,11 +82,11 @@ public class TestPulsingReuse extends LuceneTestCase { /** tests reuse with Pulsing1(Pulsing2(Standard)) */ public void testNestedPulsing() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new NestedPulsing()); + Codec cp = _TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat()); MockDirectoryWrapper dir = newDirectory(); dir.setCheckIndexOnClose(false); // will do this ourselves, custom codec RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document doc = new Document(); doc.add(new Field("foo", "a b b c c c d e f g g g h i i j j k l l m m m", TextField.TYPE_UNSTORED)); // note: the reuse is imperfect, here we would have 4 enums (lost reuse when we get an enum for 'm') @@ -138,79 +119,7 @@ public class TestPulsingReuse extends LuceneTestCase { ir.close(); CheckIndex ci = new CheckIndex(dir); - ci.checkIndex(null, cp); + ci.checkIndex(null); dir.close(); } - - static class NestedPulsing extends Codec { - public NestedPulsing() { - super("NestedPulsing"); - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase docsWriter = new StandardPostingsWriter(state); - - PostingsWriterBase pulsingWriterInner = new PulsingPostingsWriter(2, docsWriter); - PostingsWriterBase pulsingWriter = new PulsingPostingsWriter(1, pulsingWriterInner); - - // Terms dict - boolean success = false; - try { - FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, - BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); - success = true; - return ret; - } finally { - if (!success) { - pulsingWriter.close(); - } - } - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); - PostingsReaderBase pulsingReaderInner = new PulsingPostingsReader(docsReader); - PostingsReaderBase pulsingReader = new PulsingPostingsReader(pulsingReaderInner); - boolean success = false; - try { - FieldsProducer ret = new BlockTreeTermsReader( - state.dir, state.fieldInfos, state.segmentInfo.name, - pulsingReader, - state.context, - state.codecId, - state.termsIndexDivisor); - success = true; - return ret; - } finally { - if (!success) { - pulsingReader.close(); - } - } - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); - } - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, id, files); - BlockTreeTermsReader.files(dir, segmentInfo, id, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); - } - - @Override - public void getExtensions(Set extensions) { - StandardCodec.getStandardExtensions(extensions); - DefaultDocValuesConsumer.getExtensions(extensions); - } - } } diff --git a/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java b/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java index 526fc2b..a28465a 100644 --- a/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java +++ b/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java @@ -39,7 +39,7 @@ import org.apache.lucene.index.LogMergePolicy; import org.apache.lucene.index.MultiPerDocValues; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.search.*; @@ -65,7 +65,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { @Before public void setUp() throws Exception { super.setUp(); - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); + assumeFalse("cannot work with preflex codec", Codec.getDefault().getName().equals("Lucene3x")); } /* @@ -140,7 +140,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { indexValues(w_1, valuesPerIndex, first, values, false, 7); w_1.commit(); assertEquals(valuesPerIndex, w_1.maxDoc()); - _TestUtil.checkIndex(d_1, w_1.getConfig().getCodecProvider()); + _TestUtil.checkIndex(d_1); // index second index Directory d_2 = newDirectory(); @@ -148,7 +148,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { indexValues(w_2, valuesPerIndex, second, values, false, 7); w_2.commit(); assertEquals(valuesPerIndex, w_2.maxDoc()); - _TestUtil.checkIndex(d_2, w_2.getConfig().getCodecProvider()); + _TestUtil.checkIndex(d_2); Directory target = newDirectory(); IndexWriter w = new IndexWriter(target, writerConfig(random.nextBoolean())); @@ -162,7 +162,7 @@ public class TestDocValuesIndexing extends LuceneTestCase { w.optimize(true); w.commit(); - _TestUtil.checkIndex(target, w.getConfig().getCodecProvider()); + _TestUtil.checkIndex(target); assertEquals(valuesPerIndex * 2, w.maxDoc()); // check values @@ -546,7 +546,6 @@ public class TestDocValuesIndexing extends LuceneTestCase { } public void testMultiValuedIndexDocValuesField() throws Exception { - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); Directory d = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random, d); Document doc = new Document(); @@ -575,7 +574,6 @@ public class TestDocValuesIndexing extends LuceneTestCase { } public void testDifferentTypedDocValuesField() throws Exception { - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); Directory d = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random, d); Document doc = new Document(); diff --git a/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java b/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java index 53733af..6bb4341 100644 --- a/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java +++ b/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java @@ -15,7 +15,7 @@ import org.apache.lucene.index.IndexReader.ReaderContext; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -42,8 +42,7 @@ public class TestTypePromotion extends LuceneTestCase { @Before public void setUp() throws Exception { super.setUp(); - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault() - .getDefaultFieldCodec().equals("PreFlex")); + assumeFalse("cannot work with preflex codec", Codec.getDefault().getName().equals("Lucene3x")); } private static EnumSet INTEGERS = EnumSet.of(ValueType.VAR_INTS, diff --git a/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java b/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java index 3099014..21463ac 100644 --- a/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java +++ b/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java @@ -28,7 +28,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.search.similarities.Similarity; @@ -49,7 +49,7 @@ public class TestDocValuesScoring extends LuceneTestCase { public void testSimple() throws Exception { assumeFalse("PreFlex codec cannot work with IndexDocValues!", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + "Lucene3x".equals(Codec.getDefault().getName())); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir); diff --git a/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java b/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java index 000c13a..e43ab23 100644 --- a/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java +++ b/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java @@ -30,7 +30,7 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.Directory; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; @@ -60,8 +60,8 @@ public class TestPrefixRandom extends LuceneTestCase { // we generate aweful prefixes: good for testing. // but for preflex codec, the test can be very slow, so use less iterations. - final String codec = CodecProvider.getDefault().getFieldCodec("field"); - int num = codec.equals("PreFlex") ? 200 * RANDOM_MULTIPLIER : atLeast(1000); + final String codec = Codec.getDefault().getName(); + int num = codec.equals("Lucene3x") ? 200 * RANDOM_MULTIPLIER : atLeast(1000); for (int i = 0; i < num; i++) { field.setValue(_TestUtil.randomUnicodeString(random, 10)); writer.addDocument(doc); diff --git a/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java b/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java index 83db492..9fdd6a9 100644 --- a/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java +++ b/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java @@ -33,7 +33,7 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.Directory; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; @@ -143,7 +143,7 @@ public class TestRegexpRandom2 extends LuceneTestCase { public void testRegexps() throws Exception { // we generate aweful regexps: good for testing. // but for preflex codec, the test can be very slow, so use less iterations. - int num = CodecProvider.getDefault().getFieldCodec(fieldName).equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : atLeast(1000); + int num = Codec.getDefault().getName().equals("Lucene3x") ? 100 * RANDOM_MULTIPLIER : atLeast(1000); for (int i = 0; i < num; i++) { String reg = AutomatonTestUtil.randomRegexp(random); if (VERBOSE) { diff --git a/lucene/src/test/org/apache/lucene/search/TestSort.java b/lucene/src/test/org/apache/lucene/search/TestSort.java index 4b82e1f..a45c40f 100644 --- a/lucene/src/test/org/apache/lucene/search/TestSort.java +++ b/lucene/src/test/org/apache/lucene/search/TestSort.java @@ -40,7 +40,7 @@ import org.apache.lucene.index.IndexableField; import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.FieldValueHitQueue.Entry; @@ -70,7 +70,7 @@ import org.junit.BeforeClass; public class TestSort extends LuceneTestCase { // true if our codec supports docvalues: true unless codec is preflex (3.x) - boolean supportsDocValues = CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex") == false; + boolean supportsDocValues = Codec.getDefault().getName().equals("Lucene3x") == false; private static int NUM_STRINGS; private IndexSearcher full; private IndexSearcher searchX; diff --git a/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java b/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java index 8591be6..aff92fc 100644 --- a/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java +++ b/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java @@ -29,7 +29,7 @@ import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.OrdTermState; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -566,7 +566,7 @@ public class TestSimilarityBase extends LuceneTestCase { /** Test whether all similarities return document 3 before documents 7 and 8. */ public void testHeartRanking() throws IOException { assumeFalse("PreFlex codec does not support the stats necessary for this test!", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + "Lucene3x".equals(Codec.getDefault().getName())); Query q = new TermQuery(new Term(FIELD_BODY, "heart")); diff --git a/lucene/src/test/org/apache/lucene/util/TestNamedSPILoader.java b/lucene/src/test/org/apache/lucene/util/TestNamedSPILoader.java new file mode 100644 index 0000000..60c2a49 --- /dev/null +++ b/lucene/src/test/org/apache/lucene/util/TestNamedSPILoader.java @@ -0,0 +1,44 @@ +package org.apache.lucene.util; + +import java.util.Set; + +import org.apache.lucene.index.codecs.Codec; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// TODO: maybe we should test this with mocks, but its easy +// enough to test the basics via Codec +public class TestNamedSPILoader extends LuceneTestCase { + public void testLookup() { + Codec codec = Codec.forName("Lucene40"); + assertEquals("Lucene40", codec.getName()); + } + + // we want an exception if its not found. + public void testBogusLookup() { + try { + Codec codec = Codec.forName("dskfdskfsdfksdfdsf"); + fail(); + } catch (IllegalArgumentException expected) {} + } + + public void testAvailableServices() { + Set codecs = Codec.availableCodecs(); + assertTrue(codecs.contains("Lucene40")); + } +} diff --git a/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java index 33496b6..68b264e 100644 --- a/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java +++ b/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java @@ -40,7 +40,8 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; @@ -1013,10 +1014,11 @@ public class TestFSTs extends LuceneTestCase { // file, up until a time limit public void testRealTerms() throws Exception { - final String defaultCodec = CodecProvider.getDefault().getDefaultFieldCodec(); - if (defaultCodec.equals("SimpleText") || defaultCodec.equals("Memory")) { + // TODO: is this necessary? we use the annotation... + final String defaultFormat = _TestUtil.getPostingsFormat("abracadabra"); + if (defaultFormat.equals("SimpleText") || defaultFormat.equals("Memory")) { // no - CodecProvider.getDefault().setDefaultFieldCodec("Standard"); + Codec.setDefault(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())); } final LineFileDocs docs = new LineFileDocs(random); diff --git a/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java b/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java index e3a0a29..00c035b 100644 --- a/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java +++ b/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java @@ -20,7 +20,7 @@ package org.apache.lucene.collation; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CollationTestBase; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.util.BytesRef; import java.text.Collator; @@ -47,7 +47,7 @@ public class TestCollationKeyAnalyzer extends CollationTestBase { @Override public void setUp() throws Exception { super.setUp(); - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); } public void testFarsiRangeFilterCollating() throws Exception { diff --git a/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java b/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java index 305ee27..681290a 100644 --- a/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java +++ b/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java @@ -22,7 +22,7 @@ import com.ibm.icu.text.Collator; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CollationTestBase; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.util.BytesRef; import java.util.Locale; @@ -45,7 +45,7 @@ public class TestICUCollationKeyAnalyzer extends CollationTestBase { @Override public void setUp() throws Exception { super.setUp(); - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); } public void testFarsiRangeFilterCollating() throws Exception { diff --git a/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java b/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java index 217818c..ac00ffa 100644 --- a/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java +++ b/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java @@ -33,7 +33,7 @@ import org.apache.lucene.index.NoDeletionPolicy; import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.NoMergeScheduler; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.Version; @@ -133,7 +133,12 @@ public class CreateIndexTask extends PerfTask { final String defaultCodec = config.get("default.codec", null); if (defaultCodec != null) { - CodecProvider.getDefault().setDefaultFieldCodec(defaultCodec); + try { + Class clazz = Class.forName(defaultCodec).asSubclass(Codec.class); + Codec.setDefault(clazz.newInstance()); + } catch (Exception e) { + throw new RuntimeException("Couldn't instantiate Codec: " + defaultCodec, e); + } } final String mergePolicy = config.get("merge.policy", diff --git a/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java b/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java index 253d82e..97c4eed 100755 --- a/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java +++ b/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java @@ -486,11 +486,17 @@ public class SpellChecker implements java.io.Closeable { * @throws IOException */ public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB, boolean optimize) throws IOException { + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null) + .setRAMBufferSizeMB(ramMB); + ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(mergeFactor); + indexDictionary(dict, config, optimize); + } + + public final void indexDictionary(Dictionary dict, IndexWriterConfig config, boolean optimize) throws IOException { synchronized (modifyCurrentIndexLock) { ensureOpen(); final Directory dir = this.spellIndex; - final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, null).setRAMBufferSizeMB(ramMB)); - ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setMaxMergeAtOnce(mergeFactor); + final IndexWriter writer = new IndexWriter(dir, config); IndexSearcher indexSearcher = obtainSearcher(); final List termsEnums = new ArrayList(); diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index c12c10a..c6e5359 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -133,10 +133,8 @@ New Features fq={!join from=name to=parent}eyes:blue (yonik) -* SOLR-1942: Added the ability to select codec per fieldType in schema.xml - as well as support custom CodecProviders in solrconfig.xml. - NOTE: IndexReaderFactory now has a codecProvider that should be passed - to IndexReader.open (in the case you have a custom IndexReaderFactory). +* SOLR-1942: Added the ability to select postings format per fieldType in schema.xml + as well as support custom Codecs in solrconfig.xml. (simonw via rmuir) * SOLR-2136: Boolean type added to function queries, along with diff --git a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java index 63fcfbc..8d2ebbe 100644 --- a/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java +++ b/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java @@ -22,7 +22,7 @@ import java.io.FileOutputStream; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; @@ -37,7 +37,7 @@ public class TestICUCollationField extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); String home = setupSolrHome(); initCore("solrconfig.xml","schema.xml", home); // add some docs diff --git a/solr/core/src/java/org/apache/solr/core/CodecFactory.java b/solr/core/src/java/org/apache/solr/core/CodecFactory.java new file mode 100644 index 0000000..e047181 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/CodecFactory.java @@ -0,0 +1,33 @@ +package org.apache.solr.core; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.util.plugin.NamedListInitializedPlugin; + +/** + * Factory for plugging in a custom {@link Codec} + */ +public abstract class CodecFactory implements NamedListInitializedPlugin { + public void init(NamedList args) { + } + + public abstract Codec create(IndexSchema Schema); +} diff --git a/solr/core/src/java/org/apache/solr/core/CodecProviderFactory.java b/solr/core/src/java/org/apache/solr/core/CodecProviderFactory.java deleted file mode 100644 index e3bc985..0000000 diff --git a/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java b/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java new file mode 100644 index 0000000..602bfca --- /dev/null +++ b/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java @@ -0,0 +1,59 @@ +package org.apache.solr.core; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; + +/** + * Default CodecFactory implementation, extends Lucene's + * and returns postings format implementations according to the + * schema configuration. + * @lucene.experimental + */ +public class DefaultCodecFactory extends CodecFactory { + + // TODO: we need to change how solr does this? + // rather than a string like "Pulsing" you need to be able to pass parameters + // and everything to a field in the schema, e.g. we should provide factories for + // the Lucene's core formats (Memory, Pulsing, ...) and such. + // + // So I think a FieldType should return PostingsFormat, not a String. + // how it constructs this from the XML... i don't care. + + @Override + public Codec create(final IndexSchema schema) { + return new Lucene40Codec() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + final SchemaField fieldOrNull = schema.getFieldOrNull(field); + if (fieldOrNull == null) { + throw new IllegalArgumentException("no such field " + field); + } + String postingsFormatName = fieldOrNull.getType().getPostingsFormat(); + if (postingsFormatName != null) { + return PostingsFormat.forName(postingsFormatName); + } + return super.getPostingsFormatForField(field); + } + }; + } +} diff --git a/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java b/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java index fb87650..4204a9e 100644 --- a/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java +++ b/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java @@ -19,7 +19,6 @@ package org.apache.solr.core; import java.io.IOException; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.solr.common.util.NamedList; import org.apache.solr.util.plugin.NamedListInitializedPlugin; @@ -29,7 +28,6 @@ import org.apache.solr.util.plugin.NamedListInitializedPlugin; */ public abstract class IndexReaderFactory implements NamedListInitializedPlugin { protected int termInfosIndexDivisor = 1;//IndexReader.DEFAULT_TERMS_INDEX_DIVISOR; Set this once Lucene makes this public. - protected CodecProvider provider; /** * Potentially initializes {@link #termInfosIndexDivisor}. Overriding classes should call super.init() in order * to make sure termInfosIndexDivisor is set. @@ -65,11 +63,4 @@ public abstract class IndexReaderFactory implements NamedListInitializedPlugin { */ public abstract IndexReader newReader(Directory indexDir, boolean readOnly) throws IOException; - - /** - * Sets the codec provider for this IndexReaderFactory - */ - public void setCodecProvider(CodecProvider provider) { - this.provider = provider; - } } diff --git a/solr/core/src/java/org/apache/solr/core/SchemaCodecProvider.java b/solr/core/src/java/org/apache/solr/core/SchemaCodecProvider.java deleted file mode 100644 index 3d0ffab..0000000 diff --git a/solr/core/src/java/org/apache/solr/core/SolrConfig.java b/solr/core/src/java/org/apache/solr/core/SolrConfig.java index 3a9257d..4128d13 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrConfig.java +++ b/solr/core/src/java/org/apache/solr/core/SolrConfig.java @@ -35,8 +35,6 @@ import org.apache.solr.update.processor.UpdateRequestProcessorChain; import org.apache.solr.spelling.QueryConverter; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.index.IndexDeletionPolicy; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.Version; import org.slf4j.Logger; @@ -200,7 +198,7 @@ public class SolrConfig extends Config { loadPluginInfo(DirectoryFactory.class,"directoryFactory",false, true); loadPluginInfo(IndexDeletionPolicy.class,"mainIndex/deletionPolicy",false, true); - loadPluginInfo(CodecProviderFactory.class,"mainIndex/codecProviderFactory",false, false); + loadPluginInfo(CodecFactory.class,"mainIndex/codecFactory",false, false); loadPluginInfo(IndexReaderFactory.class,"indexReaderFactory",false, true); loadPluginInfo(UpdateRequestProcessorChain.class,"updateRequestProcessorChain",false, false); loadPluginInfo(UpdateLog.class,"updateHandler/updateLog",false, false); diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.java index fbea86f..26afa4e 100644 --- a/solr/core/src/java/org/apache/solr/core/SolrCore.java +++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java @@ -20,7 +20,7 @@ package org.apache.solr.core; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.store.Directory; import org.apache.solr.common.SolrException; @@ -91,7 +91,7 @@ public final class SolrCore implements SolrInfoMBean { private IndexDeletionPolicyWrapper solrDelPolicy; private DirectoryFactory directoryFactory; private IndexReaderFactory indexReaderFactory; - private final CodecProvider codecProvider; + private final Codec codec; public long getStartTime() { return startTime; } @@ -347,7 +347,6 @@ public final class SolrCore implements SolrInfoMBean { indexReaderFactory = new StandardIndexReaderFactory(); } this.indexReaderFactory = indexReaderFactory; - this.indexReaderFactory.setCodecProvider(codecProvider); } // protect via synchronized(SolrCore.class) @@ -383,7 +382,7 @@ public final class SolrCore implements SolrInfoMBean { log.warn(logid+"Solr index directory '" + new File(indexDir) + "' doesn't exist." + " Creating new index..."); - SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.mainIndexConfig, solrDelPolicy, codecProvider, false); + SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.mainIndexConfig, solrDelPolicy, codec, false); writer.close(); } @@ -560,7 +559,7 @@ public final class SolrCore implements SolrInfoMBean { initDeletionPolicy(); - this.codecProvider = initCodecProvider(solrConfig, schema); + this.codec= initCodec(solrConfig, schema); if (updateHandler == null) { initDirectoryFactory(); @@ -638,18 +637,16 @@ public final class SolrCore implements SolrInfoMBean { resourceLoader.inform(infoRegistry); } - private CodecProvider initCodecProvider(SolrConfig solrConfig, IndexSchema schema) { - final PluginInfo info = solrConfig.getPluginInfo(CodecProviderFactory.class.getName()); - CodecProvider cp; + private Codec initCodec(SolrConfig solrConfig, final IndexSchema schema) { + final PluginInfo info = solrConfig.getPluginInfo(CodecFactory.class.getName()); + final CodecFactory factory; if (info != null) { - CodecProviderFactory factory = (CodecProviderFactory) schema.getResourceLoader().newInstance(info.className); + factory = (CodecFactory) schema.getResourceLoader().newInstance(info.className); factory.init(info.initArgs); - cp = factory.create(); } else { - // make sure we use the default if nothing is configured - cp = CodecProvider.getDefault(); + factory = new DefaultCodecFactory(); } - return new SchemaCodecProvider(schema, cp); + return factory.create(schema); } /** @@ -1853,8 +1850,8 @@ public final class SolrCore implements SolrInfoMBean { return lst; } - public CodecProvider getCodecProvider() { - return codecProvider; + public Codec getCodec() { + return codec; } public final class LazyQueryResponseWriterWrapper implements QueryResponseWriter { diff --git a/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java b/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java index 5767d29..2695cb2 100644 --- a/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java +++ b/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java @@ -35,6 +35,6 @@ public class StandardIndexReaderFactory extends IndexReaderFactory { @Override public IndexReader newReader(Directory indexDir, boolean readOnly) throws IOException { - return IndexReader.open(indexDir, null, readOnly, termInfosIndexDivisor, provider); + return IndexReader.open(indexDir, null, readOnly, termInfosIndexDivisor); } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java index d345e19..4fbc58e 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java @@ -213,6 +213,7 @@ public class CoreAdminHandler extends RequestHandlerBase { for (int i = 0; i < dirNames.length; i++) { Directory dir = dirFactory.get(dirNames[i], core.getSolrConfig().mainIndexConfig.lockType); dirsToBeReleased[i] = dir; + // TODO: why doesn't this use the IR factory? what is going on here? readersToBeClosed[i] = IndexReader.open(dir, true); } } diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.java index 05633d1..2bdbeef 100644 --- a/solr/core/src/java/org/apache/solr/schema/FieldType.java +++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java @@ -167,10 +167,10 @@ public abstract class FieldType extends FieldProperties { initArgs.remove("positionIncrementGap"); } - final String codec = initArgs.get("codec"); - if (codec != null) { - this.codec = codec; - initArgs.remove("codec"); + final String postingsFormat = initArgs.get("postingsFormat"); + if (postingsFormat != null) { + this.postingsFormat = postingsFormat; + initArgs.remove("postingsFormat"); } if (initArgs.size() > 0) { @@ -527,12 +527,12 @@ public abstract class FieldType extends FieldProperties { } /** - * The codec ID used for this field type + * The postings format used for this field type */ - protected String codec; + protected String postingsFormat; - public String getCodec() { - return codec; + public String getPostingsFormat() { + return postingsFormat; } /** diff --git a/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java index 459feb3..0afbd06 100644 --- a/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java +++ b/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java @@ -29,6 +29,7 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.search.spell.HighFrequencyDictionary; import org.apache.lucene.search.spell.PlainTextDictionary; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.schema.FieldType; @@ -62,7 +63,14 @@ public class FileBasedSpellChecker extends AbstractLuceneSpellChecker { try { loadExternalFileDictionary(core); spellChecker.clearIndex(); - spellChecker.indexDictionary(dictionary); + // TODO: you should be able to specify the IWC params? + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null); + // TODO: if we enable this, codec gets angry since field won't exist in the schema + // config.setCodec(core.getCodec()); + ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(300); + // TODO: does Solr really want to continue passing 'optimize=true' to the spellchecker here? + // (its been doing this behind the scenes all along, but its wasteful. + spellChecker.indexDictionary(dictionary, config, true); } catch (IOException e) { throw new RuntimeException(e); } @@ -94,6 +102,8 @@ public class FileBasedSpellChecker extends AbstractLuceneSpellChecker { setMaxBufferedDocs(150). setMergePolicy(mp). setOpenMode(IndexWriterConfig.OpenMode.CREATE) + // TODO: if we enable this, codec gets angry since field won't exist in the schema + // .setCodec(core.getCodec()) ); List lines = core.getResourceLoader().getLines(sourceLocation, characterEncoding); @@ -106,7 +116,7 @@ public class FileBasedSpellChecker extends AbstractLuceneSpellChecker { writer.optimize(); writer.close(); - dictionary = new HighFrequencyDictionary(IndexReader.open(ramDir), + dictionary = new HighFrequencyDictionary(IndexReader.open(ramDir, true), WORD_FIELD_NAME, 0.0f); } else { // check if character encoding is defined diff --git a/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java b/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java index 1963006..63eaba5 100644 --- a/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java +++ b/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java @@ -17,8 +17,11 @@ package org.apache.solr.spelling; */ import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.search.spell.HighFrequencyDictionary; +import org.apache.lucene.util.Version; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; @@ -63,7 +66,7 @@ public class IndexBasedSpellChecker extends AbstractLuceneSpellChecker { if (sourceLocation != null) { try { FSDirectory luceneIndexDir = FSDirectory.open(new File(sourceLocation)); - this.reader = IndexReader.open(luceneIndexDir); + this.reader = IndexReader.open(luceneIndexDir, true); } catch (IOException e) { throw new RuntimeException(e); } @@ -85,8 +88,19 @@ public class IndexBasedSpellChecker extends AbstractLuceneSpellChecker { // Create the dictionary dictionary = new HighFrequencyDictionary(reader, field, threshold); + // TODO: maybe whether or not to clear the index should be configurable? + // an incremental update is faster (just adds new terms), but if you 'expunged' + // old terms I think they might hang around. spellChecker.clearIndex(); - spellChecker.indexDictionary(dictionary); + // TODO: you should be able to specify the IWC params? + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null); + + // TODO: if we enable this, codec gets angry since field won't exist in the schema + // config.setCodec(core.getCodec()); + ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(300); + // TODO: does Solr really want to continue passing 'optimize=true' to the spellchecker here? + // (its been doing this behind the scenes all along, but its wasteful. + spellChecker.indexDictionary(dictionary, config, true); } catch (IOException e) { throw new RuntimeException(e); diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java index bb6b699..40f9460 100644 --- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java +++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java @@ -78,7 +78,7 @@ public final class DefaultSolrCoreState extends SolrCoreState { boolean removeAllExisting, boolean forceNewDirectory) throws IOException { return new SolrIndexWriter(name, core.getNewIndexDir(), core.getDirectoryFactory(), removeAllExisting, core.getSchema(), - core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy(), core.getCodecProvider(), forceNewDirectory); + core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy(), core.getCodec(), forceNewDirectory); } @Override diff --git a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java index b84e232..f56fefe 100644 --- a/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java +++ b/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java @@ -29,7 +29,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.Directory; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.schema.IndexSchema; @@ -52,12 +52,12 @@ public class SolrIndexWriter extends IndexWriter { private PrintStream infoStream; private DirectoryFactory directoryFactory; - public SolrIndexWriter(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, CodecProvider codecProvider, boolean forceNewDirectory) throws IOException { + public SolrIndexWriter(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec, boolean forceNewDirectory) throws IOException { super( directoryFactory.get(path, config.lockType, forceNewDirectory), config.toIndexWriterConfig(schema). setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND). - setIndexDeletionPolicy(delPolicy).setCodecProvider(codecProvider) + setIndexDeletionPolicy(delPolicy).setCodec(codec) ); log.debug("Opened Writer " + name); this.name = name; diff --git a/solr/core/src/test-files/solr/conf/schema_codec.xml b/solr/core/src/test-files/solr/conf/schema_codec.xml index 2a69f11..e08ab8d 100644 --- a/solr/core/src/test-files/solr/conf/schema_codec.xml +++ b/solr/core/src/test-files/solr/conf/schema_codec.xml @@ -17,9 +17,9 @@ --> - - - + + + diff --git a/solr/core/src/test-files/solr/conf/solrconfig_codec.xml b/solr/core/src/test-files/solr/conf/solrconfig_codec.xml index 0a1a7e1..1ff30e0 100644 --- a/solr/core/src/test-files/solr/conf/solrconfig_codec.xml +++ b/solr/core/src/test-files/solr/conf/solrconfig_codec.xml @@ -19,15 +19,5 @@ ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - Pulsing - - org.apache.lucene.index.codecs.simpletext.SimpleTextCodec - org.apache.lucene.index.codecs.preflex.PreFlexCodec - - - - diff --git a/solr/core/src/test/org/apache/solr/core/MockCodecProviderFactory.java b/solr/core/src/test/org/apache/solr/core/MockCodecProviderFactory.java deleted file mode 100644 index ace2e09..0000000 diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecProviderSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecProviderSupport.java deleted file mode 100644 index 10897a9..0000000 diff --git a/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java new file mode 100644 index 0000000..459d765 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java @@ -0,0 +1,73 @@ +package org.apache.solr.core; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Map; + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.schema.SchemaField; +import org.junit.BeforeClass; + +public class TestCodecSupport extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-basic.xml", "schema_codec.xml"); + } + + public void testPostingsFormats() { + Codec codec = h.getCore().getCodec(); + Map fields = h.getCore().getSchema().getFields(); + SchemaField schemaField = fields.get("string_pulsing_f"); + PerFieldPostingsFormat format = (PerFieldPostingsFormat) codec.postingsFormat(); + assertEquals("Pulsing40", format.getPostingsFormatForField(schemaField.getName()).getName()); + schemaField = fields.get("string_simpletext_f"); + assertEquals("SimpleText", + format.getPostingsFormatForField(schemaField.getName()).getName()); + schemaField = fields.get("string_standard_f"); + assertEquals("Lucene40", format.getPostingsFormatForField(schemaField.getName()).getName()); + schemaField = fields.get("string_f"); + assertEquals("Lucene40", format.getPostingsFormatForField(schemaField.getName()).getName()); + } + + public void testDynamicFields() { + Codec codec = h.getCore().getCodec(); + PerFieldPostingsFormat format = (PerFieldPostingsFormat) codec.postingsFormat(); + + assertEquals("SimpleText", format.getPostingsFormatForField("foo_simple").getName()); + assertEquals("SimpleText", format.getPostingsFormatForField("bar_simple").getName()); + assertEquals("Pulsing40", format.getPostingsFormatForField("foo_pulsing").getName()); + assertEquals("Pulsing40", format.getPostingsFormatForField("bar_pulsing").getName()); + assertEquals("Lucene40", format.getPostingsFormatForField("foo_standard").getName()); + assertEquals("Lucene40", format.getPostingsFormatForField("bar_standard").getName()); + } + + public void testUnknownField() { + Codec codec = h.getCore().getCodec(); + PerFieldPostingsFormat format = (PerFieldPostingsFormat) codec.postingsFormat(); + try { + format.getPostingsFormatForField("notexisting"); + fail("field is not existing"); + } catch (IllegalArgumentException e) { + // + } + + } +} diff --git a/solr/core/src/test/org/apache/solr/schema/TestCollationField.java b/solr/core/src/test/org/apache/solr/schema/TestCollationField.java index e92c14e..175b0f8 100644 --- a/solr/core/src/test/org/apache/solr/schema/TestCollationField.java +++ b/solr/core/src/test/org/apache/solr/schema/TestCollationField.java @@ -25,7 +25,7 @@ import java.util.Locale; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; @@ -36,7 +36,8 @@ public class TestCollationField extends SolrTestCaseJ4 { @BeforeClass public static void beforeClass() throws Exception { - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", + "Lucene3x".equals(Codec.getDefault().getName())); String home = setupSolrHome(); initCore("solrconfig.xml","schema.xml", home); // add some docs diff --git a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java index b5065d4..fff65ec 100755 --- a/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java +++ b/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java @@ -18,7 +18,7 @@ package org.apache.solr.search.function; import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.TFIDFSimilarity; @@ -413,7 +413,7 @@ public class TestFunctionQuery extends SolrTestCaseJ4 { */ public void testTotalTermFreq() throws Exception { assumeFalse("PreFlex codec does not support collection-level term stats", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + "Lucene3x".equals(Codec.getDefault().getName())); clearIndex();