diff -ruN -x .svn -x build lucene-clean-trunk/lucene/build.xml lucene-2621/lucene/build.xml --- lucene-clean-trunk/lucene/build.xml 2011-09-08 17:01:39.556036271 -0400 +++ lucene-2621/lucene/build.xml 2011-11-02 21:53:55.060564056 -0400 @@ -30,15 +30,15 @@ - - + + - + @@ -556,11 +556,11 @@ - + diff -ruN -x .svn -x build lucene-clean-trunk/lucene/CHANGES.txt lucene-2621/lucene/CHANGES.txt --- lucene-clean-trunk/lucene/CHANGES.txt 2011-11-01 18:40:47.472614538 -0400 +++ lucene-2621/lucene/CHANGES.txt 2011-11-02 21:32:28.288564720 -0400 @@ -215,7 +215,7 @@ * LUCENE-2881: FieldInfos is now tracked per segment. Before it was tracked per IndexWriter session, which resulted in FieldInfos that had the FieldInfo properties from all previous segments combined. Field numbers are now tracked - globally across IndexWriter sessions and persisted into a X.fnx file on + globally across IndexWriter sessions and persisted into a _X.fnx file on successful commit. The corresponding file format changes are backwards- compatible. (Michael Busch, Simon Willnauer) @@ -375,9 +375,8 @@ * LUCENE-1458, LUCENE-2111: With flexible indexing it is now possible for an application to create its own postings codec, to alter how fields, terms, docs and positions are encoded into the index. The - standard codec is the default codec. Both IndexWriter and - IndexReader accept a CodecProvider class to obtain codecs for newly - written segments as well as existing segments opened for reading. + standard codec is the default codec. IndexWriter accepts a Codec + class to obtain codecs for newly written segments. * LUCENE-1458, LUCENE-2111: Some experimental codecs have been added for flexible indexing, including pulsing codec (inlines @@ -403,7 +402,7 @@ * LUCENE-2489: Added PerFieldCodecWrapper (in oal.index.codecs) which lets you set the Codec per field (Mike McCandless) -* LUCENE-2373: Extend CodecProvider to use SegmentInfosWriter and +* LUCENE-2373: Extend Codec to use SegmentInfosWriter and SegmentInfosReader to allow customization of SegmentInfos data. (Andrzej Bialecki) @@ -445,10 +444,10 @@ (i.e. \* or "*") Custom QueryParser subclasses overriding getRangeQuery() will be passed null for any open endpoint. (Adriano Crestani, yonik) -* LUCENE-2742: Add native per-field codec support. CodecProvider lets you now - register a codec for each field and which is in turn recorded in the segment - and field information. Codecs are maintained on a per-segment basis and be - resolved without knowing the actual codec used for writing the segment. +* LUCENE-2742: Add native per-field postings format support. Codec lets you now + register a postings format for each field and which is in turn recorded + into the index. Postings formtas are maintained on a per-segment basis and be + resolved without knowing the actual postings format used for writing the segment. (Simon Willnauer) * LUCENE-2741: Add support for multiple codecs that use the same file diff -ruN -x .svn -x build lucene-clean-trunk/lucene/common-build.xml lucene-2621/lucene/common-build.xml --- lucene-clean-trunk/lucene/common-build.xml 2011-11-01 18:41:12.488614526 -0400 +++ lucene-2621/lucene/common-build.xml 2011-11-02 12:17:18.636581879 -0400 @@ -83,8 +83,7 @@ - - + @@ -473,8 +472,13 @@ - + + + + + @@ -551,9 +555,7 @@ - - - + diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/contrib-build.xml lucene-2621/lucene/contrib/contrib-build.xml --- lucene-clean-trunk/lucene/contrib/contrib-build.xml 2011-10-10 14:34:12.638772412 -0400 +++ lucene-2621/lucene/contrib/contrib-build.xml 2011-11-01 21:10:35.776609906 -0400 @@ -39,8 +39,8 @@ - + diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java lucene-2621/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java --- lucene-clean-trunk/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java 2011-08-29 12:22:46.171459551 -0400 +++ lucene-2621/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java 2011-11-02 11:31:08.624583308 -0400 @@ -34,6 +34,7 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TopDocs; @@ -107,7 +108,7 @@ Directory ramdir = newDirectory(); Analyzer analyzer = randomAnalyzer(); IndexWriter writer = new IndexWriter(ramdir, - new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); Document doc = new Document(); Field field1 = newField("foo", fooField.toString(), TextField.TYPE_UNSTORED); Field field2 = newField("term", termField.toString(), TextField.TYPE_UNSTORED); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java --- lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java 2011-09-30 20:57:07.228773589 -0400 +++ lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java 2011-11-02 21:51:27.600564131 -0400 @@ -17,33 +17,17 @@ * limitations under the License. */ -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; -import org.apache.lucene.index.codecs.BlockTermsReader; -import org.apache.lucene.index.codecs.TermsIndexReaderBase; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; +import org.apache.lucene.index.codecs.DefaultDocValuesFormat; +import org.apache.lucene.index.codecs.DefaultFieldsFormat; +import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldsFormat; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.SegmentInfosFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; /** - * This codec extends {@link StandardCodec} to work on append-only outputs, such + * This codec extends {@link Lucene40Codec} to work on append-only outputs, such * as plain output streams and append-only filesystems. * *

Note: compound file format feature is not compatible with @@ -54,105 +38,32 @@ * @lucene.experimental */ public class AppendingCodec extends Codec { - public static String CODEC_NAME = "Appending"; - public AppendingCodec() { - super(CODEC_NAME); - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) - throws IOException { - PostingsWriterBase docsWriter = new StandardPostingsWriter(state); - boolean success = false; - AppendingTermsIndexWriter indexWriter = null; - try { - indexWriter = new AppendingTermsIndexWriter(state); - success = true; - } finally { - if (!success) { - docsWriter.close(); - } - } - success = false; - try { - FieldsConsumer ret = new AppendingTermsDictWriter(indexWriter, state, docsWriter); - success = true; - return ret; - } finally { - if (!success) { - try { - docsWriter.close(); - } finally { - indexWriter.close(); - } - } - } + super("Appending"); } + private final PostingsFormat postings = new AppendingPostingsFormat(); + private final SegmentInfosFormat infos = new AppendingSegmentInfosFormat(); + private final FieldsFormat fields = new DefaultFieldsFormat(); + private final DocValuesFormat docValues = new DefaultDocValuesFormat(); + @Override - public FieldsProducer fieldsProducer(SegmentReadState state) - throws IOException { - PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); - TermsIndexReaderBase indexReader; - - boolean success = false; - try { - indexReader = new AppendingTermsIndexReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId, state.context); - success = true; - } finally { - if (!success) { - docsReader.close(); - } - } - success = false; - try { - FieldsProducer ret = new AppendingTermsDictReader(indexReader, - state.dir, state.fieldInfos, state.segmentInfo.name, - docsReader, - state.context, - StandardCodec.TERMS_CACHE_SIZE, - state.codecId); - success = true; - return ret; - } finally { - if (!success) { - try { - docsReader.close(); - } finally { - indexReader.close(); - } - } - } + public PostingsFormat postingsFormat() { + return postings; } @Override - public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) - throws IOException { - StandardPostingsReader.files(dir, segmentInfo, codecId, files); - BlockTermsReader.files(dir, segmentInfo, codecId, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files); + public FieldsFormat fieldsFormat() { + return fields; } @Override - public void getExtensions(Set extensions) { - StandardCodec.getStandardExtensions(extensions); - DefaultDocValuesConsumer.getExtensions(extensions); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); + public DocValuesFormat docValuesFormat() { + return docValues; } @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); + public SegmentInfosFormat segmentInfosFormat() { + return infos; } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java --- lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingPostingsFormat.java 2011-11-02 21:26:22.436564908 -0400 @@ -0,0 +1,128 @@ +package org.apache.lucene.index.codecs.appending; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.BlockTermsReader; +import org.apache.lucene.index.codecs.TermsIndexReaderBase; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; + +/** + * Appending postings impl + */ +class AppendingPostingsFormat extends PostingsFormat { + public static String CODEC_NAME = "Appending"; + + public AppendingPostingsFormat() { + super(CODEC_NAME); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) + throws IOException { + PostingsWriterBase docsWriter = new Lucene40PostingsWriter(state); + boolean success = false; + AppendingTermsIndexWriter indexWriter = null; + try { + indexWriter = new AppendingTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + docsWriter.close(); + } + } + success = false; + try { + FieldsConsumer ret = new AppendingTermsDictWriter(indexWriter, state, docsWriter); + success = true; + return ret; + } finally { + if (!success) { + try { + docsWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) + throws IOException { + PostingsReaderBase docsReader = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + TermsIndexReaderBase indexReader; + + boolean success = false; + try { + indexReader = new AppendingTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.segmentSuffix, state.context); + success = true; + } finally { + if (!success) { + docsReader.close(); + } + } + success = false; + try { + FieldsProducer ret = new AppendingTermsDictReader(indexReader, + state.dir, state.fieldInfos, state.segmentInfo.name, + docsReader, + state.context, + Lucene40PostingsFormat.TERMS_CACHE_SIZE, + state.segmentSuffix); + success = true; + return ret; + } finally { + if (!success) { + try { + docsReader.close(); + } finally { + indexReader.close(); + } + } + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) + throws IOException { + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java --- lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosFormat.java 2011-11-01 23:41:00.472605256 -0400 @@ -0,0 +1,30 @@ +package org.apache.lucene.index.codecs.appending; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; +import org.apache.lucene.index.codecs.SegmentInfosWriter; + +public class AppendingSegmentInfosFormat extends DefaultSegmentInfosFormat { + private final SegmentInfosWriter writer = new AppendingSegmentInfosWriter(); + + @Override + public SegmentInfosWriter getSegmentInfosWriter() { + return writer; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java --- lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java 2011-11-03 08:49:02.324543807 -0400 +++ lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java 2011-11-01 23:41:00.472605256 -0400 @@ -20,8 +20,6 @@ import java.io.IOException; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; public class AppendingSegmentInfosWriter extends DefaultSegmentInfosWriter { diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java --- lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java 2011-07-08 07:13:00.451820239 -0400 +++ lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java 2011-11-02 21:26:22.436564908 -0400 @@ -34,9 +34,9 @@ public AppendingTermsDictReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext context, - int termsCacheSize, int codecId) throws IOException { + int termsCacheSize, String segmentSuffix) throws IOException { super(indexReader, dir, fieldInfos, segment, postingsReader, context, - termsCacheSize, codecId); + termsCacheSize, segmentSuffix); } @Override diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java --- lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java 2011-07-08 07:13:00.451820239 -0400 +++ lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java 2011-11-02 21:26:22.436564908 -0400 @@ -31,9 +31,9 @@ public class AppendingTermsIndexReader extends FixedGapTermsIndexReader { public AppendingTermsIndexReader(Directory dir, FieldInfos fieldInfos, - String segment, int indexDivisor, Comparator termComp, int codecId, IOContext context) + String segment, int indexDivisor, Comparator termComp, String segmentSuffix, IOContext context) throws IOException { - super(dir, fieldInfos, segment, indexDivisor, termComp, codecId, context); + super(dir, fieldInfos, segment, indexDivisor, termComp, segmentSuffix, context); } @Override diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java --- lucene-clean-trunk/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java 2011-06-15 16:16:28.299076474 -0400 +++ lucene-2621/lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java 2011-11-01 21:10:35.760609905 -0400 @@ -27,7 +27,6 @@ import java.util.List; import org.apache.lucene.index.IndexWriter; // Required for javadocs -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.FSDirectory; /** @@ -54,8 +53,6 @@ */ public class IndexSplitter { public SegmentInfos infos; - - private final CodecProvider codecs; FSDirectory fsDir; @@ -96,17 +93,12 @@ is.split(targetDir, segs.toArray(new String[0])); } } - - public IndexSplitter(File dir) throws IOException { - this(dir, CodecProvider.getDefault()); - } - public IndexSplitter(File dir, CodecProvider codecs) throws IOException { + public IndexSplitter(File dir) throws IOException { this.dir = dir; - this.codecs = codecs; fsDir = FSDirectory.open(dir); - infos = new SegmentInfos(codecs); - infos.read(fsDir, codecs); + infos = new SegmentInfos(); + infos.read(fsDir); } public void listSegments() throws IOException { @@ -140,13 +132,13 @@ infos.remove(idx); } infos.changed(); - infos.commit(fsDir); + infos.commit(fsDir, infos.codecFormat()); } public void split(File destDir, String[] segs) throws IOException { destDir.mkdirs(); FSDirectory destFSDir = FSDirectory.open(destDir); - SegmentInfos destInfos = new SegmentInfos(codecs); + SegmentInfos destInfos = new SegmentInfos(); destInfos.counter = infos.counter; for (String n : segs) { SegmentInfo info = getInfo(n); @@ -160,7 +152,7 @@ } } destInfos.changed(); - destInfos.commit(destFSDir); + destInfos.commit(destFSDir, infos.codecFormat()); // System.out.println("destDir:"+destDir.getAbsolutePath()); } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java lucene-2621/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java --- lucene-clean-trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java 2011-11-03 08:49:02.324543807 -0400 +++ lucene-2621/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java 2011-11-03 13:29:09.740535148 -0400 @@ -34,11 +34,6 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.DefaultSegmentInfosReader; -import org.apache.lucene.index.codecs.SegmentInfosReader; -import org.apache.lucene.index.codecs.SegmentInfosWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -50,30 +45,7 @@ public class TestAppendingCodec extends LuceneTestCase { - static class AppendingCodecProvider extends CodecProvider { - Codec appending = new AppendingCodec(); - SegmentInfosWriter infosWriter = new AppendingSegmentInfosWriter(); - SegmentInfosReader infosReader = new DefaultSegmentInfosReader(); - public AppendingCodecProvider() { - setDefaultFieldCodec(appending.name); - } - @Override - public Codec lookup(String name) { - return appending; - } - - @Override - public SegmentInfosReader getSegmentInfosReader() { - return infosReader; - } - @Override - public SegmentInfosWriter getSegmentInfosWriter() { - return infosWriter; - } - - } - - private static class AppendingIndexOutputWrapper extends IndexOutput { + private static class AppendingIndexOutputWrapper extends IndexOutput { IndexOutput wrapped; public AppendingIndexOutputWrapper(IndexOutput wrapped) { @@ -137,7 +109,7 @@ Directory dir = new AppendingRAMDirectory(random, new RAMDirectory()); IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(random)); - cfg.setCodecProvider(new AppendingCodecProvider()); + cfg.setCodec(new AppendingCodec()); ((TieredMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false); IndexWriter writer = new IndexWriter(dir, cfg); Document doc = new Document(); @@ -151,7 +123,7 @@ writer.addDocument(doc); writer.optimize(); writer.close(); - IndexReader reader = IndexReader.open(dir, null, true, 1, new AppendingCodecProvider()); + IndexReader reader = IndexReader.open(dir, null, true, 1); assertEquals(2, reader.numDocs()); Document doc2 = reader.document(0); assertEquals(text, doc2.get("f")); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java lucene-2621/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java --- lucene-clean-trunk/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java 2011-10-08 09:16:20.698772680 -0400 +++ lucene-2621/lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java 2011-10-29 14:42:27.280755439 -0400 @@ -27,7 +27,9 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NRTCachingDirectory; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; +@UseNoMemoryExpensiveCodec public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase { private final ThreadLocal lastGens = new ThreadLocal(); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java lucene-2621/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java --- lucene-clean-trunk/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java 2011-10-23 17:07:48.605018015 -0400 +++ lucene-2621/lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java 2011-10-29 14:42:44.320755431 -0400 @@ -34,9 +34,11 @@ import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; import org.apache.lucene.util.NamedThreadFactory; import org.apache.lucene.util._TestUtil; +@UseNoMemoryExpensiveCodec public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase { boolean warmCalled; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java lucene-2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/CheckIndex.java 2011-10-25 12:50:53.540936932 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/CheckIndex.java 2011-11-01 21:10:35.940609908 -0400 @@ -25,7 +25,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.document.Document; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import java.io.File; import java.io.IOException; @@ -143,8 +143,8 @@ /** Name of the segment. */ public String name; - /** CodecInfo used to read this segment. */ - public SegmentCodecs codec; + /** Codec used to read this segment. */ + public Codec codec; /** Document count (does not take deletions into account). */ public int docCount; @@ -322,10 +322,6 @@ public Status checkIndex() throws IOException { return checkIndex(null); } - - public Status checkIndex(List onlySegments) throws IOException { - return checkIndex(onlySegments, CodecProvider.getDefault()); - } /** Returns a {@link Status} instance detailing * the state of the index. @@ -339,13 +335,13 @@ *

WARNING: make sure * you only call this when the index is not opened by any * writer. */ - public Status checkIndex(List onlySegments, CodecProvider codecs) throws IOException { + public Status checkIndex(List onlySegments) throws IOException { NumberFormat nf = NumberFormat.getInstance(); - SegmentInfos sis = new SegmentInfos(codecs); + SegmentInfos sis = new SegmentInfos(); Status result = new Status(); result.dir = dir; try { - sis.read(dir, codecs); + sis.read(dir); } catch (Throwable t) { msg("ERROR: could not read any segments file in directory"); result.missingSegments = true; @@ -377,6 +373,7 @@ final int numSegments = sis.size(); final String segmentsFileName = sis.getCurrentSegmentFileName(); + // note: we only read the format byte (required preamble) here! IndexInput input = null; try { input = dir.openInput(segmentsFileName, IOContext.DEFAULT); @@ -489,7 +486,7 @@ SegmentReader reader = null; try { - final SegmentCodecs codec = info.getSegmentCodecs(); + final Codec codec = info.getCodec(); msg(" codec=" + codec); segInfoStat.codec = codec; msg(" compound=" + info.getUseCompoundFile()); @@ -1182,11 +1179,11 @@ * *

WARNING: Make sure you only call this when the * index is not opened by any writer. */ - public void fixIndex(Status result) throws IOException { + public void fixIndex(Status result, Codec codec) throws IOException { if (result.partial) throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)"); result.newSegments.changed(); - result.newSegments.commit(result.dir); + result.newSegments.commit(result.dir, codec); } private static boolean assertsOn; @@ -1236,6 +1233,7 @@ public static void main(String[] args) throws IOException, InterruptedException { boolean doFix = false; + Codec codec = Codec.getDefault(); // only used when fixing boolean verbose = false; List onlySegments = new ArrayList(); String indexPath = null; @@ -1244,6 +1242,13 @@ if (args[i].equals("-fix")) { doFix = true; i++; + } else if (args[i].equals("-codec")) { + if (i == args.length-1) { + System.out.println("ERROR: missing name for -codec option"); + System.exit(1); + } + codec = Codec.forName(args[i+1]); + i+=2; } else if (args[i].equals("-verbose")) { verbose = true; i++; @@ -1269,6 +1274,7 @@ System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + + " -codec X: when fixing, codec to write the new segments_N file with\n" + " -verbose: print additional details\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + @@ -1329,7 +1335,7 @@ System.out.println(" " + (5-s) + "..."); } System.out.println("Writing..."); - checker.fixIndex(result); + checker.fixIndex(result, codec); System.out.println("OK"); System.out.println("Wrote new segments file \"" + result.newSegments.getCurrentSegmentFileName() + "\""); } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java 2011-09-15 13:19:31.216254551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java 2011-11-02 21:26:22.544564907 -0400 @@ -109,14 +109,14 @@ // private String segment; public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext context, - int termsCacheSize, int codecId) + int termsCacheSize, String segmentSuffix) throws IOException { this.postingsReader = postingsReader; termsCache = new DoubleBarrelLRUCache(termsCacheSize); // this.segment = segment; - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTermsWriter.TERMS_EXTENSION), + in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTermsWriter.TERMS_EXTENSION), context); boolean success = false; @@ -194,12 +194,8 @@ } } - public static void files(Directory dir, SegmentInfo segmentInfo, int id, Collection files) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, BlockTermsWriter.TERMS_EXTENSION)); - } - - public static void getExtensions(Collection extensions) { - extensions.add(BlockTermsWriter.TERMS_EXTENSION); + public static void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTermsWriter.TERMS_EXTENSION)); } @Override diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java 2011-09-15 13:19:31.216254551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java 2011-11-02 21:26:22.540564909 -0400 @@ -71,7 +71,7 @@ public BlockTermsWriter(TermsIndexWriterBase termsIndexWriter, SegmentWriteState state, PostingsWriterBase postingsWriter) throws IOException { - final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_EXTENSION); this.termsIndexWriter = termsIndexWriter; out = state.directory.createOutput(termsFileName, state.context); boolean success = false; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java 2011-10-15 21:56:47.713365171 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java 2011-11-02 21:26:22.544564907 -0400 @@ -110,13 +110,13 @@ public BlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext ioContext, - int codecId, int indexDivisor) + String segmentSuffix, int indexDivisor) throws IOException { this.postingsReader = postingsReader; this.segment = segment; - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTreeTermsWriter.TERMS_EXTENSION), + in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTreeTermsWriter.TERMS_EXTENSION), ioContext); boolean success = false; @@ -125,7 +125,7 @@ try { readHeader(in); if (indexDivisor != -1) { - indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION), + indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION), ioContext); readIndexHeader(indexIn); } @@ -206,14 +206,9 @@ } } - public static void files(Directory dir, SegmentInfo segmentInfo, int codecID, Collection files) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, BlockTreeTermsWriter.TERMS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION)); - } - - public static void getExtensions(Collection extensions) { - extensions.add(BlockTreeTermsWriter.TERMS_EXTENSION); - extensions.add(BlockTreeTermsWriter.TERMS_INDEX_EXTENSION); + public static void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTreeTermsWriter.TERMS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION)); } @Override diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java 2011-09-15 13:19:31.216254551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java 2011-11-02 21:26:22.544564907 -0400 @@ -145,7 +145,7 @@ throw new IllegalArgumentException("maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock); } - final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_EXTENSION); out = state.directory.createOutput(termsFileName, state.context); boolean success = false; IndexOutput indexOut = null; @@ -157,7 +157,7 @@ //DEBUG = state.segmentName.equals("_4a"); - final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); + final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_INDEX_EXTENSION); indexOut = state.directory.createOutput(termsIndexFileName, state.context); writeIndexHeader(indexOut); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/Codec.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/Codec.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/Codec.java 2011-09-30 20:57:07.298773589 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/Codec.java 2011-11-02 21:26:22.544564907 -0400 @@ -20,59 +20,67 @@ import java.io.IOException; import java.util.Set; -import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.util.NamedSPILoader; import org.apache.lucene.store.Directory; -/** @lucene.experimental */ -public abstract class Codec { - public static final Codec[] EMPTY = new Codec[0]; - /** Unique name that's used to retrieve this codec when - * reading the index */ - public final String name; - - protected Codec(String name) { - this.name = name; - } +/** + * Encodes/decodes an inverted index segment + */ +public abstract class Codec implements NamedSPILoader.NamedSPI { - /** Writes a new segment */ - public abstract FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException; + private static final NamedSPILoader loader = + new NamedSPILoader(Codec.class); - public static void debug(String s, String desc) { - if (desc != null) { - System.out.println(Thread.currentThread().getName()+ " [" + desc + "]:" + s); - } else { - System.out.println(Thread.currentThread().getName() + ": " + s); - } - } - public static void debug(String s) { - debug(s, null); - } - - /** Reads a segment. NOTE: by the time this call - * returns, it must hold open any files it will need to - * use; else, those files may be deleted. */ - public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException; - - public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException; - - public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException; - - /** - * Gathers files associated with this segment - * - * @param dir the {@link Directory} this segment was written to - * @param segmentInfo the {@link SegmentInfo} for this segment - * @param id the codec id within this segment - * @param files the of files to add the codec files to. - */ - public abstract void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException; + private final String name; - /** Records all file extensions this codec uses */ - public abstract void getExtensions(Set extensions); + public Codec(String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + postingsFormat().files(dir, info, "", files); + //TODO: not yet fieldsFormat().files(dir, info, files); + docValuesFormat().files(dir, info, files); + } + + /** Encodes/decodes postings */ + public abstract PostingsFormat postingsFormat(); + /** Encodes/decodes docvalues */ + public abstract DocValuesFormat docValuesFormat(); + + /** Encodes/decodes stored fields, term vectors, fieldinfos */ + public abstract FieldsFormat fieldsFormat(); + + public abstract SegmentInfosFormat segmentInfosFormat(); + + /** looks up a codec by name */ + public static Codec forName(String name) { + return loader.lookup(name); + } + + /** returns a list of all available codec names */ + public static Set availableCodecs() { + return loader.availableServices(); + } + + private static Codec defaultCodec = Codec.forName("Lucene40"); + + // TODO: should we use this, or maybe a system property is better? + public static Codec getDefault() { + return defaultCodec; + } + + public static void setDefault(Codec codec) { + defaultCodec = codec; + } + @Override public String toString() { return name; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java 2011-10-07 11:55:34.568772788 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,199 +0,0 @@ -package org.apache.lucene.index.codecs; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; - -/** Holds a set of codecs, keyed by name. You subclass - * this, instantiate it, and register your codecs, then - * pass this instance to IndexReader/IndexWriter (via - * package private APIs) to use different codecs when - * reading & writing segments. - * - * @lucene.experimental */ - -public class CodecProvider { - private SegmentInfosWriter infosWriter = new DefaultSegmentInfosWriter(); - private SegmentInfosReader infosReader = new DefaultSegmentInfosReader(); - private String defaultFieldCodec = "Standard"; - private final Map perFieldMap = new HashMap(); - - - private final HashMap codecs = new HashMap(); - - private final Set knownExtensions = new HashSet(); - - - public final static String[] CORE_CODECS = new String[] {"Standard", "Pulsing", "PreFlex", "SimpleText", "Memory"}; - - public synchronized void register(Codec codec) { - if (codec.name == null) { - throw new IllegalArgumentException("code.name is null"); - } - if (!codecs.containsKey(codec.name)) { - codecs.put(codec.name, codec); - codec.getExtensions(knownExtensions); - } else if (codecs.get(codec.name) != codec) { - throw new IllegalArgumentException("codec '" + codec.name + "' is already registered as a different codec instance"); - } - } - - /** @lucene.internal */ - public synchronized void unregister(Codec codec) { - if (codec.name == null) { - throw new IllegalArgumentException("code.name is null"); - } - if (codecs.containsKey(codec.name)) { - Codec c = codecs.get(codec.name); - if (codec == c) { - codecs.remove(codec.name); - } else { - throw new IllegalArgumentException("codec '" + codec.name + "' is being impersonated by a different codec instance!!!"); - } - } - } - - /** @lucene.internal */ - public synchronized Set listAll() { - return codecs.keySet(); - } - - public Collection getAllExtensions() { - return knownExtensions; - } - - public synchronized Codec lookup(String name) { - final Codec codec = codecs.get(name); - if (codec == null) { - throw new IllegalArgumentException("required codec '" + name + "' not found; known codecs: " + codecs.keySet()); - } - return codec; - } - - /** - * Returns true iff a codec with the given name is registered - * @param name codec name - * @return true iff a codec with the given name is registered, otherwise false. - */ - public synchronized boolean isCodecRegistered(String name) { - return codecs.containsKey(name); - } - - public SegmentInfosWriter getSegmentInfosWriter() { - return infosWriter; - } - - public SegmentInfosReader getSegmentInfosReader() { - return infosReader; - } - - /** expert */ - public FieldsReader fieldsReader(Directory directory, String segment, FieldInfos fn, IOContext context, int docStoreOffset, int size) throws IOException { - return new DefaultFieldsReader(directory, segment, fn, context, docStoreOffset, size); - } - - /** expert */ - public FieldsWriter fieldsWriter(Directory directory, String segment, IOContext context) throws IOException { - return new DefaultFieldsWriter(directory, segment, context); - } - - static private CodecProvider defaultCodecs = new CoreCodecProvider(); - - public static CodecProvider getDefault() { - return defaultCodecs; - } - - /** For testing only - * @lucene.internal */ - public static void setDefault(CodecProvider cp) { - defaultCodecs = cp; - } - - /** - * Sets the {@link Codec} for a given field. Not that setting a field's codec is - * write-once. If the field's codec is already set this method will throw an - * {@link IllegalArgumentException}. - * - * @param field - * the name of the field - * @param codec - * the name of the codec - * @throws IllegalArgumentException - * if the codec for the given field is already set - * - */ - public synchronized void setFieldCodec(String field, String codec) { - if (perFieldMap.containsKey(field)) - throw new IllegalArgumentException("codec for field: " + field - + " already set to " + perFieldMap.get(field)); - perFieldMap.put(field, codec); - } - - /** - * Returns the {@link Codec} name for the given field or the default codec if - * not set. - * - * @param name - * the fields name - * @return the {@link Codec} name for the given field or the default codec if - * not set. - */ - public synchronized String getFieldCodec(String name) { - final String codec; - if ((codec = perFieldMap.get(name)) == null) { - return defaultFieldCodec; - } - return codec; - } - - /** - * Returns true if this provider has a Codec registered for this - * field. - */ - public synchronized boolean hasFieldCodec(String name) { - return perFieldMap.containsKey(name); - } - - /** - * Returns the default {@link Codec} for this {@link CodecProvider} - * - * @return the default {@link Codec} for this {@link CodecProvider} - */ - public synchronized String getDefaultFieldCodec() { - return defaultFieldCodec; - } - - /** - * Sets the default {@link Codec} for this {@link CodecProvider} - * - * @param codec - * the codecs name - */ - public synchronized void setDefaultFieldCodec(String codec) { - defaultFieldCodec = codec; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java 2011-07-07 14:01:09.601820326 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,51 +0,0 @@ -package org.apache.lucene.index.codecs; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.index.codecs.memory.MemoryCodec; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; -import org.apache.lucene.index.codecs.pulsing.PulsingCodec; -import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; -import org.apache.lucene.index.codecs.standard.StandardCodec; - -/** - * A CodecProvider that registers all core codecs that ship - * with Lucene. This will not register any user codecs, but - * you can easily instantiate this class and register them - * yourself and specify per-field codecs: - * - * 

- *   CodecProvider cp = new CoreCodecProvider();
- *   cp.register(new MyFastCodec());
- *   cp.setDefaultFieldCodec("Standard");
- *   cp.setFieldCodec("id", "Pulsing");
- *   cp.setFieldCodec("body", "MyFastCodec");
- *   IndexWriterConfig iwc = new IndexWriterConfig(analyzer);
- *   iwc.setCodecProvider(cp);
- *
- */ - -public class CoreCodecProvider extends CodecProvider { - public CoreCodecProvider() { - register(new StandardCodec()); - register(new PreFlexCodec()); - register(new PulsingCodec()); - register(new SimpleTextCodec()); - register(new MemoryCodec()); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java 2011-09-30 20:57:07.298773589 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java 2011-11-03 13:43:22.052534709 -0400 @@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.codecs.DocValuesWriterBase; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; @@ -34,42 +33,45 @@ * @lucene.experimental */ public class DefaultDocValuesConsumer extends DocValuesWriterBase { - private final Directory directory; + private final Directory mainDirectory; + private Directory directory; + + final static String DOC_VALUES_SEGMENT_SUFFIX = "dv"; public DefaultDocValuesConsumer(PerDocWriteState state) throws IOException { super(state); + mainDirectory = state.directory; //TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files? - this.directory = new CompoundFileDirectory(state.directory, - IndexFileNames.segmentFileName(state.segmentName, state.codecId, - IndexFileNames.COMPOUND_FILE_EXTENSION), state.context, true); } @Override - protected Directory getDirectory() { + protected Directory getDirectory() throws IOException { + // lazy init + if (directory == null) { + directory = new CompoundFileDirectory(mainDirectory, + IndexFileNames.segmentFileName(segmentName, DOC_VALUES_SEGMENT_SUFFIX, + IndexFileNames.COMPOUND_FILE_EXTENSION), context, true); + } return directory; } @Override public void close() throws IOException { - this.directory.close(); + if (directory != null) { + directory.close(); + } } - @SuppressWarnings("fallthrough") - public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { + public static void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException { FieldInfos fieldInfos = segmentInfo.getFieldInfos(); for (FieldInfo fieldInfo : fieldInfos) { - if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); - assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); - assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); - return; + if (fieldInfo.hasDocValues()) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_EXTENSION)); + break; } } } - - public static void getExtensions(Set extensions) { - extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION); - extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION); - } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesFormat.java 2011-11-02 21:26:22.540564909 -0400 @@ -0,0 +1,44 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.PerDocWriteState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.store.Directory; + +public class DefaultDocValuesFormat extends DocValuesFormat { + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new DefaultDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new DefaultDocValuesProducer(state); + } + + @Override + public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + DefaultDocValuesConsumer.files(dir, info, files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java 2011-09-30 20:57:07.298773589 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java 2011-11-03 13:43:22.052534709 -0400 @@ -26,7 +26,6 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.DocValuesReaderBase; import org.apache.lucene.index.values.IndexDocValues; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; @@ -37,7 +36,7 @@ * @lucene.experimental */ public class DefaultDocValuesProducer extends DocValuesReaderBase { - protected final TreeMap docValues; + protected final TreeMap docValues; private final Directory cfs; /** @@ -45,10 +44,16 @@ * {@link IndexDocValues} instances for this segment and codec. */ public DefaultDocValuesProducer(SegmentReadState state) throws IOException { - cfs = new CompoundFileDirectory(state.dir, - IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), - state.context, false); - docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.codecId, state.context); + if (state.fieldInfos.anyDocValuesFields()) { + cfs = new CompoundFileDirectory(state.dir, + IndexFileNames.segmentFileName(state.segmentInfo.name, + DefaultDocValuesConsumer.DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_EXTENSION), + state.context, false); + docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.context); + } else { + cfs = null; + docValues = new TreeMap(); + } } @Override @@ -58,8 +63,12 @@ @Override protected void closeInternal(Collection closeables) throws IOException { - final ArrayList list = new ArrayList(closeables); - list.add(cfs); - IOUtils.close(list); + if (cfs != null) { + final ArrayList list = new ArrayList(closeables); + list.add(cfs); + IOUtils.close(list); + } else { + IOUtils.close(closeables); + } } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsFormat.java 2011-10-28 16:08:30.432797291 -0400 @@ -0,0 +1,48 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +/** @lucene.experimental */ +public class DefaultFieldsFormat extends FieldsFormat { + + @Override + public FieldsReader fieldsReader(Directory directory, String segment, + FieldInfos fn, IOContext context, int docStoreOffset, int size) + throws IOException { + return new DefaultFieldsReader(directory, segment, fn, context, docStoreOffset, size); + } + + @Override + public FieldsWriter fieldsWriter(Directory directory, String segment, + IOContext context) throws IOException { + return new DefaultFieldsWriter(directory, segment, context); + } + + @Override + public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + // TODO! + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java 2011-10-07 11:55:34.568772788 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java 2011-10-26 10:46:52.992896257 -0400 @@ -82,7 +82,7 @@ /** Verifies that the code version which wrote the segment is supported. */ public static void checkCodeVersion(Directory dir, String segment) throws IOException { - final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); + final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_INDEX_EXTENSION); IndexInput idxStream = dir.openInput(indexStreamFN, IOContext.DEFAULT); try { @@ -121,8 +121,8 @@ try { fieldInfos = fn; - cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); - final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); + cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_EXTENSION), context); + final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_INDEX_EXTENSION); cloneableIndexStream = d.openInput(indexStreamFN, context); format = cloneableIndexStream.readInt(); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java 2011-10-07 11:55:34.568772788 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java 2011-10-26 10:45:56.648896284 -0400 @@ -61,6 +61,10 @@ // when removing support for old versions, leave the last supported version here static final int FORMAT_MINIMUM = FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS; + // TODO: remove from IndexFileNames + public static final String FIELDS_EXTENSION = IndexFileNames.FIELDS_EXTENSION; + public static final String FIELDS_INDEX_EXTENSION = IndexFileNames.FIELDS_INDEX_EXTENSION; + // If null - we were supplied with streams, if notnull - we manage them ourselves private Directory directory; private String segment; @@ -73,8 +77,8 @@ boolean success = false; try { - fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); - indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION), context); + fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context); + indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION), context); fieldsStream.writeInt(FORMAT_CURRENT); indexStream.writeInt(FORMAT_CURRENT); @@ -129,11 +133,11 @@ } catch (IOException ignored) { } try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION)); + directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION)); } catch (IOException ignored) { } try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); + directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION)); } catch (IOException ignored) { } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosFormat.java 2011-11-01 21:10:35.936609908 -0400 @@ -0,0 +1,36 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * @lucene.experimental + */ +public class DefaultSegmentInfosFormat extends SegmentInfosFormat { + private final SegmentInfosReader reader = new DefaultSegmentInfosReader(); + private final SegmentInfosWriter writer = new DefaultSegmentInfosWriter(); + + @Override + public SegmentInfosReader getSegmentInfosReader() { + return reader; + } + + @Override + public SegmentInfosWriter getSegmentInfosWriter() { + return writer; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java 2011-10-07 11:55:34.568772788 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java 2011-11-02 00:53:01.976603032 -0400 @@ -19,17 +19,14 @@ import java.io.IOException; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; /** * Default implementation of {@link SegmentInfosReader}. @@ -37,90 +34,60 @@ */ public class DefaultSegmentInfosReader extends SegmentInfosReader { + // TODO: shove all backwards code to preflex! + // this is a little tricky, because of IR.commit(), two options: + // 1. PreFlex writes 4.x SIS format, but reads both 3.x and 4.x + // (and maybe RW always only writes the 3.x one? for that to work well, + // we have to move .fnx file to codec too, not too bad but more work). + // or we just have crappier RW testing like today. + // 2. PreFlex writes 3.x SIS format, and only reads 3.x + // (in this case we have to move .fnx file to codec as well) @Override - public void read(Directory directory, String segmentsFileName, CodecProvider codecs, - SegmentInfos infos, IOContext context) throws IOException { - IndexInput input = null; - try { - input = openInput(directory, segmentsFileName, context); - final int format = input.readInt(); - infos.setFormat(format); - - // check that it is a format we can understand - if (format > DefaultSegmentInfosWriter.FORMAT_MINIMUM) - throw new IndexFormatTooOldException(segmentsFileName, format, - DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); - if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT) - throw new IndexFormatTooNewException(segmentsFileName, format, - DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); - - infos.version = input.readLong(); // read version - infos.counter = input.readInt(); // read counter - if (infos.getFormat() <= DefaultSegmentInfosWriter.FORMAT_4_0) { - infos.setGlobalFieldMapVersion(input.readLong()); - } - for (int i = input.readInt(); i > 0; i--) { // read segmentInfos - SegmentInfo si = new SegmentInfo(directory, format, input, codecs); - if (si.getVersion() == null) { - // Could be a 3.0 - try to open the doc stores - if it fails, it's a - // 2.x segment, and an IndexFormatTooOldException will be thrown, - // which is what we want. - Directory dir = directory; - if (si.getDocStoreOffset() != -1) { - if (si.getDocStoreIsCompoundFile()) { - dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName( - si.getDocStoreSegment(), "", - IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false); - } - } else if (si.getUseCompoundFile()) { - dir = new CompoundFileDirectory(dir,IndexFileNames.segmentFileName( - si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); + public void read(Directory directory, String segmentsFileName, ChecksumIndexInput input, SegmentInfos infos, IOContext context) throws IOException { + infos.version = input.readLong(); // read version + infos.counter = input.readInt(); // read counter + final int format = infos.getFormat(); + if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { + infos.setGlobalFieldMapVersion(input.readLong()); + } + for (int i = input.readInt(); i > 0; i--) { // read segmentInfos + SegmentInfo si = new SegmentInfo(directory, format, input); + if (si.getVersion() == null) { + // Could be a 3.0 - try to open the doc stores - if it fails, it's a + // 2.x segment, and an IndexFormatTooOldException will be thrown, + // which is what we want. + Directory dir = directory; + if (si.getDocStoreOffset() != -1) { + if (si.getDocStoreIsCompoundFile()) { + dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName( + si.getDocStoreSegment(), "", + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false); } + } else if (si.getUseCompoundFile()) { + dir = new CompoundFileDirectory(dir,IndexFileNames.segmentFileName( + si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); + } - try { - DefaultFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment()); - } finally { - // If we opened the directory, close it - if (dir != directory) dir.close(); - } - - // Above call succeeded, so it's a 3.0 segment. Upgrade it so the next - // time the segment is read, its version won't be null and we won't - // need to open FieldsReader every time for each such segment. - si.setVersion("3.0"); - } else if (si.getVersion().equals("2.x")) { - // If it's a 3x index touched by 3.1+ code, then segments record their - // version, whether they are 2.x ones or not. We detect that and throw - // appropriate exception. - throw new IndexFormatTooOldException(si.name, si.getVersion()); + try { + DefaultFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment()); + } finally { + // If we opened the directory, close it + if (dir != directory) dir.close(); } - infos.add(si); - } - - infos.userData = input.readStringStringMap(); - finalizeInput(input); - - } finally { - if (input != null) { - input.close(); + + // Above call succeeded, so it's a 3.0 segment. Upgrade it so the next + // time the segment is read, its version won't be null and we won't + // need to open FieldsReader every time for each such segment. + si.setVersion("3.0"); + } else if (si.getVersion().equals("2.x")) { + // If it's a 3x index touched by 3.1+ code, then segments record their + // version, whether they are 2.x ones or not. We detect that and throw + // appropriate exception. + throw new IndexFormatTooOldException(si.name, si.getVersion()); } + infos.add(si); } - - } - - public IndexInput openInput(Directory dir, String segmentsFileName, IOContext context) throws IOException { - IndexInput in = dir.openInput(segmentsFileName, context); - return new ChecksumIndexInput(in); - - } - - public void finalizeInput(IndexInput input) throws IOException, CorruptIndexException { - ChecksumIndexInput cksumInput = (ChecksumIndexInput)input; - final long checksumNow = cksumInput.getChecksum(); - final long checksumThen = cksumInput.readLong(); - if (checksumNow != checksumThen) - throw new CorruptIndexException("checksum mismatch in segments file"); - + + infos.userData = input.readStringStringMap(); } - } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java 2011-08-29 12:22:46.841459551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java 2011-11-02 13:42:27.980579247 -0400 @@ -56,12 +56,13 @@ public static final int FORMAT_MINIMUM = FORMAT_DIAGNOSTICS; @Override - public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos, IOContext context) + public IndexOutput writeInfos(Directory dir, String segmentFileName, String codecID, SegmentInfos infos, IOContext context) throws IOException { IndexOutput out = createOutput(dir, segmentFileName, new IOContext(new FlushInfo(infos.size(), infos.totalDocCount()))); boolean success = false; try { out.writeInt(FORMAT_CURRENT); // write FORMAT + out.writeString(codecID); // write codecID out.writeLong(infos.version); out.writeInt(infos.counter); // write counter out.writeLong(infos.getGlobalFieldMapVersion()); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesFormat.java 2011-11-02 21:26:22.544564907 -0400 @@ -0,0 +1,32 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.PerDocWriteState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.store.Directory; + +public abstract class DocValuesFormat { + public abstract PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException; + public abstract PerDocValues docsProducer(SegmentReadState state) throws IOException; + public abstract void files(Directory dir, SegmentInfo info, Set files) throws IOException; +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java 2011-10-07 11:55:34.568772788 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java 2011-11-02 21:26:22.540564909 -0400 @@ -65,19 +65,19 @@ // Only opens files... doesn't actually load any values protected TreeMap load(FieldInfos fieldInfos, - String segment, int docCount, Directory dir, int codecId, IOContext context) + String segment, int docCount, Directory dir, IOContext context) throws IOException { TreeMap values = new TreeMap(); boolean success = false; try { for (FieldInfo fieldInfo : fieldInfos) { - if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) { + if (fieldInfo.hasDocValues()) { final String field = fieldInfo.name; // TODO can we have a compound file per segment and codec for // docvalues? final String id = DefaultDocValuesConsumer.docValuesId(segment, - codecId, fieldInfo.number); + fieldInfo.number); values.put(field, loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), context)); } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java 2011-10-07 11:55:34.568772788 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java 2011-11-03 13:43:22.052534709 -0400 @@ -33,19 +33,19 @@ * @lucene.experimental */ public abstract class DocValuesWriterBase extends PerDocConsumer { - private final String segmentName; - private final int codecId; + protected final String segmentName; + protected final String segmentSuffix; private final Counter bytesUsed; - private final IOContext context; + protected final IOContext context; protected DocValuesWriterBase(PerDocWriteState state) { this.segmentName = state.segmentName; - this.codecId = state.codecId; + this.segmentSuffix = state.segmentSuffix; this.bytesUsed = state.bytesUsed; this.context = state.context; } - protected abstract Directory getDirectory(); + protected abstract Directory getDirectory() throws IOException; @Override public void close() throws IOException { @@ -54,12 +54,12 @@ @Override public DocValuesConsumer addValuesField(FieldInfo field) throws IOException { return Writer.create(field.getDocValues(), - docValuesId(segmentName, codecId, field.number), + docValuesId(segmentName, field.number), getDirectory(), getComparator(), bytesUsed, context); } - public static String docValuesId(String segmentsName, int codecID, int fieldId) { - return segmentsName + "_" + codecID + "-" + fieldId; + public static String docValuesId(String segmentsName, int fieldId) { + return segmentsName + "_" + fieldId; } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/FieldsFormat.java 2011-10-28 16:08:30.432797291 -0400 @@ -0,0 +1,35 @@ +package org.apache.lucene.index.codecs; + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Controls the format of stored fields/termvectors/... + */ +public abstract class FieldsFormat { + public abstract FieldsReader fieldsReader(Directory directory, String segment, FieldInfos fn, IOContext context, int docStoreOffset, int size) throws IOException; + public abstract FieldsWriter fieldsWriter(Directory directory, String segment, IOContext context) throws IOException; + public abstract void files(Directory dir, SegmentInfo info, Set files) throws IOException; +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java 2011-07-20 22:08:38.168158123 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java 2011-10-28 12:47:18.284803510 -0400 @@ -34,23 +34,4 @@ public abstract class FieldsProducer extends Fields implements Closeable { public abstract void close() throws IOException; - - public static final FieldsProducer EMPTY = new FieldsProducer() { - - @Override - public Terms terms(String field) throws IOException { - return null; - } - - @Override - public FieldsEnum iterator() throws IOException { - return FieldsEnum.EMPTY; - } - - @Override - public void close() throws IOException { - - } - }; - } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java 2011-08-29 12:22:46.841459551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java 2011-11-02 21:26:22.540564909 -0400 @@ -68,14 +68,14 @@ // start of the field info data protected long dirOffset; - public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, int codecId, IOContext context) + public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, String segmentSuffix, IOContext context) throws IOException { this.termComp = termComp; assert indexDivisor == -1 || indexDivisor > 0; - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); + in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); boolean success = false; @@ -387,17 +387,8 @@ } } - public static void files(Directory dir, SegmentInfo info, int id, Collection files) { - files.add(IndexFileNames.segmentFileName(info.name, id, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); - } - - public static void getIndexExtensions(Collection extensions) { - extensions.add(FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION); - } - - @Override - public void getExtensions(Collection extensions) { - getIndexExtensions(extensions); + public static void files(Directory dir, SegmentInfo info, String segmentSuffix, Collection files) { + files.add(IndexFileNames.segmentFileName(info.name, segmentSuffix, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); } @Override diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java 2011-08-29 12:22:46.841459551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java 2011-11-02 21:26:22.544564907 -0400 @@ -56,7 +56,7 @@ private final FieldInfos fieldInfos; // unread public FixedGapTermsIndexWriter(SegmentWriteState state) throws IOException { - final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); + final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_INDEX_EXTENSION); termIndexInterval = state.termIndexInterval; out = state.directory.createOutput(indexFileName, state.context); boolean success = false; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xCodec.java 2011-11-02 21:26:22.512564909 -0400 @@ -0,0 +1,90 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.PerDocWriteState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultFieldsFormat; +import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; +import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldsFormat; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.SegmentInfosFormat; +import org.apache.lucene.store.Directory; + +/** + * Supports the Lucene 3.x index format (readonly) + */ +public class Lucene3xCodec extends Codec { + public Lucene3xCodec() { + super("Lucene3x"); + } + + private final PostingsFormat postingsFormat = new Lucene3xPostingsFormat(); + + // TODO: this should really be a different impl + private final FieldsFormat fieldsFormat = new DefaultFieldsFormat(); + + // TODO: this should really be a different impl + // also if we want preflex to *really* be read-only it should throw exception for the writer? + // this way IR.commit fails on delete/undelete/setNorm/etc ? + private final SegmentInfosFormat infosFormat = new DefaultSegmentInfosFormat(); + + // 3.x doesn't support docvalues + private final DocValuesFormat docValuesFormat = new DocValuesFormat() { + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return null; + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return null; + } + + @Override + public void files(Directory dir, SegmentInfo info, Set files) throws IOException {} + }; + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + @Override + public FieldsFormat fieldsFormat() { + return fieldsFormat; + } + + @Override + public SegmentInfosFormat segmentInfosFormat() { + return infosFormat; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java 2011-10-26 14:02:53.812890194 -0400 @@ -0,0 +1,1115 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.TreeMap; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.store.CompoundFileDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.UnicodeUtil; + +/** Exposes flex API on a pre-flex index, as a codec. + * @lucene.experimental + * @deprecated (4.0) + */ +@Deprecated +public class Lucene3xFields extends FieldsProducer { + + private static final boolean DEBUG_SURROGATES = false; + + public TermInfosReader tis; + public final TermInfosReader tisNoIndex; + + public final IndexInput freqStream; + public final IndexInput proxStream; + final private FieldInfos fieldInfos; + private final SegmentInfo si; + final TreeMap fields = new TreeMap(); + final Map preTerms = new HashMap(); + private final Directory dir; + private final IOContext context; + private Directory cfsReader; + + public Lucene3xFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, IOContext context, int indexDivisor) + throws IOException { + + si = info; + + // NOTE: we must always load terms index, even for + // "sequential" scan during merging, because what is + // sequential to merger may not be to TermInfosReader + // since we do the surrogates dance: + if (indexDivisor < 0) { + indexDivisor = -indexDivisor; + } + + boolean success = false; + try { + TermInfosReader r = new TermInfosReader(dir, info.name, fieldInfos, context, indexDivisor); + if (indexDivisor == -1) { + tisNoIndex = r; + } else { + tisNoIndex = null; + tis = r; + } + this.context = context; + this.fieldInfos = fieldInfos; + + // make sure that all index files have been read or are kept open + // so that if an index update removes them we'll still have them + freqStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.FREQ_EXTENSION), context); + boolean anyProx = false; + for (FieldInfo fi : fieldInfos) { + if (fi.isIndexed) { + fields.put(fi.name, fi); + preTerms.put(fi.name, new PreTerms(fi)); + if (fi.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { + anyProx = true; + } + } + } + + if (anyProx) { + proxStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.PROX_EXTENSION), context); + } else { + proxStream = null; + } + success = true; + } finally { + // With lock-less commits, it's entirely possible (and + // fine) to hit a FileNotFound exception above. In + // this case, we want to explicitly close any subset + // of things that were opened so that we don't have to + // wait for a GC to do so. + if (!success) { + close(); + } + } + this.dir = dir; + } + + // If this returns, we do the surrogates dance so that the + // terms are sorted by unicode sort order. This should be + // true when segments are used for "normal" searching; + // it's only false during testing, to create a pre-flex + // index, using the test-only PreFlexRW. + protected boolean sortTermsByUnicode() { + return true; + } + + static void files(Directory dir, SegmentInfo info, Collection files) throws IOException { + files.add(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.TERMS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION)); + files.add(IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.FREQ_EXTENSION)); + if (info.getHasProx()) { + // LUCENE-1739: for certain versions of 2.9-dev, + // hasProx would be incorrectly computed during + // indexing as true, and then stored into the segments + // file, when it should have been false. So we do the + // extra check, here: + final String prx = IndexFileNames.segmentFileName(info.name, "", Lucene3xPostingsFormat.PROX_EXTENSION); + if (dir.fileExists(prx)) { + files.add(prx); + } + } + } + + @Override + public FieldsEnum iterator() throws IOException { + return new PreFlexFieldsEnum(); + } + + @Override + public Terms terms(String field) { + return preTerms.get(field); + } + + @Override + public long getUniqueTermCount() throws IOException { + return getTermsDict().size(); + } + + synchronized private TermInfosReader getTermsDict() { + if (tis != null) { + return tis; + } else { + return tisNoIndex; + } + } + + @Override + public void close() throws IOException { + if (tis != null) { + tis.close(); + } + if (tisNoIndex != null) { + tisNoIndex.close(); + } + if (cfsReader != null) { + cfsReader.close(); + } + if (freqStream != null) { + freqStream.close(); + } + if (proxStream != null) { + proxStream.close(); + } + } + + private class PreFlexFieldsEnum extends FieldsEnum { + final Iterator it; + private final PreTermsEnum termsEnum; + FieldInfo current; + + public PreFlexFieldsEnum() throws IOException { + it = fields.values().iterator(); + termsEnum = new PreTermsEnum(); + } + + @Override + public String next() { + if (it.hasNext()) { + current = it.next(); + return current.name; + } else { + return null; + } + } + + @Override + public TermsEnum terms() throws IOException { + termsEnum.reset(current); + return termsEnum; + } + } + + private class PreTerms extends Terms { + final FieldInfo fieldInfo; + PreTerms(FieldInfo fieldInfo) { + this.fieldInfo = fieldInfo; + } + + @Override + public TermsEnum iterator() throws IOException { + PreTermsEnum termsEnum = new PreTermsEnum(); + termsEnum.reset(fieldInfo); + return termsEnum; + } + + @Override + public Comparator getComparator() { + // Pre-flex indexes always sorted in UTF16 order, but + // we remap on-the-fly to unicode order + if (sortTermsByUnicode()) { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } else { + return BytesRef.getUTF8SortedAsUTF16Comparator(); + } + } + + @Override + public long getUniqueTermCount() throws IOException { + return -1; + } + + @Override + public long getSumTotalTermFreq() { + return -1; + } + + @Override + public long getSumDocFreq() throws IOException { + return -1; + } + + @Override + public int getDocCount() throws IOException { + return -1; + } + } + + private class PreTermsEnum extends TermsEnum { + private SegmentTermEnum termEnum; + private FieldInfo fieldInfo; + private String internedFieldName; + private boolean skipNext; + private BytesRef current; + + private SegmentTermEnum seekTermEnum; + + private static final byte UTF8_NON_BMP_LEAD = (byte) 0xf0; + private static final byte UTF8_HIGH_BMP_LEAD = (byte) 0xee; + + // Returns true if the unicode char is "after" the + // surrogates in UTF16, ie >= U+E000 and <= U+FFFF: + private final boolean isHighBMPChar(byte[] b, int idx) { + return (b[idx] & UTF8_HIGH_BMP_LEAD) == UTF8_HIGH_BMP_LEAD; + } + + // Returns true if the unicode char in the UTF8 byte + // sequence starting at idx encodes a char outside of + // BMP (ie what would be a surrogate pair in UTF16): + private final boolean isNonBMPChar(byte[] b, int idx) { + return (b[idx] & UTF8_NON_BMP_LEAD) == UTF8_NON_BMP_LEAD; + } + + private final byte[] scratch = new byte[4]; + private final BytesRef prevTerm = new BytesRef(); + private final BytesRef scratchTerm = new BytesRef(); + private int newSuffixStart; + + // Swap in S, in place of E: + private boolean seekToNonBMP(SegmentTermEnum te, BytesRef term, int pos) throws IOException { + final int savLength = term.length; + + assert term.offset == 0; + + // The 3 bytes starting at downTo make up 1 + // unicode character: + assert isHighBMPChar(term.bytes, pos); + + // NOTE: we cannot make this assert, because + // AutomatonQuery legitimately sends us malformed UTF8 + // (eg the UTF8 bytes with just 0xee) + // assert term.length >= pos + 3: "term.length=" + term.length + " pos+3=" + (pos+3) + " byte=" + Integer.toHexString(term.bytes[pos]) + " term=" + term.toString(); + + // Save the bytes && length, since we need to + // restore this if seek "back" finds no matching + // terms + if (term.bytes.length < 4+pos) { + term.grow(4+pos); + } + + scratch[0] = term.bytes[pos]; + scratch[1] = term.bytes[pos+1]; + scratch[2] = term.bytes[pos+2]; + + term.bytes[pos] = (byte) 0xf0; + term.bytes[pos+1] = (byte) 0x90; + term.bytes[pos+2] = (byte) 0x80; + term.bytes[pos+3] = (byte) 0x80; + term.length = 4+pos; + + if (DEBUG_SURROGATES) { + System.out.println(" try seek term=" + UnicodeUtil.toHexString(term.utf8ToString())); + } + + // Seek "back": + getTermsDict().seekEnum(te, new Term(fieldInfo.name, term), true); + + // Test if the term we seek'd to in fact found a + // surrogate pair at the same position as the E: + Term t2 = te.term(); + + // Cannot be null (or move to next field) because at + // "worst" it'd seek to the same term we are on now, + // unless we are being called from seek + if (t2 == null || t2.field() != internedFieldName) { + return false; + } + + if (DEBUG_SURROGATES) { + System.out.println(" got term=" + UnicodeUtil.toHexString(t2.text())); + } + + // Now test if prefix is identical and we found + // a non-BMP char at the same position: + BytesRef b2 = t2.bytes(); + assert b2.offset == 0; + + boolean matches; + if (b2.length >= term.length && isNonBMPChar(b2.bytes, pos)) { + matches = true; + for(int i=0;i limit) { + + if (isHighBMPChar(prevTerm.bytes, downTo)) { + + if (DEBUG_SURROGATES) { + System.out.println(" found E pos=" + downTo + " vs len=" + prevTerm.length); + } + + if (seekToNonBMP(seekTermEnum, prevTerm, downTo)) { + // TODO: more efficient seek? + getTermsDict().seekEnum(termEnum, seekTermEnum.term(), true); + //newSuffixStart = downTo+4; + newSuffixStart = downTo; + scratchTerm.copy(termEnum.term().bytes()); + didSeek = true; + if (DEBUG_SURROGATES) { + System.out.println(" seek!"); + } + break; + } else { + if (DEBUG_SURROGATES) { + System.out.println(" no seek"); + } + } + } + + // Shorten prevTerm in place so that we don't redo + // this loop if we come back here: + if ((prevTerm.bytes[downTo] & 0xc0) == 0xc0 || (prevTerm.bytes[downTo] & 0x80) == 0) { + prevTerm.length = downTo; + } + + downTo--; + } + + return didSeek; + } + + // Look for seek type 3 ("pop"): if the delta from + // prev -> current was replacing an S with an E, + // we must now seek to beyond that E. This seek + // "finishes" the dance at this character + // position. + private boolean doPop() throws IOException { + + if (DEBUG_SURROGATES) { + System.out.println(" try pop"); + } + + assert newSuffixStart <= prevTerm.length; + assert newSuffixStart < scratchTerm.length || newSuffixStart == 0; + + if (prevTerm.length > newSuffixStart && + isNonBMPChar(prevTerm.bytes, newSuffixStart) && + isHighBMPChar(scratchTerm.bytes, newSuffixStart)) { + + // Seek type 2 -- put 0xFF at this position: + scratchTerm.bytes[newSuffixStart] = (byte) 0xff; + scratchTerm.length = newSuffixStart+1; + + if (DEBUG_SURROGATES) { + System.out.println(" seek to term=" + UnicodeUtil.toHexString(scratchTerm.utf8ToString()) + " " + scratchTerm.toString()); + } + + // TODO: more efficient seek? can we simply swap + // the enums? + getTermsDict().seekEnum(termEnum, new Term(fieldInfo.name, scratchTerm), true); + + final Term t2 = termEnum.term(); + + // We could hit EOF or different field since this + // was a seek "forward": + if (t2 != null && t2.field() == internedFieldName) { + + if (DEBUG_SURROGATES) { + System.out.println(" got term=" + UnicodeUtil.toHexString(t2.text()) + " " + t2.bytes()); + } + + final BytesRef b2 = t2.bytes(); + assert b2.offset == 0; + + + // Set newSuffixStart -- we can't use + // termEnum's since the above seek may have + // done no scanning (eg, term was precisely + // and index term, or, was in the term seek + // cache): + scratchTerm.copy(b2); + setNewSuffixStart(prevTerm, scratchTerm); + + return true; + } else if (newSuffixStart != 0 || scratchTerm.length != 0) { + if (DEBUG_SURROGATES) { + System.out.println(" got term=null (or next field)"); + } + newSuffixStart = 0; + scratchTerm.length = 0; + return true; + } + } + + return false; + } + + // Pre-flex indices store terms in UTF16 sort order, but + // certain queries require Unicode codepoint order; this + // method carefully seeks around surrogates to handle + // this impedance mismatch + + private void surrogateDance() throws IOException { + + if (!unicodeSortOrder) { + return; + } + + // We are invoked after TIS.next() (by UTF16 order) to + // possibly seek to a different "next" (by unicode + // order) term. + + // We scan only the "delta" from the last term to the + // current term, in UTF8 bytes. We look at 1) the bytes + // stripped from the prior term, and then 2) the bytes + // appended to that prior term's prefix. + + // We don't care about specific UTF8 sequences, just + // the "category" of the UTF16 character. Category S + // is a high/low surrogate pair (it non-BMP). + // Category E is any BMP char > UNI_SUR_LOW_END (and < + // U+FFFF). Category A is the rest (any unicode char + // <= UNI_SUR_HIGH_START). + + // The core issue is that pre-flex indices sort the + // characters as ASE, while flex must sort as AES. So + // when scanning, when we hit S, we must 1) seek + // forward to E and enum the terms there, then 2) seek + // back to S and enum all terms there, then 3) seek to + // after E. Three different seek points (1, 2, 3). + + // We can easily detect S in UTF8: if a byte has + // prefix 11110 (0xf0), then that byte and the + // following 3 bytes encode a single unicode codepoint + // in S. Similarly, we can detect E: if a byte has + // prefix 1110111 (0xee), then that byte and the + // following 2 bytes encode a single unicode codepoint + // in E. + + // Note that this is really a recursive process -- + // maybe the char at pos 2 needs to dance, but any + // point in its dance, suddenly pos 4 needs to dance + // so you must finish pos 4 before returning to pos + // 2. But then during pos 4's dance maybe pos 7 needs + // to dance, etc. However, despite being recursive, + // we don't need to hold any state because the state + // can always be derived by looking at prior term & + // current term. + + // TODO: can we avoid this copy? + if (termEnum.term() == null || termEnum.term().field() != internedFieldName) { + scratchTerm.length = 0; + } else { + scratchTerm.copy(termEnum.term().bytes()); + } + + if (DEBUG_SURROGATES) { + System.out.println(" dance"); + System.out.println(" prev=" + UnicodeUtil.toHexString(prevTerm.utf8ToString())); + System.out.println(" " + prevTerm.toString()); + System.out.println(" term=" + UnicodeUtil.toHexString(scratchTerm.utf8ToString())); + System.out.println(" " + scratchTerm.toString()); + } + + // This code assumes TermInfosReader/SegmentTermEnum + // always use BytesRef.offset == 0 + assert prevTerm.offset == 0; + assert scratchTerm.offset == 0; + + // Need to loop here because we may need to do multiple + // pops, and possibly a continue in the end, ie: + // + // cont + // pop, cont + // pop, pop, cont + // + // + + while(true) { + if (doContinue()) { + break; + } else { + if (!doPop()) { + break; + } + } + } + + if (DEBUG_SURROGATES) { + System.out.println(" finish bmp ends"); + } + + doPushes(); + } + + + // Look for seek type 1 ("push"): if the newly added + // suffix contains any S, we must try to seek to the + // corresponding E. If we find a match, we go there; + // else we keep looking for additional S's in the new + // suffix. This "starts" the dance, at this character + // position: + private void doPushes() throws IOException { + + int upTo = newSuffixStart; + if (DEBUG_SURROGATES) { + System.out.println(" try push newSuffixStart=" + newSuffixStart + " scratchLen=" + scratchTerm.length); + } + + while(upTo < scratchTerm.length) { + if (isNonBMPChar(scratchTerm.bytes, upTo) && + (upTo > newSuffixStart || + (upTo >= prevTerm.length || + (!isNonBMPChar(prevTerm.bytes, upTo) && + !isHighBMPChar(prevTerm.bytes, upTo))))) { + + // A non-BMP char (4 bytes UTF8) starts here: + assert scratchTerm.length >= upTo + 4; + + final int savLength = scratchTerm.length; + scratch[0] = scratchTerm.bytes[upTo]; + scratch[1] = scratchTerm.bytes[upTo+1]; + scratch[2] = scratchTerm.bytes[upTo+2]; + + scratchTerm.bytes[upTo] = UTF8_HIGH_BMP_LEAD; + scratchTerm.bytes[upTo+1] = (byte) 0x80; + scratchTerm.bytes[upTo+2] = (byte) 0x80; + scratchTerm.length = upTo+3; + + if (DEBUG_SURROGATES) { + System.out.println(" try seek 1 pos=" + upTo + " term=" + UnicodeUtil.toHexString(scratchTerm.utf8ToString()) + " " + scratchTerm.toString() + " len=" + scratchTerm.length); + } + + // Seek "forward": + // TODO: more efficient seek? + getTermsDict().seekEnum(seekTermEnum, new Term(fieldInfo.name, scratchTerm), true); + + scratchTerm.bytes[upTo] = scratch[0]; + scratchTerm.bytes[upTo+1] = scratch[1]; + scratchTerm.bytes[upTo+2] = scratch[2]; + scratchTerm.length = savLength; + + // Did we find a match? + final Term t2 = seekTermEnum.term(); + + if (DEBUG_SURROGATES) { + if (t2 == null) { + System.out.println(" hit term=null"); + } else { + System.out.println(" hit term=" + UnicodeUtil.toHexString(t2.text()) + " " + (t2==null? null:t2.bytes())); + } + } + + // Since this was a seek "forward", we could hit + // EOF or a different field: + boolean matches; + + if (t2 != null && t2.field() == internedFieldName) { + final BytesRef b2 = t2.bytes(); + assert b2.offset == 0; + if (b2.length >= upTo+3 && isHighBMPChar(b2.bytes, upTo)) { + matches = true; + for(int i=0;i BMP + upTo += 3; + + // NOTE: we keep iterating, now, since this + // can easily "recurse". Ie, after seeking + // forward at a certain char position, we may + // find another surrogate in our [new] suffix + // and must then do another seek (recurse) + } else { + upTo++; + } + } else { + upTo++; + } + } + } + + private boolean unicodeSortOrder; + + void reset(FieldInfo fieldInfo) throws IOException { + //System.out.println("pff.reset te=" + termEnum); + this.fieldInfo = fieldInfo; + internedFieldName = fieldInfo.name.intern(); + final Term term = new Term(internedFieldName); + if (termEnum == null) { + termEnum = getTermsDict().terms(term); + seekTermEnum = getTermsDict().terms(term); + //System.out.println(" term=" + termEnum.term()); + } else { + getTermsDict().seekEnum(termEnum, term, true); + } + skipNext = true; + + unicodeSortOrder = sortTermsByUnicode(); + + final Term t = termEnum.term(); + if (t != null && t.field() == internedFieldName) { + newSuffixStart = 0; + prevTerm.length = 0; + surrogateDance(); + } + } + + @Override + public Comparator getComparator() { + // Pre-flex indexes always sorted in UTF16 order, but + // we remap on-the-fly to unicode order + if (unicodeSortOrder) { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } else { + return BytesRef.getUTF8SortedAsUTF16Comparator(); + } + } + + @Override + public void seekExact(long ord) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long ord() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public SeekStatus seekCeil(BytesRef term, boolean useCache) throws IOException { + if (DEBUG_SURROGATES) { + System.out.println("TE.seek target=" + UnicodeUtil.toHexString(term.utf8ToString())); + } + skipNext = false; + final TermInfosReader tis = getTermsDict(); + final Term t0 = new Term(fieldInfo.name, term); + + assert termEnum != null; + + tis.seekEnum(termEnum, t0, useCache); + + final Term t = termEnum.term(); + + if (t != null && t.field() == internedFieldName && term.bytesEquals(t.bytes())) { + // If we found an exact match, no need to do the + // surrogate dance + if (DEBUG_SURROGATES) { + System.out.println(" seek exact match"); + } + current = t.bytes(); + return SeekStatus.FOUND; + } else if (t == null || t.field() != internedFieldName) { + + // TODO: maybe we can handle this like the next() + // into null? set term as prevTerm then dance? + + if (DEBUG_SURROGATES) { + System.out.println(" seek hit EOF"); + } + + // We hit EOF; try end-case surrogate dance: if we + // find an E, try swapping in S, backwards: + scratchTerm.copy(term); + + assert scratchTerm.offset == 0; + + for(int i=scratchTerm.length-1;i>=0;i--) { + if (isHighBMPChar(scratchTerm.bytes, i)) { + if (DEBUG_SURROGATES) { + System.out.println(" found E pos=" + i + "; try seek"); + } + + if (seekToNonBMP(seekTermEnum, scratchTerm, i)) { + + scratchTerm.copy(seekTermEnum.term().bytes()); + getTermsDict().seekEnum(termEnum, seekTermEnum.term(), useCache); + + newSuffixStart = 1+i; + + doPushes(); + + // Found a match + // TODO: faster seek? + current = termEnum.term().bytes(); + return SeekStatus.NOT_FOUND; + } + } + } + + if (DEBUG_SURROGATES) { + System.out.println(" seek END"); + } + + current = null; + return SeekStatus.END; + } else { + + // We found a non-exact but non-null term; this one + // is fun -- just treat it like next, by pretending + // requested term was prev: + prevTerm.copy(term); + + if (DEBUG_SURROGATES) { + System.out.println(" seek hit non-exact term=" + UnicodeUtil.toHexString(t.text())); + } + + final BytesRef br = t.bytes(); + assert br.offset == 0; + + setNewSuffixStart(term, br); + + surrogateDance(); + + final Term t2 = termEnum.term(); + if (t2 == null || t2.field() != internedFieldName) { + // PreFlex codec interns field names; verify: + assert t2 == null || !t2.field().equals(internedFieldName); + current = null; + return SeekStatus.END; + } else { + current = t2.bytes(); + assert !unicodeSortOrder || term.compareTo(current) < 0 : "term=" + UnicodeUtil.toHexString(term.utf8ToString()) + " vs current=" + UnicodeUtil.toHexString(current.utf8ToString()); + return SeekStatus.NOT_FOUND; + } + } + } + + private void setNewSuffixStart(BytesRef br1, BytesRef br2) { + final int limit = Math.min(br1.length, br2.length); + int lastStart = 0; + for(int i=0;i files) throws IOException { + // preflex fields have no segmentSuffix - we ignore it here + Lucene3xFields.files(dir, info, files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java 2011-10-26 14:02:53.800890197 -0400 @@ -0,0 +1,228 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.codecs.lucene40.DefaultSkipListReader; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.Bits; + +/** @deprecated (4.0) + * @lucene.experimental */ +@Deprecated +public class SegmentTermDocs { + //protected SegmentReader parent; + private final FieldInfos fieldInfos; + private final TermInfosReader tis; + protected Bits liveDocs; + protected IndexInput freqStream; + protected int count; + protected int df; + int doc = 0; + int freq; + + private int skipInterval; + private int maxSkipLevels; + private DefaultSkipListReader skipListReader; + + private long freqBasePointer; + private long proxBasePointer; + + private long skipPointer; + private boolean haveSkipped; + + protected boolean currentFieldStoresPayloads; + protected IndexOptions indexOptions; + + public SegmentTermDocs(IndexInput freqStream, TermInfosReader tis, FieldInfos fieldInfos) { + this.freqStream = (IndexInput) freqStream.clone(); + this.tis = tis; + this.fieldInfos = fieldInfos; + skipInterval = tis.getSkipInterval(); + maxSkipLevels = tis.getMaxSkipLevels(); + } + + public void seek(Term term) throws IOException { + TermInfo ti = tis.get(term); + seek(ti, term); + } + + public void setLiveDocs(Bits liveDocs) { + this.liveDocs = liveDocs; + } + + public void seek(SegmentTermEnum segmentTermEnum) throws IOException { + TermInfo ti; + Term term; + + // use comparison of fieldinfos to verify that termEnum belongs to the same segment as this SegmentTermDocs + if (segmentTermEnum.fieldInfos == fieldInfos) { // optimized case + term = segmentTermEnum.term(); + ti = segmentTermEnum.termInfo(); + } else { // punt case + term = segmentTermEnum.term(); + ti = tis.get(term); + } + + seek(ti, term); + } + + void seek(TermInfo ti, Term term) throws IOException { + count = 0; + FieldInfo fi = fieldInfos.fieldInfo(term.field()); + this.indexOptions = (fi != null) ? fi.indexOptions : IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + currentFieldStoresPayloads = (fi != null) ? fi.storePayloads : false; + if (ti == null) { + df = 0; + } else { + df = ti.docFreq; + doc = 0; + freqBasePointer = ti.freqPointer; + proxBasePointer = ti.proxPointer; + skipPointer = freqBasePointer + ti.skipOffset; + freqStream.seek(freqBasePointer); + haveSkipped = false; + } + } + + public void close() throws IOException { + freqStream.close(); + if (skipListReader != null) + skipListReader.close(); + } + + public final int doc() { return doc; } + public final int freq() { return freq; } + + protected void skippingDoc() throws IOException { + } + + public boolean next() throws IOException { + while (true) { + if (count == df) + return false; + final int docCode = freqStream.readVInt(); + + if (indexOptions == IndexOptions.DOCS_ONLY) { + doc += docCode; + freq = 1; + } else { + doc += docCode >>> 1; // shift off low bit + if ((docCode & 1) != 0) // if low bit is set + freq = 1; // freq is one + else { + freq = freqStream.readVInt(); // else read freq + assert freq != 1; + } + } + + count++; + + if (liveDocs == null || liveDocs.get(doc)) { + break; + } + skippingDoc(); + } + return true; + } + + /** Optimized implementation. */ + public int read(final int[] docs, final int[] freqs) + throws IOException { + final int length = docs.length; + if (indexOptions == IndexOptions.DOCS_ONLY) { + return readNoTf(docs, freqs, length); + } else { + int i = 0; + while (i < length && count < df) { + // manually inlined call to next() for speed + final int docCode = freqStream.readVInt(); + doc += docCode >>> 1; // shift off low bit + if ((docCode & 1) != 0) // if low bit is set + freq = 1; // freq is one + else + freq = freqStream.readVInt(); // else read freq + count++; + + if (liveDocs == null || liveDocs.get(doc)) { + docs[i] = doc; + freqs[i] = freq; + ++i; + } + } + return i; + } + } + + private final int readNoTf(final int[] docs, final int[] freqs, final int length) throws IOException { + int i = 0; + while (i < length && count < df) { + // manually inlined call to next() for speed + doc += freqStream.readVInt(); + count++; + + if (liveDocs == null || liveDocs.get(doc)) { + docs[i] = doc; + // Hardware freq to 1 when term freqs were not + // stored in the index + freqs[i] = 1; + ++i; + } + } + return i; + } + + + /** Overridden by SegmentTermPositions to skip in prox stream. */ + protected void skipProx(long proxPointer, int payloadLength) throws IOException {} + + /** Optimized implementation. */ + public boolean skipTo(int target) throws IOException { + // don't skip if the target is close (within skipInterval docs away) + if ((target - skipInterval) >= doc && df >= skipInterval) { // optimized case + if (skipListReader == null) + skipListReader = new DefaultSkipListReader((IndexInput) freqStream.clone(), maxSkipLevels, skipInterval); // lazily clone + + if (!haveSkipped) { // lazily initialize skip stream + skipListReader.init(skipPointer, freqBasePointer, proxBasePointer, df, currentFieldStoresPayloads); + haveSkipped = true; + } + + int newCount = skipListReader.skipTo(target); + if (newCount > count) { + freqStream.seek(skipListReader.getFreqPointer()); + skipProx(skipListReader.getProxPointer(), skipListReader.getPayloadLength()); + + doc = skipListReader.getDoc(); + count = newCount; + } + } + + // done skipping, now just scan + do { + if (!next()) + return false; + } while (target > doc); + return true; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermEnum.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermEnum.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermEnum.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermEnum.java 2011-10-26 14:02:03.768890220 -0400 @@ -0,0 +1,225 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexFormatTooOldException; +import org.apache.lucene.index.IndexFormatTooNewException; + +/** + * @deprecated (4.0) No longer used with flex indexing, except for + * reading old segments + * @lucene.experimental */ + +@Deprecated +public final class SegmentTermEnum implements Cloneable { + private IndexInput input; + FieldInfos fieldInfos; + long size; + long position = -1; + + // Changed strings to true utf8 with length-in-bytes not + // length-in-chars + public static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = -4; + + // NOTE: always change this if you switch to a new format! + // whenever you add a new format, make it 1 smaller (negative version logic)! + public static final int FORMAT_CURRENT = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; + + // when removing support for old versions, leave the last supported version here + public static final int FORMAT_MINIMUM = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; + + private TermBuffer termBuffer = new TermBuffer(); + private TermBuffer prevBuffer = new TermBuffer(); + private TermBuffer scanBuffer = new TermBuffer(); // used for scanning + + TermInfo termInfo = new TermInfo(); + + private int format; + private boolean isIndex = false; + long indexPointer = 0; + int indexInterval; + int skipInterval; + int newSuffixStart; + int maxSkipLevels; + private boolean first = true; + + SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi) + throws CorruptIndexException, IOException { + input = i; + fieldInfos = fis; + isIndex = isi; + maxSkipLevels = 1; // use single-level skip lists for formats > -3 + + int firstInt = input.readInt(); + if (firstInt >= 0) { + // original-format file, without explicit format version number + format = 0; + size = firstInt; + + // back-compatible settings + indexInterval = 128; + skipInterval = Integer.MAX_VALUE; // switch off skipTo optimization + } else { + // we have a format version number + format = firstInt; + + // check that it is a format we can understand + if (format > FORMAT_MINIMUM) + throw new IndexFormatTooOldException(null, format, FORMAT_MINIMUM, FORMAT_CURRENT); + if (format < FORMAT_CURRENT) + throw new IndexFormatTooNewException(null, format, FORMAT_MINIMUM, FORMAT_CURRENT); + + size = input.readLong(); // read the size + + indexInterval = input.readInt(); + skipInterval = input.readInt(); + maxSkipLevels = input.readInt(); + assert indexInterval > 0: "indexInterval=" + indexInterval + " is negative; must be > 0"; + assert skipInterval > 0: "skipInterval=" + skipInterval + " is negative; must be > 0"; + } + } + + @Override + protected Object clone() { + SegmentTermEnum clone = null; + try { + clone = (SegmentTermEnum) super.clone(); + } catch (CloneNotSupportedException e) {} + + clone.input = (IndexInput) input.clone(); + clone.termInfo = new TermInfo(termInfo); + + clone.termBuffer = (TermBuffer)termBuffer.clone(); + clone.prevBuffer = (TermBuffer)prevBuffer.clone(); + clone.scanBuffer = new TermBuffer(); + + return clone; + } + + final void seek(long pointer, long p, Term t, TermInfo ti) + throws IOException { + input.seek(pointer); + position = p; + termBuffer.set(t); + prevBuffer.reset(); + //System.out.println(" ste doSeek prev=" + prevBuffer.toTerm() + " this=" + this); + termInfo.set(ti); + first = p == -1; + } + + /** Increments the enumeration to the next element. True if one exists.*/ + public final boolean next() throws IOException { + prevBuffer.set(termBuffer); + //System.out.println(" ste setPrev=" + prev() + " this=" + this); + + if (position++ >= size - 1) { + termBuffer.reset(); + //System.out.println(" EOF"); + return false; + } + + termBuffer.read(input, fieldInfos); + newSuffixStart = termBuffer.newSuffixStart; + + termInfo.docFreq = input.readVInt(); // read doc freq + termInfo.freqPointer += input.readVLong(); // read freq pointer + termInfo.proxPointer += input.readVLong(); // read prox pointer + + if (termInfo.docFreq >= skipInterval) + termInfo.skipOffset = input.readVInt(); + + if (isIndex) + indexPointer += input.readVLong(); // read index pointer + + //System.out.println(" ste ret term=" + term()); + return true; + } + + /* Optimized scan, without allocating new terms. + * Return number of invocations to next(). + * + * NOTE: LUCENE-3183: if you pass Term("", "") here then this + * will incorrectly return before positioning the enum, + * and position will be -1; caller must detect this. */ + final int scanTo(Term term) throws IOException { + scanBuffer.set(term); + int count = 0; + if (first) { + // Always force initial next() in case term is + // Term("", "") + next(); + first = false; + count++; + } + while (scanBuffer.compareTo(termBuffer) > 0 && next()) { + count++; + } + return count; + } + + /** Returns the current Term in the enumeration. + Initially invalid, valid after next() called for the first time.*/ + public final Term term() { + return termBuffer.toTerm(); + } + + /** Returns the previous Term enumerated. Initially null.*/ + final Term prev() { + return prevBuffer.toTerm(); + } + + /** Returns the current TermInfo in the enumeration. + Initially invalid, valid after next() called for the first time.*/ + final TermInfo termInfo() { + return new TermInfo(termInfo); + } + + /** Sets the argument to the current TermInfo in the enumeration. + Initially invalid, valid after next() called for the first time.*/ + final void termInfo(TermInfo ti) { + ti.set(termInfo); + } + + /** Returns the docFreq from the current TermInfo in the enumeration. + Initially invalid, valid after next() called for the first time.*/ + public final int docFreq() { + return termInfo.docFreq; + } + + /* Returns the freqPointer from the current TermInfo in the enumeration. + Initially invalid, valid after next() called for the first time.*/ + final long freqPointer() { + return termInfo.freqPointer; + } + + /* Returns the proxPointer from the current TermInfo in the enumeration. + Initially invalid, valid after next() called for the first time.*/ + final long proxPointer() { + return termInfo.proxPointer; + } + + /** Closes the enumeration to further activity, freeing resources. */ + public final void close() throws IOException { + input.close(); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermPositions.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermPositions.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermPositions.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermPositions.java 2011-10-26 14:02:53.804890197 -0400 @@ -0,0 +1,219 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.store.IndexInput; + +/** + * @lucene.experimental + * @deprecated (4.0) + */ +@Deprecated +public final class SegmentTermPositions +extends SegmentTermDocs { + private IndexInput proxStream; + private IndexInput proxStreamOrig; + private int proxCount; + private int position; + + // the current payload length + private int payloadLength; + // indicates whether the payload of the current position has + // been read from the proxStream yet + private boolean needToLoadPayload; + + // these variables are being used to remember information + // for a lazy skip + private long lazySkipPointer = -1; + private int lazySkipProxCount = 0; + + /* + SegmentTermPositions(SegmentReader p) { + super(p); + this.proxStream = null; // the proxStream will be cloned lazily when nextPosition() is called for the first time + } + */ + + public SegmentTermPositions(IndexInput freqStream, IndexInput proxStream, TermInfosReader tis, FieldInfos fieldInfos) { + super(freqStream, tis, fieldInfos); + this.proxStreamOrig = proxStream; // the proxStream will be cloned lazily when nextPosition() is called for the first time + } + + @Override + final void seek(TermInfo ti, Term term) throws IOException { + super.seek(ti, term); + if (ti != null) + lazySkipPointer = ti.proxPointer; + + lazySkipProxCount = 0; + proxCount = 0; + payloadLength = 0; + needToLoadPayload = false; + } + + @Override + public final void close() throws IOException { + super.close(); + if (proxStream != null) proxStream.close(); + } + + public final int nextPosition() throws IOException { + if (indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) + // This field does not store positions, payloads + return 0; + // perform lazy skips if necessary + lazySkip(); + proxCount--; + return position += readDeltaPosition(); + } + + private final int readDeltaPosition() throws IOException { + int delta = proxStream.readVInt(); + if (currentFieldStoresPayloads) { + // if the current field stores payloads then + // the position delta is shifted one bit to the left. + // if the LSB is set, then we have to read the current + // payload length + if ((delta & 1) != 0) { + payloadLength = proxStream.readVInt(); + } + delta >>>= 1; + needToLoadPayload = true; + } + return delta; + } + + @Override + protected final void skippingDoc() throws IOException { + // we remember to skip a document lazily + lazySkipProxCount += freq; + } + + @Override + public final boolean next() throws IOException { + // we remember to skip the remaining positions of the current + // document lazily + lazySkipProxCount += proxCount; + + if (super.next()) { // run super + proxCount = freq; // note frequency + position = 0; // reset position + return true; + } + return false; + } + + @Override + public final int read(final int[] docs, final int[] freqs) { + throw new UnsupportedOperationException("TermPositions does not support processing multiple documents in one call. Use TermDocs instead."); + } + + + /** Called by super.skipTo(). */ + @Override + protected void skipProx(long proxPointer, int payloadLength) throws IOException { + // we save the pointer, we might have to skip there lazily + lazySkipPointer = proxPointer; + lazySkipProxCount = 0; + proxCount = 0; + this.payloadLength = payloadLength; + needToLoadPayload = false; + } + + private void skipPositions(int n) throws IOException { + assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + for (int f = n; f > 0; f--) { // skip unread positions + readDeltaPosition(); + skipPayload(); + } + } + + private void skipPayload() throws IOException { + if (needToLoadPayload && payloadLength > 0) { + proxStream.seek(proxStream.getFilePointer() + payloadLength); + } + needToLoadPayload = false; + } + + // It is not always necessary to move the prox pointer + // to a new document after the freq pointer has been moved. + // Consider for example a phrase query with two terms: + // the freq pointer for term 1 has to move to document x + // to answer the question if the term occurs in that document. But + // only if term 2 also matches document x, the positions have to be + // read to figure out if term 1 and term 2 appear next + // to each other in document x and thus satisfy the query. + // So we move the prox pointer lazily to the document + // as soon as positions are requested. + private void lazySkip() throws IOException { + if (proxStream == null) { + // clone lazily + proxStream = (IndexInput)proxStreamOrig.clone(); + } + + // we might have to skip the current payload + // if it was not read yet + skipPayload(); + + if (lazySkipPointer != -1) { + proxStream.seek(lazySkipPointer); + lazySkipPointer = -1; + } + + if (lazySkipProxCount != 0) { + skipPositions(lazySkipProxCount); + lazySkipProxCount = 0; + } + } + + public int getPayloadLength() { + return payloadLength; + } + + public byte[] getPayload(byte[] data, int offset) throws IOException { + if (!needToLoadPayload) { + throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once."); + } + + // read payloads lazily + byte[] retArray; + int retOffset; + if (data == null || data.length - offset < payloadLength) { + // the array is too small to store the payload data, + // so we allocate a new one + retArray = new byte[payloadLength]; + retOffset = 0; + } else { + retArray = data; + retOffset = offset; + } + proxStream.readBytes(retArray, retOffset, payloadLength); + needToLoadPayload = false; + return retArray; + } + + public boolean isPayloadAvailable() { + return needToLoadPayload && payloadLength > 0; + } + +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermBuffer.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermBuffer.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermBuffer.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermBuffer.java 2011-10-26 14:02:03.796890220 -0400 @@ -0,0 +1,122 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Comparator; + +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.FieldInfos; + +/** + * @lucene.experimental + * @deprecated (4.0) + */ +@Deprecated +final class TermBuffer implements Cloneable { + + private String field; + private Term term; // cached + + private BytesRef bytes = new BytesRef(10); + + // Cannot be -1 since (strangely) we write that + // fieldNumber into index for first indexed term: + private int currentFieldNumber = -2; + + private static final Comparator utf8AsUTF16Comparator = BytesRef.getUTF8SortedAsUTF16Comparator(); + + int newSuffixStart; // only valid right after .read is called + + public int compareTo(TermBuffer other) { + if (field == other.field) // fields are interned + // (only by PreFlex codec) + return utf8AsUTF16Comparator.compare(bytes, other.bytes); + else + return field.compareTo(other.field); + } + + public void read(IndexInput input, FieldInfos fieldInfos) + throws IOException { + this.term = null; // invalidate cache + newSuffixStart = input.readVInt(); + int length = input.readVInt(); + int totalLength = newSuffixStart + length; + if (bytes.bytes.length < totalLength) { + bytes.grow(totalLength); + } + bytes.length = totalLength; + input.readBytes(bytes.bytes, newSuffixStart, length); + final int fieldNumber = input.readVInt(); + if (fieldNumber != currentFieldNumber) { + currentFieldNumber = fieldNumber; + field = fieldInfos.fieldName(currentFieldNumber).intern(); + } else { + assert field.equals(fieldInfos.fieldName(fieldNumber)): "currentFieldNumber=" + currentFieldNumber + " field=" + field + " vs " + fieldInfos.fieldName(fieldNumber); + } + } + + public void set(Term term) { + if (term == null) { + reset(); + return; + } + bytes.copy(term.bytes()); + field = term.field().intern(); + currentFieldNumber = -1; + this.term = term; + } + + public void set(TermBuffer other) { + field = other.field; + currentFieldNumber = other.currentFieldNumber; + // dangerous to copy Term over, since the underlying + // BytesRef could subsequently be modified: + term = null; + bytes.copy(other.bytes); + } + + public void reset() { + field = null; + term = null; + currentFieldNumber= -1; + } + + public Term toTerm() { + if (field == null) // unset + return null; + + if (term == null) { + term = new Term(field, new BytesRef(bytes)); + } + + return term; + } + + @Override + protected Object clone() { + TermBuffer clone = null; + try { + clone = (TermBuffer)super.clone(); + } catch (CloneNotSupportedException e) {} + clone.bytes = new BytesRef(bytes); + return clone; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfo.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfo.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfo.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfo.java 2011-10-26 14:02:03.784890220 -0400 @@ -0,0 +1,63 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** A TermInfo is the record of information stored for a + * term + * @deprecated (4.0) This class is no longer used in flexible + * indexing. */ + +@Deprecated +public class TermInfo { + /** The number of documents which contain the term. */ + public int docFreq = 0; + + public long freqPointer = 0; + public long proxPointer = 0; + public int skipOffset; + + public TermInfo() {} + + public TermInfo(int df, long fp, long pp) { + docFreq = df; + freqPointer = fp; + proxPointer = pp; + } + + public TermInfo(TermInfo ti) { + docFreq = ti.docFreq; + freqPointer = ti.freqPointer; + proxPointer = ti.proxPointer; + skipOffset = ti.skipOffset; + } + + public final void set(int docFreq, + long freqPointer, long proxPointer, int skipOffset) { + this.docFreq = docFreq; + this.freqPointer = freqPointer; + this.proxPointer = proxPointer; + this.skipOffset = skipOffset; + } + + public final void set(TermInfo ti) { + docFreq = ti.docFreq; + freqPointer = ti.freqPointer; + proxPointer = ti.proxPointer; + skipOffset = ti.skipOffset; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java 2011-11-02 12:57:13.172580645 -0400 @@ -0,0 +1,252 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; + +import org.apache.lucene.index.Term; +import org.apache.lucene.util.BitUtil; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.PagedBytes.PagedBytesDataInput; +import org.apache.lucene.util.PagedBytes.PagedBytesDataOutput; +import org.apache.lucene.util.PagedBytes; +import org.apache.lucene.util.packed.GrowableWriter; +import org.apache.lucene.util.packed.PackedInts; + +/** + * This stores a monotonically increasing set of pairs in an + * index segment. Pairs are accessed either by Term or by ordinal position the + * set. The Terms and TermInfo are actually serialized and stored into a byte + * array and pointers to the position of each are stored in a int array. + */ +class TermInfosReaderIndex { + + private static final int MAX_PAGE_BITS = 18; // 256 KB block + private Term[] fields; + private int totalIndexInterval; + private Comparator comparator = BytesRef.getUTF8SortedAsUTF16Comparator(); + private final PagedBytesDataInput dataInput; + private final PackedInts.Reader indexToDataOffset; + private final int indexSize; + private final int skipInterval; + + /** + * Loads the segment information at segment load time. + * + * @param indexEnum + * the term enum. + * @param indexDivisor + * the index divisor. + * @param tiiFileLength + * the size of the tii file, used to approximate the size of the + * buffer. + * @param totalIndexInterval + * the total index interval. + */ + TermInfosReaderIndex(SegmentTermEnum indexEnum, int indexDivisor, long tiiFileLength, int totalIndexInterval) throws IOException { + this.totalIndexInterval = totalIndexInterval; + indexSize = 1 + ((int) indexEnum.size - 1) / indexDivisor; + skipInterval = indexEnum.skipInterval; + // this is only an inital size, it will be GCed once the build is complete + long initialSize = (long) (tiiFileLength * 1.5) / indexDivisor; + PagedBytes dataPagedBytes = new PagedBytes(estimatePageBits(initialSize)); + PagedBytesDataOutput dataOutput = dataPagedBytes.getDataOutput(); + + GrowableWriter indexToTerms = new GrowableWriter(4, indexSize, false); + String currentField = null; + List fieldStrs = new ArrayList(); + int fieldCounter = -1; + for (int i = 0; indexEnum.next(); i++) { + Term term = indexEnum.term(); + if (currentField == null || !currentField.equals(term.field())) { + currentField = term.field(); + fieldStrs.add(currentField); + fieldCounter++; + } + TermInfo termInfo = indexEnum.termInfo(); + indexToTerms.set(i, dataOutput.getPosition()); + dataOutput.writeVInt(fieldCounter); + dataOutput.writeString(term.text()); + dataOutput.writeVInt(termInfo.docFreq); + if (termInfo.docFreq >= skipInterval) { + dataOutput.writeVInt(termInfo.skipOffset); + } + dataOutput.writeVLong(termInfo.freqPointer); + dataOutput.writeVLong(termInfo.proxPointer); + dataOutput.writeVLong(indexEnum.indexPointer); + for (int j = 1; j < indexDivisor; j++) { + if (!indexEnum.next()) { + break; + } + } + } + + fields = new Term[fieldStrs.size()]; + for (int i = 0; i < fields.length; i++) { + fields[i] = new Term(fieldStrs.get(i)); + } + + dataPagedBytes.freeze(true); + dataInput = dataPagedBytes.getDataInput(); + indexToDataOffset = indexToTerms.getMutable(); + } + + private static int estimatePageBits(long estSize) { + return Math.max(Math.min(64 - BitUtil.nlz(estSize), MAX_PAGE_BITS), 4); + } + + void seekEnum(SegmentTermEnum enumerator, int indexOffset) throws IOException { + PagedBytesDataInput input = (PagedBytesDataInput) dataInput.clone(); + + input.setPosition(indexToDataOffset.get(indexOffset)); + + // read the term + int fieldId = input.readVInt(); + Term field = fields[fieldId]; + Term term = new Term(field.field(), input.readString()); + + // read the terminfo + TermInfo termInfo = new TermInfo(); + termInfo.docFreq = input.readVInt(); + if (termInfo.docFreq >= skipInterval) { + termInfo.skipOffset = input.readVInt(); + } else { + termInfo.skipOffset = 0; + } + termInfo.freqPointer = input.readVLong(); + termInfo.proxPointer = input.readVLong(); + + long pointer = input.readVLong(); + + // perform the seek + enumerator.seek(pointer, ((long) indexOffset * totalIndexInterval) - 1, term, termInfo); + } + + /** + * Binary search for the given term. + * + * @param term + * the term to locate. + * @throws IOException + */ + int getIndexOffset(Term term) throws IOException { + int lo = 0; + int hi = indexSize - 1; + PagedBytesDataInput input = (PagedBytesDataInput) dataInput.clone(); + BytesRef scratch = new BytesRef(); + while (hi >= lo) { + int mid = (lo + hi) >>> 1; + int delta = compareTo(term, mid, input, scratch); + if (delta < 0) + hi = mid - 1; + else if (delta > 0) + lo = mid + 1; + else + return mid; + } + return hi; + } + + /** + * Gets the term at the given position. For testing. + * + * @param termIndex + * the position to read the term from the index. + * @return the term. + * @throws IOException + */ + Term getTerm(int termIndex) throws IOException { + PagedBytesDataInput input = (PagedBytesDataInput) dataInput.clone(); + input.setPosition(indexToDataOffset.get(termIndex)); + + // read the term + int fieldId = input.readVInt(); + Term field = fields[fieldId]; + return new Term(field.field(), input.readString()); + } + + /** + * Returns the number of terms. + * + * @return int. + */ + int length() { + return indexSize; + } + + /** + * The compares the given term against the term in the index specified by the + * term index. ie It returns negative N when term is less than index term; + * + * @param term + * the given term. + * @param termIndex + * the index of the of term to compare. + * @return int. + * @throws IOException + */ + int compareTo(Term term, int termIndex) throws IOException { + return compareTo(term, termIndex, (PagedBytesDataInput) dataInput.clone(), new BytesRef()); + } + + /** + * Compare the fields of the terms first, and if not equals return from + * compare. If equal compare terms. + * + * @param term + * the term to compare. + * @param termIndex + * the position of the term in the input to compare + * @param input + * the input buffer. + * @return int. + * @throws IOException + */ + private int compareTo(Term term, int termIndex, PagedBytesDataInput input, BytesRef reuse) throws IOException { + // if term field does not equal mid's field index, then compare fields + // else if they are equal, compare term's string values... + int c = compareField(term, termIndex, input); + if (c == 0) { + reuse.length = input.readVInt(); + reuse.grow(reuse.length); + input.readBytes(reuse.bytes, 0, reuse.length); + return comparator.compare(term.bytes(), reuse); + } + return c; + } + + /** + * Compares the fields before checking the text of the terms. + * + * @param term + * the given term. + * @param termIndex + * the term that exists in the data block. + * @param input + * the data block. + * @return int. + * @throws IOException + */ + private int compareField(Term term, int termIndex, PagedBytesDataInput input) throws IOException { + input.setPosition(indexToDataOffset.get(termIndex)); + return term.field().compareTo(fields[input.readVInt()].field()); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReader.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReader.java 2011-11-02 12:58:52.412580596 -0400 @@ -0,0 +1,347 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Comparator; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.Term; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CloseableThreadLocal; +import org.apache.lucene.util.DoubleBarrelLRUCache; + +/** This stores a monotonically increasing set of pairs in a + * Directory. Pairs are accessed either by Term or by ordinal position the + * set + * @deprecated (4.0) This class has been replaced by + * FormatPostingsTermsDictReader, except for reading old segments. + * @lucene.experimental + */ +@Deprecated +public final class TermInfosReader { + private final Directory directory; + private final String segment; + private final FieldInfos fieldInfos; + + private final CloseableThreadLocal threadResources = new CloseableThreadLocal(); + private final SegmentTermEnum origEnum; + private final long size; + + private final TermInfosReaderIndex index; + private final int indexLength; + + private final int totalIndexInterval; + + private final static int DEFAULT_CACHE_SIZE = 1024; + + // Just adds term's ord to TermInfo + private final static class TermInfoAndOrd extends TermInfo { + final long termOrd; + public TermInfoAndOrd(TermInfo ti, long termOrd) { + super(ti); + assert termOrd >= 0; + this.termOrd = termOrd; + } + } + + private static class CloneableTerm extends DoubleBarrelLRUCache.CloneableKey { + Term term; + public CloneableTerm(Term t) { + this.term = t; + } + + @Override + public boolean equals(Object other) { + CloneableTerm t = (CloneableTerm) other; + return this.term.equals(t.term); + } + + @Override + public int hashCode() { + return term.hashCode(); + } + + @Override + public Object clone() { + return new CloneableTerm(term); + } + } + + private final DoubleBarrelLRUCache termsCache = new DoubleBarrelLRUCache(DEFAULT_CACHE_SIZE); + + /** + * Per-thread resources managed by ThreadLocal + */ + private static final class ThreadResources { + SegmentTermEnum termEnum; + } + + TermInfosReader(Directory dir, String seg, FieldInfos fis, IOContext context, int indexDivisor) + throws CorruptIndexException, IOException { + boolean success = false; + + if (indexDivisor < 1 && indexDivisor != -1) { + throw new IllegalArgumentException("indexDivisor must be -1 (don't load terms index) or greater than 0: got " + indexDivisor); + } + + try { + directory = dir; + segment = seg; + fieldInfos = fis; + + origEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), + context), fieldInfos, false); + size = origEnum.size; + + + if (indexDivisor != -1) { + // Load terms index + totalIndexInterval = origEnum.indexInterval * indexDivisor; + + final String indexFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); + final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(indexFileName, + context), fieldInfos, true); + + try { + index = new TermInfosReaderIndex(indexEnum, indexDivisor, dir.fileLength(indexFileName), totalIndexInterval); + indexLength = index.length(); + } finally { + indexEnum.close(); + } + } else { + // Do not load terms index: + totalIndexInterval = -1; + index = null; + indexLength = -1; + } + success = true; + } finally { + // With lock-less commits, it's entirely possible (and + // fine) to hit a FileNotFound exception above. In + // this case, we want to explicitly close any subset + // of things that were opened so that we don't have to + // wait for a GC to do so. + if (!success) { + close(); + } + } + } + + public int getSkipInterval() { + return origEnum.skipInterval; + } + + public int getMaxSkipLevels() { + return origEnum.maxSkipLevels; + } + + void close() throws IOException { + if (origEnum != null) + origEnum.close(); + threadResources.close(); + } + + /** Returns the number of term/value pairs in the set. */ + long size() { + return size; + } + + private ThreadResources getThreadResources() { + ThreadResources resources = threadResources.get(); + if (resources == null) { + resources = new ThreadResources(); + resources.termEnum = terms(); + threadResources.set(resources); + } + return resources; + } + + private static final Comparator legacyComparator = + BytesRef.getUTF8SortedAsUTF16Comparator(); + + private final int compareAsUTF16(Term term1, Term term2) { + if (term1.field().equals(term2.field())) { + return legacyComparator.compare(term1.bytes(), term2.bytes()); + } else { + return term1.field().compareTo(term2.field()); + } + } + + /** Returns the TermInfo for a Term in the set, or null. */ + TermInfo get(Term term) throws IOException { + return get(term, false); + } + + /** Returns the TermInfo for a Term in the set, or null. */ + private TermInfo get(Term term, boolean mustSeekEnum) throws IOException { + if (size == 0) return null; + + ensureIndexIsRead(); + TermInfoAndOrd tiOrd = termsCache.get(new CloneableTerm(term)); + ThreadResources resources = getThreadResources(); + + if (!mustSeekEnum && tiOrd != null) { + return tiOrd; + } + + return seekEnum(resources.termEnum, term, tiOrd, true); + } + + public void cacheCurrentTerm(SegmentTermEnum enumerator) { + termsCache.put(new CloneableTerm(enumerator.term()), + new TermInfoAndOrd(enumerator.termInfo, + enumerator.position)); + } + + TermInfo seekEnum(SegmentTermEnum enumerator, Term term, boolean useCache) throws IOException { + if (useCache) { + return seekEnum(enumerator, term, termsCache.get(new CloneableTerm(term)), useCache); + } else { + return seekEnum(enumerator, term, null, useCache); + } + } + + TermInfo seekEnum(SegmentTermEnum enumerator, Term term, TermInfoAndOrd tiOrd, boolean useCache) throws IOException { + if (size == 0) { + return null; + } + + // optimize sequential access: first try scanning cached enum w/o seeking + if (enumerator.term() != null // term is at or past current + && ((enumerator.prev() != null && compareAsUTF16(term, enumerator.prev())> 0) + || compareAsUTF16(term, enumerator.term()) >= 0)) { + int enumOffset = (int)(enumerator.position/totalIndexInterval)+1; + if (indexLength == enumOffset // but before end of block + || index.compareTo(term, enumOffset) < 0) { + // no need to seek + + final TermInfo ti; + int numScans = enumerator.scanTo(term); + if (enumerator.term() != null && compareAsUTF16(term, enumerator.term()) == 0) { + ti = enumerator.termInfo; + if (numScans > 1) { + // we only want to put this TermInfo into the cache if + // scanEnum skipped more than one dictionary entry. + // This prevents RangeQueries or WildcardQueries to + // wipe out the cache when they iterate over a large numbers + // of terms in order + if (tiOrd == null) { + if (useCache) { + termsCache.put(new CloneableTerm(term), new TermInfoAndOrd(ti, enumerator.position)); + } + } else { + assert sameTermInfo(ti, tiOrd, enumerator); + assert (int) enumerator.position == tiOrd.termOrd; + } + } + } else { + ti = null; + } + + return ti; + } + } + + // random-access: must seek + final int indexPos; + if (tiOrd != null) { + indexPos = (int) (tiOrd.termOrd / totalIndexInterval); + } else { + // Must do binary search: + indexPos = index.getIndexOffset(term); + } + + index.seekEnum(enumerator, indexPos); + enumerator.scanTo(term); + final TermInfo ti; + + if (enumerator.term() != null && compareAsUTF16(term, enumerator.term()) == 0) { + ti = enumerator.termInfo; + if (tiOrd == null) { + if (useCache) { + termsCache.put(new CloneableTerm(term), new TermInfoAndOrd(ti, enumerator.position)); + } + } else { + assert sameTermInfo(ti, tiOrd, enumerator); + assert enumerator.position == tiOrd.termOrd; + } + } else { + ti = null; + } + return ti; + } + + // called only from asserts + private boolean sameTermInfo(TermInfo ti1, TermInfo ti2, SegmentTermEnum enumerator) { + if (ti1.docFreq != ti2.docFreq) { + return false; + } + if (ti1.freqPointer != ti2.freqPointer) { + return false; + } + if (ti1.proxPointer != ti2.proxPointer) { + return false; + } + // skipOffset is only valid when docFreq >= skipInterval: + if (ti1.docFreq >= enumerator.skipInterval && + ti1.skipOffset != ti2.skipOffset) { + return false; + } + return true; + } + + private void ensureIndexIsRead() { + if (index == null) { + throw new IllegalStateException("terms index was not loaded when this reader was created"); + } + } + + /** Returns the position of a Term in the set or -1. */ + long getPosition(Term term) throws IOException { + if (size == 0) return -1; + + ensureIndexIsRead(); + int indexOffset = index.getIndexOffset(term); + + SegmentTermEnum enumerator = getThreadResources().termEnum; + index.seekEnum(enumerator, indexOffset); + + while(compareAsUTF16(term, enumerator.term()) > 0 && enumerator.next()) {} + + if (compareAsUTF16(term, enumerator.term()) == 0) + return enumerator.position; + else + return -1; + } + + /** Returns an enumeration of all the Terms and TermInfos in the set. */ + public SegmentTermEnum terms() { + return (SegmentTermEnum)origEnum.clone(); + } + + /** Returns an enumeration of terms starting at or after the named term. */ + public SegmentTermEnum terms(Term term) throws IOException { + get(term, true); + return (SegmentTermEnum)getThreadResources().termEnum.clone(); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListReader.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListReader.java 2011-10-26 13:42:52.736890813 -0400 @@ -0,0 +1,118 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.index.codecs.MultiLevelSkipListReader; +import org.apache.lucene.store.IndexInput; + +/** + * Implements the skip list reader for the default posting list format + * that stores positions and payloads. + * @lucene.experimental + */ +public class DefaultSkipListReader extends MultiLevelSkipListReader { + private boolean currentFieldStoresPayloads; + private long freqPointer[]; + private long proxPointer[]; + private int payloadLength[]; + + private long lastFreqPointer; + private long lastProxPointer; + private int lastPayloadLength; + + + public DefaultSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) { + super(skipStream, maxSkipLevels, skipInterval); + freqPointer = new long[maxSkipLevels]; + proxPointer = new long[maxSkipLevels]; + payloadLength = new int[maxSkipLevels]; + } + + public void init(long skipPointer, long freqBasePointer, long proxBasePointer, int df, boolean storesPayloads) { + super.init(skipPointer, df); + this.currentFieldStoresPayloads = storesPayloads; + lastFreqPointer = freqBasePointer; + lastProxPointer = proxBasePointer; + + Arrays.fill(freqPointer, freqBasePointer); + Arrays.fill(proxPointer, proxBasePointer); + Arrays.fill(payloadLength, 0); + } + + /** Returns the freq pointer of the doc to which the last call of + * {@link MultiLevelSkipListReader#skipTo(int)} has skipped. */ + public long getFreqPointer() { + return lastFreqPointer; + } + + /** Returns the prox pointer of the doc to which the last call of + * {@link MultiLevelSkipListReader#skipTo(int)} has skipped. */ + public long getProxPointer() { + return lastProxPointer; + } + + /** Returns the payload length of the payload stored just before + * the doc to which the last call of {@link MultiLevelSkipListReader#skipTo(int)} + * has skipped. */ + public int getPayloadLength() { + return lastPayloadLength; + } + + @Override + protected void seekChild(int level) throws IOException { + super.seekChild(level); + freqPointer[level] = lastFreqPointer; + proxPointer[level] = lastProxPointer; + payloadLength[level] = lastPayloadLength; + } + + @Override + protected void setLastSkipData(int level) { + super.setLastSkipData(level); + lastFreqPointer = freqPointer[level]; + lastProxPointer = proxPointer[level]; + lastPayloadLength = payloadLength[level]; + } + + + @Override + protected int readSkipData(int level, IndexInput skipStream) throws IOException { + int delta; + if (currentFieldStoresPayloads) { + // the current field stores payloads. + // if the doc delta is odd then we have + // to read the current payload length + // because it differs from the length of the + // previous payload + delta = skipStream.readVInt(); + if ((delta & 1) != 0) { + payloadLength[level] = skipStream.readVInt(); + } + delta >>>= 1; + } else { + delta = skipStream.readVInt(); + } + freqPointer[level] += skipStream.readVInt(); + proxPointer[level] += skipStream.readVInt(); + + return delta; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListWriter.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/DefaultSkipListWriter.java 2011-10-26 13:42:52.732890813 -0400 @@ -0,0 +1,128 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.index.codecs.MultiLevelSkipListWriter; + + +/** + * Implements the skip list writer for the default posting list format + * that stores positions and payloads. + * @lucene.experimental + */ +public class DefaultSkipListWriter extends MultiLevelSkipListWriter { + private int[] lastSkipDoc; + private int[] lastSkipPayloadLength; + private long[] lastSkipFreqPointer; + private long[] lastSkipProxPointer; + + private IndexOutput freqOutput; + private IndexOutput proxOutput; + + private int curDoc; + private boolean curStorePayloads; + private int curPayloadLength; + private long curFreqPointer; + private long curProxPointer; + + public DefaultSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, IndexOutput freqOutput, IndexOutput proxOutput) { + super(skipInterval, numberOfSkipLevels, docCount); + this.freqOutput = freqOutput; + this.proxOutput = proxOutput; + + lastSkipDoc = new int[numberOfSkipLevels]; + lastSkipPayloadLength = new int[numberOfSkipLevels]; + lastSkipFreqPointer = new long[numberOfSkipLevels]; + lastSkipProxPointer = new long[numberOfSkipLevels]; + } + + /** + * Sets the values for the current skip data. + */ + public void setSkipData(int doc, boolean storePayloads, int payloadLength) { + this.curDoc = doc; + this.curStorePayloads = storePayloads; + this.curPayloadLength = payloadLength; + this.curFreqPointer = freqOutput.getFilePointer(); + if (proxOutput != null) + this.curProxPointer = proxOutput.getFilePointer(); + } + + @Override + public void resetSkip() { + super.resetSkip(); + Arrays.fill(lastSkipDoc, 0); + Arrays.fill(lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list + Arrays.fill(lastSkipFreqPointer, freqOutput.getFilePointer()); + if (proxOutput != null) + Arrays.fill(lastSkipProxPointer, proxOutput.getFilePointer()); + } + + @Override + protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { + // To efficiently store payloads in the posting lists we do not store the length of + // every payload. Instead we omit the length for a payload if the previous payload had + // the same length. + // However, in order to support skipping the payload length at every skip point must be known. + // So we use the same length encoding that we use for the posting lists for the skip data as well: + // Case 1: current field does not store payloads + // SkipDatum --> DocSkip, FreqSkip, ProxSkip + // DocSkip,FreqSkip,ProxSkip --> VInt + // DocSkip records the document number before every SkipInterval th document in TermFreqs. + // Document numbers are represented as differences from the previous value in the sequence. + // Case 2: current field stores payloads + // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip + // DocSkip,FreqSkip,ProxSkip --> VInt + // PayloadLength --> VInt + // In this case DocSkip/2 is the difference between + // the current and the previous value. If DocSkip + // is odd, then a PayloadLength encoded as VInt follows, + // if DocSkip is even, then it is assumed that the + // current payload length equals the length at the previous + // skip point + if (curStorePayloads) { + int delta = curDoc - lastSkipDoc[level]; + if (curPayloadLength == lastSkipPayloadLength[level]) { + // the current payload length equals the length at the previous skip point, + // so we don't store the length again + skipBuffer.writeVInt(delta * 2); + } else { + // the payload length is different from the previous one. We shift the DocSkip, + // set the lowest bit and store the current payload length as VInt. + skipBuffer.writeVInt(delta * 2 + 1); + skipBuffer.writeVInt(curPayloadLength); + lastSkipPayloadLength[level] = curPayloadLength; + } + } else { + // current field does not store payloads + skipBuffer.writeVInt(curDoc - lastSkipDoc[level]); + } + skipBuffer.writeVInt((int) (curFreqPointer - lastSkipFreqPointer[level])); + skipBuffer.writeVInt((int) (curProxPointer - lastSkipProxPointer[level])); + + lastSkipDoc[level] = curDoc; + + lastSkipFreqPointer[level] = curFreqPointer; + lastSkipProxPointer[level] = curProxPointer; + } + +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40Codec.java 2011-11-03 12:39:20.640536686 -0400 @@ -0,0 +1,85 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesFormat; +import org.apache.lucene.index.codecs.DefaultFieldsFormat; +import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; +import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldsFormat; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.SegmentInfosFormat; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; + +/** + * Implements the Lucene 4.0 index format, with configurable per-field postings formats + * and using {@link DefaultFieldsFormat} for stored fields and {@link + * DefaultDocValuesFormat} for doc values. + * + * @lucene.experimental + */ +// NOTE: if we make largish changes in a minor release, easier to just make Lucene42Codec or whatever +// if they are backwards compatible or smallish we can probably do the backwards in the postingsreader +// (it writes a minor version, etc). +public class Lucene40Codec extends Codec { + private final FieldsFormat fieldsFormat = new DefaultFieldsFormat(); + private final DocValuesFormat docValuesFormat = new DefaultDocValuesFormat(); + private final SegmentInfosFormat infosFormat = new DefaultSegmentInfosFormat(); + private final PostingsFormat postingsFormat = new PerFieldPostingsFormat() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return Lucene40Codec.this.getPostingsFormatForField(field); + } + }; + + public Lucene40Codec() { + super("Lucene40"); + } + + @Override + public FieldsFormat fieldsFormat() { + return fieldsFormat; + } + + @Override + public DocValuesFormat docValuesFormat() { + return docValuesFormat; + } + + @Override + public PostingsFormat postingsFormat() { + return postingsFormat; + } + + @Override + public SegmentInfosFormat segmentInfosFormat() { + return infosFormat; + } + + /** Returns the postings format that should be used for writing + * new segments of field. + * + * The default implementation always returns "Lucene40" + */ + public PostingsFormat getPostingsFormatForField(String field) { + return defaultFormat; + } + + private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene40"); +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsBaseFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsBaseFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsBaseFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsBaseFormat.java 2011-11-03 13:24:53.424535280 -0400 @@ -0,0 +1,58 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.PostingsBaseFormat; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.store.Directory; + +/** + * Provides a {@link PostingsReaderBase} and {@link + * PostingsWriterBase}. + * + * @lucene.experimental */ + +// TODO: should these also be named / looked up via SPI? +public final class Lucene40PostingsBaseFormat extends PostingsBaseFormat { + + public Lucene40PostingsBaseFormat() { + super("Lucene40"); + } + + @Override + public PostingsReaderBase postingsReaderBase(SegmentReadState state) throws IOException { + return new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + } + + @Override + public PostingsWriterBase postingsWriterBase(SegmentWriteState state) throws IOException { + return new Lucene40PostingsWriter(state); + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsFormat.java 2011-11-03 13:24:53.424535280 -0400 @@ -0,0 +1,118 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.BlockTreeTermsReader; +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.store.Directory; + +/** Default codec. + * @lucene.experimental */ + +// TODO: this class could be created by wrapping +// BlockTreeTermsDict around Lucene40PostingsBaseFormat; ie +// we should not duplicate the code from that class here: +public class Lucene40PostingsFormat extends PostingsFormat { + + private final int minBlockSize; + private final int maxBlockSize; + + public Lucene40PostingsFormat() { + this(BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); + } + + public Lucene40PostingsFormat(int minBlockSize, int maxBlockSize) { + super("Lucene40"); + this.minBlockSize = minBlockSize; + assert minBlockSize > 1; + this.maxBlockSize = maxBlockSize; + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docs = new Lucene40PostingsWriter(state); + + // TODO: should we make the terms index more easily + // pluggable? Ie so that this codec would record which + // index impl was used, and switch on loading? + // Or... you must make a new Codec for this? + boolean success = false; + try { + FieldsConsumer ret = new BlockTreeTermsWriter(state, docs, minBlockSize, maxBlockSize); + success = true; + return ret; + } finally { + if (!success) { + docs.close(); + } + } + } + + public final static int TERMS_CACHE_SIZE = 1024; + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase postings = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + + boolean success = false; + try { + FieldsProducer ret = new BlockTreeTermsReader( + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postings, + state.context, + state.segmentSuffix, + state.termsIndexDivisor); + success = true; + return ret; + } finally { + if (!success) { + postings.close(); + } + } + } + + /** Extension of freq postings file */ + static final String FREQ_EXTENSION = "frq"; + + /** Extension of prox postings file */ + static final String PROX_EXTENSION = "prx"; + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); + } + + @Override + public String toString() { + return getName() + "(minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java 2011-11-02 21:26:22.476564906 -0400 @@ -0,0 +1,874 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Collection; + +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.TermState; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.BlockTermState; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CodecUtil; + +/** Concrete class that reads the current doc/freq/skip + * postings format. + * @lucene.experimental */ + +public class Lucene40PostingsReader extends PostingsReaderBase { + + private final IndexInput freqIn; + private final IndexInput proxIn; + // public static boolean DEBUG = BlockTreeTermsWriter.DEBUG; + + int skipInterval; + int maxSkipLevels; + int skipMinimum; + + // private String segment; + + public Lucene40PostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext ioContext, String segmentSuffix) throws IOException { + freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION), + ioContext); + // this.segment = segmentInfo.name; + if (segmentInfo.getHasProx()) { + boolean success = false; + try { + proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION), + ioContext); + success = true; + } finally { + if (!success) { + freqIn.close(); + } + } + } else { + proxIn = null; + } + } + + public static void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Collection files) throws IOException { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION)); + if (segmentInfo.getHasProx()) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION)); + } + } + + @Override + public void init(IndexInput termsIn) throws IOException { + + // Make sure we are talking to the matching past writer + CodecUtil.checkHeader(termsIn, Lucene40PostingsWriter.CODEC, + Lucene40PostingsWriter.VERSION_START, Lucene40PostingsWriter.VERSION_START); + + skipInterval = termsIn.readInt(); + maxSkipLevels = termsIn.readInt(); + skipMinimum = termsIn.readInt(); + } + + // Must keep final because we do non-standard clone + private final static class StandardTermState extends BlockTermState { + long freqOffset; + long proxOffset; + int skipOffset; + + // Only used by the "primary" TermState -- clones don't + // copy this (basically they are "transient"): + ByteArrayDataInput bytesReader; // TODO: should this NOT be in the TermState...? + byte[] bytes; + + @Override + public Object clone() { + StandardTermState other = new StandardTermState(); + other.copyFrom(this); + return other; + } + + @Override + public void copyFrom(TermState _other) { + super.copyFrom(_other); + StandardTermState other = (StandardTermState) _other; + freqOffset = other.freqOffset; + proxOffset = other.proxOffset; + skipOffset = other.skipOffset; + + // Do not copy bytes, bytesReader (else TermState is + // very heavy, ie drags around the entire block's + // byte[]). On seek back, if next() is in fact used + // (rare!), they will be re-read from disk. + } + + @Override + public String toString() { + return super.toString() + " freqFP=" + freqOffset + " proxFP=" + proxOffset + " skipOffset=" + skipOffset; + } + } + + @Override + public BlockTermState newTermState() { + return new StandardTermState(); + } + + @Override + public void close() throws IOException { + try { + if (freqIn != null) { + freqIn.close(); + } + } finally { + if (proxIn != null) { + proxIn.close(); + } + } + } + + /* Reads but does not decode the byte[] blob holding + metadata for the current terms block */ + @Override + public void readTermsBlock(IndexInput termsIn, FieldInfo fieldInfo, BlockTermState _termState) throws IOException { + final StandardTermState termState = (StandardTermState) _termState; + + final int len = termsIn.readVInt(); + + // if (DEBUG) System.out.println(" SPR.readTermsBlock bytes=" + len + " ts=" + _termState); + if (termState.bytes == null) { + termState.bytes = new byte[ArrayUtil.oversize(len, 1)]; + termState.bytesReader = new ByteArrayDataInput(); + } else if (termState.bytes.length < len) { + termState.bytes = new byte[ArrayUtil.oversize(len, 1)]; + } + + termsIn.readBytes(termState.bytes, 0, len); + termState.bytesReader.reset(termState.bytes, 0, len); + } + + @Override + public void nextTerm(FieldInfo fieldInfo, BlockTermState _termState) + throws IOException { + final StandardTermState termState = (StandardTermState) _termState; + // if (DEBUG) System.out.println("SPR: nextTerm seg=" + segment + " tbOrd=" + termState.termBlockOrd + " bytesReader.fp=" + termState.bytesReader.getPosition()); + final boolean isFirstTerm = termState.termBlockOrd == 0; + + if (isFirstTerm) { + termState.freqOffset = termState.bytesReader.readVLong(); + } else { + termState.freqOffset += termState.bytesReader.readVLong(); + } + /* + if (DEBUG) { + System.out.println(" dF=" + termState.docFreq); + System.out.println(" freqFP=" + termState.freqOffset); + } + */ + assert termState.freqOffset < freqIn.length(); + + if (termState.docFreq >= skipMinimum) { + termState.skipOffset = termState.bytesReader.readVInt(); + // if (DEBUG) System.out.println(" skipOffset=" + termState.skipOffset + " vs freqIn.length=" + freqIn.length()); + assert termState.freqOffset + termState.skipOffset < freqIn.length(); + } else { + // undefined + } + + if (fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { + if (isFirstTerm) { + termState.proxOffset = termState.bytesReader.readVLong(); + } else { + termState.proxOffset += termState.bytesReader.readVLong(); + } + // if (DEBUG) System.out.println(" proxFP=" + termState.proxOffset); + } + } + + @Override + public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse) throws IOException { + SegmentDocsEnum docsEnum; + if (reuse == null || !(reuse instanceof SegmentDocsEnum)) { + docsEnum = new SegmentDocsEnum(freqIn); + } else { + docsEnum = (SegmentDocsEnum) reuse; + if (docsEnum.startFreqIn != freqIn) { + // If you are using ParellelReader, and pass in a + // reused DocsEnum, it could have come from another + // reader also using standard codec + docsEnum = new SegmentDocsEnum(freqIn); + } + } + // if (DEBUG) System.out.println("SPR.docs ts=" + termState); + return docsEnum.reset(fieldInfo, (StandardTermState) termState, liveDocs); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { + if (fieldInfo.indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { + return null; + } + + // TODO: refactor + if (fieldInfo.storePayloads) { + SegmentDocsAndPositionsAndPayloadsEnum docsEnum; + if (reuse == null || !(reuse instanceof SegmentDocsAndPositionsAndPayloadsEnum)) { + docsEnum = new SegmentDocsAndPositionsAndPayloadsEnum(freqIn, proxIn); + } else { + docsEnum = (SegmentDocsAndPositionsAndPayloadsEnum) reuse; + if (docsEnum.startFreqIn != freqIn) { + // If you are using ParellelReader, and pass in a + // reused DocsEnum, it could have come from another + // reader also using standard codec + docsEnum = new SegmentDocsAndPositionsAndPayloadsEnum(freqIn, proxIn); + } + } + return docsEnum.reset(fieldInfo, (StandardTermState) termState, liveDocs); + } else { + SegmentDocsAndPositionsEnum docsEnum; + if (reuse == null || !(reuse instanceof SegmentDocsAndPositionsEnum)) { + docsEnum = new SegmentDocsAndPositionsEnum(freqIn, proxIn); + } else { + docsEnum = (SegmentDocsAndPositionsEnum) reuse; + if (docsEnum.startFreqIn != freqIn) { + // If you are using ParellelReader, and pass in a + // reused DocsEnum, it could have come from another + // reader also using standard codec + docsEnum = new SegmentDocsAndPositionsEnum(freqIn, proxIn); + } + } + return docsEnum.reset(fieldInfo, (StandardTermState) termState, liveDocs); + } + } + + // Decodes only docs + private class SegmentDocsEnum extends DocsEnum { + final IndexInput freqIn; + final IndexInput startFreqIn; + + boolean omitTF; // does current field omit term freq? + boolean storePayloads; // does current field store payloads? + + int limit; // number of docs in this posting + int ord; // how many docs we've read + int doc; // doc we last read + int freq; // freq we last read + + Bits liveDocs; + + long freqOffset; + int skipOffset; + + boolean skipped; + DefaultSkipListReader skipper; + + public SegmentDocsEnum(IndexInput freqIn) throws IOException { + startFreqIn = freqIn; + this.freqIn = (IndexInput) freqIn.clone(); + } + + public SegmentDocsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException { + omitTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY; + if (omitTF) { + freq = 1; + } + storePayloads = fieldInfo.storePayloads; + this.liveDocs = liveDocs; + freqOffset = termState.freqOffset; + skipOffset = termState.skipOffset; + + // TODO: for full enum case (eg segment merging) this + // seek is unnecessary; maybe we can avoid in such + // cases + freqIn.seek(termState.freqOffset); + limit = termState.docFreq; + assert limit > 0; + ord = 0; + doc = 0; + // if (DEBUG) System.out.println(" sde limit=" + limit + " freqFP=" + freqOffset); + + skipped = false; + + return this; + } + + @Override + public int nextDoc() throws IOException { + //if (DEBUG) System.out.println(" stpr.nextDoc seg=" + segment + " fp=" + freqIn.getFilePointer()); + while(true) { + if (ord == limit) { + //if (DEBUG) System.out.println(" return doc=" + NO_MORE_DOCS); + return doc = NO_MORE_DOCS; + } + + ord++; + + // Decode next doc/freq pair + final int code = freqIn.readVInt(); + // if (DEBUG) System.out.println(" code=" + code); + if (omitTF) { + doc += code; + } else { + doc += code >>> 1; // shift off low bit + if ((code & 1) != 0) { // if low bit is set + freq = 1; // freq is one + } else { + freq = freqIn.readVInt(); // else read freq + } + } + + if (liveDocs == null || liveDocs.get(doc)) { + break; + } + } + + //if (DEBUG) System.out.println(" stpr.nextDoc return doc=" + doc); + return doc; + } + + @Override + public int read() throws IOException { + + final int[] docs = bulkResult.docs.ints; + final int[] freqs = bulkResult.freqs.ints; + int i = 0; + final int length = docs.length; + while (i < length && ord < limit) { + ord++; + // manually inlined call to next() for speed + final int code = freqIn.readVInt(); + if (omitTF) { + doc += code; + } else { + doc += code >>> 1; // shift off low bit + if ((code & 1) != 0) { // if low bit is set + freq = 1; // freq is one + } else { + freq = freqIn.readVInt(); // else read freq + } + } + + if (liveDocs == null || liveDocs.get(doc)) { + docs[i] = doc; + freqs[i] = freq; + ++i; + } + } + + return i; + } + + @Override + public int docID() { + return doc; + } + + @Override + public int freq() { + return freq; + } + + @Override + public int advance(int target) throws IOException { + + if ((target - skipInterval) >= doc && limit >= skipMinimum) { + + // There are enough docs in the posting to have + // skip data, and it isn't too close. + + if (skipper == null) { + // This is the first time this enum has ever been used for skipping -- do lazy init + skipper = new DefaultSkipListReader((IndexInput) freqIn.clone(), maxSkipLevels, skipInterval); + } + + if (!skipped) { + + // This is the first time this posting has + // skipped since reset() was called, so now we + // load the skip data for this posting + + skipper.init(freqOffset + skipOffset, + freqOffset, 0, + limit, storePayloads); + + skipped = true; + } + + final int newOrd = skipper.skipTo(target); + + if (newOrd > ord) { + // Skipper moved + + ord = newOrd; + doc = skipper.getDoc(); + freqIn.seek(skipper.getFreqPointer()); + } + } + + // scan for the rest: + do { + nextDoc(); + } while (target > doc); + + return doc; + } + } + + // Decodes docs & positions. payloads are not present. + private class SegmentDocsAndPositionsEnum extends DocsAndPositionsEnum { + final IndexInput startFreqIn; + private final IndexInput freqIn; + private final IndexInput proxIn; + + int limit; // number of docs in this posting + int ord; // how many docs we've read + int doc; // doc we last read + int freq; // freq we last read + int position; + + Bits liveDocs; + + long freqOffset; + int skipOffset; + long proxOffset; + + int posPendingCount; + + boolean skipped; + DefaultSkipListReader skipper; + private long lazyProxPointer; + + public SegmentDocsAndPositionsEnum(IndexInput freqIn, IndexInput proxIn) throws IOException { + startFreqIn = freqIn; + this.freqIn = (IndexInput) freqIn.clone(); + this.proxIn = (IndexInput) proxIn.clone(); + } + + public SegmentDocsAndPositionsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException { + assert fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + assert !fieldInfo.storePayloads; + + this.liveDocs = liveDocs; + + // TODO: for full enum case (eg segment merging) this + // seek is unnecessary; maybe we can avoid in such + // cases + freqIn.seek(termState.freqOffset); + lazyProxPointer = termState.proxOffset; + + limit = termState.docFreq; + assert limit > 0; + + ord = 0; + doc = 0; + position = 0; + + skipped = false; + posPendingCount = 0; + + freqOffset = termState.freqOffset; + proxOffset = termState.proxOffset; + skipOffset = termState.skipOffset; + // if (DEBUG) System.out.println("StandardR.D&PE reset seg=" + segment + " limit=" + limit + " freqFP=" + freqOffset + " proxFP=" + proxOffset); + + return this; + } + + @Override + public int nextDoc() throws IOException { + // if (DEBUG) System.out.println("SPR.nextDoc seg=" + segment + " freqIn.fp=" + freqIn.getFilePointer()); + while(true) { + if (ord == limit) { + // if (DEBUG) System.out.println(" return END"); + return doc = NO_MORE_DOCS; + } + + ord++; + + // Decode next doc/freq pair + final int code = freqIn.readVInt(); + + doc += code >>> 1; // shift off low bit + if ((code & 1) != 0) { // if low bit is set + freq = 1; // freq is one + } else { + freq = freqIn.readVInt(); // else read freq + } + posPendingCount += freq; + + if (liveDocs == null || liveDocs.get(doc)) { + break; + } + } + + position = 0; + + // if (DEBUG) System.out.println(" return doc=" + doc); + return doc; + } + + @Override + public int docID() { + return doc; + } + + @Override + public int freq() { + return freq; + } + + @Override + public int advance(int target) throws IOException { + + //System.out.println("StandardR.D&PE advance target=" + target); + + if ((target - skipInterval) >= doc && limit >= skipMinimum) { + + // There are enough docs in the posting to have + // skip data, and it isn't too close + + if (skipper == null) { + // This is the first time this enum has ever been used for skipping -- do lazy init + skipper = new DefaultSkipListReader((IndexInput) freqIn.clone(), maxSkipLevels, skipInterval); + } + + if (!skipped) { + + // This is the first time this posting has + // skipped, since reset() was called, so now we + // load the skip data for this posting + + skipper.init(freqOffset+skipOffset, + freqOffset, proxOffset, + limit, false); + + skipped = true; + } + + final int newOrd = skipper.skipTo(target); + + if (newOrd > ord) { + // Skipper moved + ord = newOrd; + doc = skipper.getDoc(); + freqIn.seek(skipper.getFreqPointer()); + lazyProxPointer = skipper.getProxPointer(); + posPendingCount = 0; + position = 0; + } + } + + // Now, linear scan for the rest: + do { + nextDoc(); + } while (target > doc); + + return doc; + } + + @Override + public int nextPosition() throws IOException { + + if (lazyProxPointer != -1) { + proxIn.seek(lazyProxPointer); + lazyProxPointer = -1; + } + + // scan over any docs that were iterated without their positions + if (posPendingCount > freq) { + position = 0; + while(posPendingCount != freq) { + if ((proxIn.readByte() & 0x80) == 0) { + posPendingCount--; + } + } + } + + position += proxIn.readVInt(); + + posPendingCount--; + + assert posPendingCount >= 0: "nextPosition() was called too many times (more than freq() times) posPendingCount=" + posPendingCount; + + return position; + } + + /** Returns the payload at this position, or null if no + * payload was indexed. */ + @Override + public BytesRef getPayload() throws IOException { + throw new IOException("No payloads exist for this field!"); + } + + @Override + public boolean hasPayload() { + return false; + } + } + + // Decodes docs & positions & payloads + private class SegmentDocsAndPositionsAndPayloadsEnum extends DocsAndPositionsEnum { + final IndexInput startFreqIn; + private final IndexInput freqIn; + private final IndexInput proxIn; + + int limit; // number of docs in this posting + int ord; // how many docs we've read + int doc; // doc we last read + int freq; // freq we last read + int position; + + Bits liveDocs; + + long freqOffset; + int skipOffset; + long proxOffset; + + int posPendingCount; + int payloadLength; + boolean payloadPending; + + boolean skipped; + DefaultSkipListReader skipper; + private BytesRef payload; + private long lazyProxPointer; + + public SegmentDocsAndPositionsAndPayloadsEnum(IndexInput freqIn, IndexInput proxIn) throws IOException { + startFreqIn = freqIn; + this.freqIn = (IndexInput) freqIn.clone(); + this.proxIn = (IndexInput) proxIn.clone(); + } + + public SegmentDocsAndPositionsAndPayloadsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException { + assert fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; + assert fieldInfo.storePayloads; + if (payload == null) { + payload = new BytesRef(); + payload.bytes = new byte[1]; + } + + this.liveDocs = liveDocs; + + // TODO: for full enum case (eg segment merging) this + // seek is unnecessary; maybe we can avoid in such + // cases + freqIn.seek(termState.freqOffset); + lazyProxPointer = termState.proxOffset; + + limit = termState.docFreq; + ord = 0; + doc = 0; + position = 0; + + skipped = false; + posPendingCount = 0; + payloadPending = false; + + freqOffset = termState.freqOffset; + proxOffset = termState.proxOffset; + skipOffset = termState.skipOffset; + //System.out.println("StandardR.D&PE reset seg=" + segment + " limit=" + limit + " freqFP=" + freqOffset + " proxFP=" + proxOffset + " this=" + this); + + return this; + } + + @Override + public int nextDoc() throws IOException { + while(true) { + if (ord == limit) { + //System.out.println("StandardR.D&PE seg=" + segment + " nextDoc return doc=END"); + return doc = NO_MORE_DOCS; + } + + ord++; + + // Decode next doc/freq pair + final int code = freqIn.readVInt(); + + doc += code >>> 1; // shift off low bit + if ((code & 1) != 0) { // if low bit is set + freq = 1; // freq is one + } else { + freq = freqIn.readVInt(); // else read freq + } + posPendingCount += freq; + + if (liveDocs == null || liveDocs.get(doc)) { + break; + } + } + + position = 0; + + //System.out.println("StandardR.D&PE nextDoc seg=" + segment + " return doc=" + doc); + return doc; + } + + @Override + public int docID() { + return doc; + } + + @Override + public int freq() { + return freq; + } + + @Override + public int advance(int target) throws IOException { + + //System.out.println("StandardR.D&PE advance seg=" + segment + " target=" + target + " this=" + this); + + if ((target - skipInterval) >= doc && limit >= skipMinimum) { + + // There are enough docs in the posting to have + // skip data, and it isn't too close + + if (skipper == null) { + // This is the first time this enum has ever been used for skipping -- do lazy init + skipper = new DefaultSkipListReader((IndexInput) freqIn.clone(), maxSkipLevels, skipInterval); + } + + if (!skipped) { + + // This is the first time this posting has + // skipped, since reset() was called, so now we + // load the skip data for this posting + //System.out.println(" init skipper freqOffset=" + freqOffset + " skipOffset=" + skipOffset + " vs len=" + freqIn.length()); + skipper.init(freqOffset+skipOffset, + freqOffset, proxOffset, + limit, true); + + skipped = true; + } + + final int newOrd = skipper.skipTo(target); + + if (newOrd > ord) { + // Skipper moved + ord = newOrd; + doc = skipper.getDoc(); + freqIn.seek(skipper.getFreqPointer()); + lazyProxPointer = skipper.getProxPointer(); + posPendingCount = 0; + position = 0; + payloadPending = false; + payloadLength = skipper.getPayloadLength(); + } + } + + // Now, linear scan for the rest: + do { + nextDoc(); + } while (target > doc); + + return doc; + } + + @Override + public int nextPosition() throws IOException { + + if (lazyProxPointer != -1) { + proxIn.seek(lazyProxPointer); + lazyProxPointer = -1; + } + + if (payloadPending && payloadLength > 0) { + // payload of last position as never retrieved -- skip it + proxIn.seek(proxIn.getFilePointer() + payloadLength); + payloadPending = false; + } + + // scan over any docs that were iterated without their positions + while(posPendingCount > freq) { + + final int code = proxIn.readVInt(); + + if ((code & 1) != 0) { + // new payload length + payloadLength = proxIn.readVInt(); + assert payloadLength >= 0; + } + + assert payloadLength != -1; + proxIn.seek(proxIn.getFilePointer() + payloadLength); + + posPendingCount--; + position = 0; + payloadPending = false; + //System.out.println("StandardR.D&PE skipPos"); + } + + // read next position + if (payloadPending && payloadLength > 0) { + // payload wasn't retrieved for last position + proxIn.seek(proxIn.getFilePointer()+payloadLength); + } + + final int code = proxIn.readVInt(); + if ((code & 1) != 0) { + // new payload length + payloadLength = proxIn.readVInt(); + assert payloadLength >= 0; + } + assert payloadLength != -1; + + payloadPending = true; + position += code >>> 1; + + posPendingCount--; + + assert posPendingCount >= 0: "nextPosition() was called too many times (more than freq() times) posPendingCount=" + posPendingCount; + + //System.out.println("StandardR.D&PE nextPos return pos=" + position); + return position; + } + + /** Returns the payload at this position, or null if no + * payload was indexed. */ + @Override + public BytesRef getPayload() throws IOException { + assert lazyProxPointer == -1; + assert posPendingCount < freq; + if (!payloadPending) { + throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once."); + } + if (payloadLength > payload.bytes.length) { + payload.grow(payloadLength); + } + + proxIn.readBytes(payload.bytes, 0, payloadLength); + payload.length = payloadLength; + payloadPending = false; + + return payload; + } + + @Override + public boolean hasPayload() { + return payloadPending && payloadLength > 0; + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsWriter.java 2011-11-02 21:26:22.472564907 -0400 @@ -0,0 +1,325 @@ +package org.apache.lucene.index.codecs.lucene40; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** Consumes doc & freq, writing them using the current + * index file format */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.TermStats; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CodecUtil; + +/** @lucene.experimental */ +public final class Lucene40PostingsWriter extends PostingsWriterBase { + final static String CODEC = "Lucene40PostingsWriter"; + + //private static boolean DEBUG = BlockTreeTermsWriter.DEBUG; + + // Increment version to change it: + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; + + final IndexOutput freqOut; + final IndexOutput proxOut; + final DefaultSkipListWriter skipListWriter; + /** Expert: The fraction of TermDocs entries stored in skip tables, + * used to accelerate {@link DocsEnum#advance(int)}. Larger values result in + * smaller indexes, greater acceleration, but fewer accelerable cases, while + * smaller values result in bigger indexes, less acceleration and more + * accelerable cases. More detailed experiments would be useful here. */ + static final int DEFAULT_SKIP_INTERVAL = 16; + final int skipInterval; + + /** + * Expert: minimum docFreq to write any skip data at all + */ + final int skipMinimum; + + /** Expert: The maximum number of skip levels. Smaller values result in + * slightly smaller indexes, but slower skipping in big posting lists. + */ + final int maxSkipLevels = 10; + final int totalNumDocs; + IndexOutput termsOut; + + IndexOptions indexOptions; + boolean storePayloads; + // Starts a new term + long freqStart; + long proxStart; + FieldInfo fieldInfo; + int lastPayloadLength; + int lastPosition; + + // private String segment; + + public Lucene40PostingsWriter(SegmentWriteState state) throws IOException { + this(state, DEFAULT_SKIP_INTERVAL); + } + + public Lucene40PostingsWriter(SegmentWriteState state, int skipInterval) throws IOException { + super(); + this.skipInterval = skipInterval; + this.skipMinimum = skipInterval; /* set to the same for now */ + // this.segment = state.segmentName; + String fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.FREQ_EXTENSION); + freqOut = state.directory.createOutput(fileName, state.context); + if (state.fieldInfos.hasProx()) { + // At least one field does not omit TF, so create the + // prox file + fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, Lucene40PostingsFormat.PROX_EXTENSION); + proxOut = state.directory.createOutput(fileName, state.context); + } else { + // Every field omits TF so we will write no prox file + proxOut = null; + } + + totalNumDocs = state.numDocs; + + skipListWriter = new DefaultSkipListWriter(skipInterval, + maxSkipLevels, + state.numDocs, + freqOut, + proxOut); + } + + @Override + public void start(IndexOutput termsOut) throws IOException { + this.termsOut = termsOut; + CodecUtil.writeHeader(termsOut, CODEC, VERSION_CURRENT); + termsOut.writeInt(skipInterval); // write skipInterval + termsOut.writeInt(maxSkipLevels); // write maxSkipLevels + termsOut.writeInt(skipMinimum); // write skipMinimum + } + + @Override + public void startTerm() { + freqStart = freqOut.getFilePointer(); + //if (DEBUG) System.out.println("SPW: startTerm freqOut.fp=" + freqStart); + if (proxOut != null) { + proxStart = proxOut.getFilePointer(); + // force first payload to write its length + lastPayloadLength = -1; + } + skipListWriter.resetSkip(); + } + + // Currently, this instance is re-used across fields, so + // our parent calls setField whenever the field changes + @Override + public void setField(FieldInfo fieldInfo) { + //System.out.println("SPW: setField"); + /* + if (BlockTreeTermsWriter.DEBUG && fieldInfo.name.equals("id")) { + DEBUG = true; + } else { + DEBUG = false; + } + */ + this.fieldInfo = fieldInfo; + indexOptions = fieldInfo.indexOptions; + storePayloads = fieldInfo.storePayloads; + //System.out.println(" set init blockFreqStart=" + freqStart); + //System.out.println(" set init blockProxStart=" + proxStart); + } + + int lastDocID; + int df; + + /** Adds a new doc in this term. If this returns null + * then we just skip consuming positions/payloads. */ + @Override + public void startDoc(int docID, int termDocFreq) throws IOException { + // if (DEBUG) System.out.println("SPW: startDoc seg=" + segment + " docID=" + docID + " tf=" + termDocFreq + " freqOut.fp=" + freqOut.getFilePointer()); + + final int delta = docID - lastDocID; + + if (docID < 0 || (df > 0 && delta <= 0)) { + throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )"); + } + + if ((++df % skipInterval) == 0) { + skipListWriter.setSkipData(lastDocID, storePayloads, lastPayloadLength); + skipListWriter.bufferSkip(df); + } + + assert docID < totalNumDocs: "docID=" + docID + " totalNumDocs=" + totalNumDocs; + + lastDocID = docID; + if (indexOptions == IndexOptions.DOCS_ONLY) { + freqOut.writeVInt(delta); + } else if (1 == termDocFreq) { + freqOut.writeVInt((delta<<1) | 1); + } else { + freqOut.writeVInt(delta<<1); + freqOut.writeVInt(termDocFreq); + } + + lastPosition = 0; + } + + /** Add a new position & payload */ + @Override + public void addPosition(int position, BytesRef payload) throws IOException { + //if (DEBUG) System.out.println("SPW: addPos pos=" + position + " payload=" + (payload == null ? "null" : (payload.length + " bytes")) + " proxFP=" + proxOut.getFilePointer()); + assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS: "invalid indexOptions: " + indexOptions; + assert proxOut != null; + + final int delta = position - lastPosition; + + assert delta >= 0: "position=" + position + " lastPosition=" + lastPosition; // not quite right (if pos=0 is repeated twice we don't catch it) + + lastPosition = position; + + if (storePayloads) { + final int payloadLength = payload == null ? 0 : payload.length; + + if (payloadLength != lastPayloadLength) { + lastPayloadLength = payloadLength; + proxOut.writeVInt((delta<<1)|1); + proxOut.writeVInt(payloadLength); + } else { + proxOut.writeVInt(delta << 1); + } + + if (payloadLength > 0) { + proxOut.writeBytes(payload.bytes, payload.offset, payloadLength); + } + } else { + proxOut.writeVInt(delta); + } + } + + @Override + public void finishDoc() { + } + + private static class PendingTerm { + public final long freqStart; + public final long proxStart; + public final int skipOffset; + + public PendingTerm(long freqStart, long proxStart, int skipOffset) { + this.freqStart = freqStart; + this.proxStart = proxStart; + this.skipOffset = skipOffset; + } + } + + private final List pendingTerms = new ArrayList(); + + /** Called when we are done adding docs to this term */ + @Override + public void finishTerm(TermStats stats) throws IOException { + + // if (DEBUG) System.out.println("SPW: finishTerm seg=" + segment + " freqStart=" + freqStart); + assert stats.docFreq > 0; + + // TODO: wasteful we are counting this (counting # docs + // for this term) in two places? + assert stats.docFreq == df; + + final int skipOffset; + if (df >= skipMinimum) { + skipOffset = (int) (skipListWriter.writeSkip(freqOut)-freqStart); + } else { + skipOffset = -1; + } + + pendingTerms.add(new PendingTerm(freqStart, proxStart, skipOffset)); + + lastDocID = 0; + df = 0; + } + + private final RAMOutputStream bytesWriter = new RAMOutputStream(); + + @Override + public void flushTermsBlock(int start, int count) throws IOException { + //if (DEBUG) System.out.println("SPW: flushTermsBlock start=" + start + " count=" + count + " left=" + (pendingTerms.size()-count) + " pendingTerms.size()=" + pendingTerms.size()); + + if (count == 0) { + termsOut.writeByte((byte) 0); + return; + } + + assert start <= pendingTerms.size(); + assert count <= start; + + final int limit = pendingTerms.size() - start + count; + final PendingTerm firstTerm = pendingTerms.get(limit - count); + // First term in block is abs coded: + bytesWriter.writeVLong(firstTerm.freqStart); + + if (firstTerm.skipOffset != -1) { + assert firstTerm.skipOffset > 0; + bytesWriter.writeVInt(firstTerm.skipOffset); + } + if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { + bytesWriter.writeVLong(firstTerm.proxStart); + } + long lastFreqStart = firstTerm.freqStart; + long lastProxStart = firstTerm.proxStart; + for(int idx=limit-count+1; idxNote that this codec implements advance as a linear - * scan! This means if you store large fields in here, - * queries that rely on advance will (AND BooleanQuery, - * PhraseQuery) will be relatively slow! - * - *

NOTE: this codec cannot address more than ~2.1 GB - * of postings, because the underlying FST uses an int - * to address the underlying byte[]. - * - * @lucene.experimental */ - -public class MemoryCodec extends Codec { - - public MemoryCodec() { - super("Memory"); - } - - private static final boolean VERBOSE = false; - - private final static class TermsWriter extends TermsConsumer { - private final IndexOutput out; - private final FieldInfo field; - private final Builder builder; - private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton(); - private int termCount; - - public TermsWriter(IndexOutput out, FieldInfo field) { - this.out = out; - this.field = field; - builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); - } - - private class PostingsWriter extends PostingsConsumer { - private int lastDocID; - private int lastPos; - private int lastPayloadLen; - - // NOTE: not private so we don't pay access check at runtime: - int docCount; - RAMOutputStream buffer = new RAMOutputStream(); - - @Override - public void startDoc(int docID, int termDocFreq) throws IOException { - if (VERBOSE) System.out.println(" startDoc docID=" + docID + " freq=" + termDocFreq); - final int delta = docID - lastDocID; - assert docID == 0 || delta > 0; - lastDocID = docID; - docCount++; - - if (field.indexOptions == IndexOptions.DOCS_ONLY) { - buffer.writeVInt(delta); - } else if (termDocFreq == 1) { - buffer.writeVInt((delta<<1) | 1); - } else { - buffer.writeVInt(delta<<1); - assert termDocFreq > 0; - buffer.writeVInt(termDocFreq); - } - - lastPos = 0; - } - - @Override - public void addPosition(int pos, BytesRef payload) throws IOException { - assert payload == null || field.storePayloads; - - if (VERBOSE) System.out.println(" addPos pos=" + pos + " payload=" + payload); - - final int delta = pos - lastPos; - assert delta >= 0; - lastPos = pos; - - if (field.storePayloads) { - final int payloadLen = payload == null ? 0 : payload.length; - if (payloadLen != lastPayloadLen) { - lastPayloadLen = payloadLen; - buffer.writeVInt((delta<<1)|1); - buffer.writeVInt(payloadLen); - } else { - buffer.writeVInt(delta<<1); - } - - if (payloadLen > 0) { - buffer.writeBytes(payload.bytes, payload.offset, payloadLen); - } - } else { - buffer.writeVInt(delta); - } - } - - @Override - public void finishDoc() { - } - - public PostingsWriter reset() { - assert buffer.getFilePointer() == 0; - lastDocID = 0; - docCount = 0; - lastPayloadLen = 0; - return this; - } - } - - private final PostingsWriter postingsWriter = new PostingsWriter(); - - @Override - public PostingsConsumer startTerm(BytesRef text) { - if (VERBOSE) System.out.println(" startTerm term=" + text.utf8ToString()); - return postingsWriter.reset(); - } - - private final RAMOutputStream buffer2 = new RAMOutputStream(); - private final BytesRef spare = new BytesRef(); - private byte[] finalBuffer = new byte[128]; - - @Override - public void finishTerm(BytesRef text, TermStats stats) throws IOException { - - assert postingsWriter.docCount == stats.docFreq; - - assert buffer2.getFilePointer() == 0; - - buffer2.writeVInt(stats.docFreq); - if (field.indexOptions != IndexOptions.DOCS_ONLY) { - buffer2.writeVLong(stats.totalTermFreq-stats.docFreq); - } - int pos = (int) buffer2.getFilePointer(); - buffer2.writeTo(finalBuffer, 0); - buffer2.reset(); - - final int totalBytes = pos + (int) postingsWriter.buffer.getFilePointer(); - if (totalBytes > finalBuffer.length) { - finalBuffer = ArrayUtil.grow(finalBuffer, totalBytes); - } - postingsWriter.buffer.writeTo(finalBuffer, pos); - postingsWriter.buffer.reset(); - - spare.bytes = finalBuffer; - spare.length = totalBytes; - if (VERBOSE) { - System.out.println(" finishTerm term=" + text.utf8ToString() + " " + totalBytes + " bytes totalTF=" + stats.totalTermFreq); - for(int i=0;i 0) { - out.writeVInt(termCount); - out.writeVInt(field.number); - if (field.indexOptions != IndexOptions.DOCS_ONLY) { - out.writeVLong(sumTotalTermFreq); - } - out.writeVLong(sumDocFreq); - out.writeVInt(docCount); - builder.finish().save(out); - if (VERBOSE) System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer()); - } - } - - @Override - public Comparator getComparator() { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } - } - - private static String EXTENSION = "ram"; - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - - final String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, EXTENSION); - final IndexOutput out = state.directory.createOutput(fileName, state.context); - - return new FieldsConsumer() { - @Override - public TermsConsumer addField(FieldInfo field) { - if (VERBOSE) System.out.println("\naddField field=" + field.name); - return new TermsWriter(out, field); - } - - @Override - public void close() throws IOException { - // EOF marker: - try { - out.writeVInt(0); - } finally { - out.close(); - } - } - }; - } - - private final static class FSTDocsEnum extends DocsEnum { - private final IndexOptions indexOptions; - private final boolean storePayloads; - private byte[] buffer = new byte[16]; - private final ByteArrayDataInput in = new ByteArrayDataInput(buffer); - - private Bits liveDocs; - private int docUpto; - private int docID; - private int freq; - private int payloadLen; - private int numDocs; - - public FSTDocsEnum(IndexOptions indexOptions, boolean storePayloads) { - this.indexOptions = indexOptions; - this.storePayloads = storePayloads; - } - - public boolean canReuse(IndexOptions indexOptions, boolean storePayloads) { - return indexOptions == this.indexOptions && storePayloads == this.storePayloads; - } - - public FSTDocsEnum reset(BytesRef bufferIn, Bits liveDocs, int numDocs) { - assert numDocs > 0; - if (buffer.length < bufferIn.length - bufferIn.offset) { - buffer = ArrayUtil.grow(buffer, bufferIn.length - bufferIn.offset); - } - in.reset(buffer, 0, bufferIn.length - bufferIn.offset); - System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length - bufferIn.offset); - this.liveDocs = liveDocs; - docID = 0; - docUpto = 0; - payloadLen = 0; - this.numDocs = numDocs; - return this; - } - - @Override - public int nextDoc() { - while(true) { - if (VERBOSE) System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this); - if (docUpto == numDocs) { - if (VERBOSE) { - System.out.println(" END"); - } - return docID = NO_MORE_DOCS; - } - docUpto++; - if (indexOptions == IndexOptions.DOCS_ONLY) { - docID += in.readVInt(); - freq = 1; - } else { - final int code = in.readVInt(); - docID += code >>> 1; - if (VERBOSE) System.out.println(" docID=" + docID + " code=" + code); - if ((code & 1) != 0) { - freq = 1; - } else { - freq = in.readVInt(); - assert freq > 0; - } - - if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { - // Skip positions - for(int posUpto=0;posUpto 0) { - nextPosition(); - } - while(true) { - if (VERBOSE) System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this); - if (docUpto == numDocs) { - if (VERBOSE) System.out.println(" END"); - return docID = NO_MORE_DOCS; - } - docUpto++; - - final int code = in.readVInt(); - docID += code >>> 1; - if ((code & 1) != 0) { - freq = 1; - } else { - freq = in.readVInt(); - assert freq > 0; - } - - if (liveDocs == null || liveDocs.get(docID)) { - pos = 0; - posPending = freq; - if (VERBOSE) System.out.println(" return docID=" + docID + " freq=" + freq); - return docID; - } - - // Skip positions - for(int posUpto=0;posUpto fstEnum; - private final ByteArrayDataInput buffer = new ByteArrayDataInput(); - private boolean didDecode; - - private int docFreq; - private long totalTermFreq; - private BytesRefFSTEnum.InputOutput current; - - public FSTTermsEnum(FieldInfo field, FST fst) { - this.field = field; - fstEnum = new BytesRefFSTEnum(fst); - } - - private void decodeMetaData() throws IOException { - if (!didDecode) { - buffer.reset(current.output.bytes, 0, current.output.length); - docFreq = buffer.readVInt(); - if (field.indexOptions != IndexOptions.DOCS_ONLY) { - totalTermFreq = docFreq + buffer.readVLong(); - } else { - totalTermFreq = -1; - } - current.output.offset = buffer.getPosition(); - if (VERBOSE) System.out.println(" df=" + docFreq + " totTF=" + totalTermFreq + " offset=" + buffer.getPosition() + " len=" + current.output.length); - didDecode = true; - } - } - - @Override - public boolean seekExact(BytesRef text, boolean useCache /* ignored */) throws IOException { - if (VERBOSE) System.out.println("te.seekExact text=" + field.name + ":" + text.utf8ToString() + " this=" + this); - current = fstEnum.seekExact(text); - didDecode = false; - return current != null; - } - - @Override - public SeekStatus seekCeil(BytesRef text, boolean useCache /* ignored */) throws IOException { - if (VERBOSE) System.out.println("te.seek text=" + field.name + ":" + text.utf8ToString() + " this=" + this); - current = fstEnum.seekCeil(text); - if (current == null) { - return SeekStatus.END; - } else { - if (VERBOSE) { - System.out.println(" got term=" + current.input.utf8ToString()); - for(int i=0;i getComparator() { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } - - @Override - public void seekExact(long ord) { - // NOTE: we could add this... - throw new UnsupportedOperationException(); - } - - @Override - public long ord() { - // NOTE: we could add this... - throw new UnsupportedOperationException(); - } - } - - private final static class TermsReader extends Terms { - - private final long sumTotalTermFreq; - private final long sumDocFreq; - private final int docCount; - private final int termCount; - private FST fst; - private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton(); - private final FieldInfo field; - - public TermsReader(FieldInfos fieldInfos, IndexInput in, int termCount) throws IOException { - this.termCount = termCount; - final int fieldNumber = in.readVInt(); - field = fieldInfos.fieldInfo(fieldNumber); - if (field.indexOptions != IndexOptions.DOCS_ONLY) { - sumTotalTermFreq = in.readVLong(); - } else { - sumTotalTermFreq = -1; - } - sumDocFreq = in.readVLong(); - docCount = in.readVInt(); - - fst = new FST(in, outputs); - } - - @Override - public long getSumTotalTermFreq() { - return sumTotalTermFreq; - } - - @Override - public long getSumDocFreq() throws IOException { - return sumDocFreq; - } - - @Override - public int getDocCount() throws IOException { - return docCount; - } - - @Override - public long getUniqueTermCount() throws IOException { - return termCount; - } - - @Override - public TermsEnum iterator() { - return new FSTTermsEnum(field, fst); - } - - @Override - public Comparator getComparator() { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, EXTENSION); - final IndexInput in = state.dir.openInput(fileName, IOContext.READONCE); - - final SortedMap fields = new TreeMap(); - - try { - while(true) { - final int termCount = in.readVInt(); - if (termCount == 0) { - break; - } - final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount); - fields.put(termsReader.field.name, termsReader); - } - } finally { - in.close(); - } - - return new FieldsProducer() { - @Override - public FieldsEnum iterator() { - final Iterator iter = fields.values().iterator(); - - return new FieldsEnum() { - - private TermsReader current; - - @Override - public String next() { - current = iter.next(); - return current.field.name; - } - - public TermsEnum terms() { - return current.iterator(); - } - }; - } - - @Override - public Terms terms(String field) { - return fields.get(field); - } - - @Override - public void close() { - // Drop ref to FST: - for(TermsReader termsReader : fields.values()) { - termsReader.fst = null; - } - } - }; - } - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, EXTENSION)); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); - } - - @Override - public void getExtensions(Set extensions) { - extensions.add(EXTENSION); - DefaultDocValuesConsumer.getExtensions(extensions); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java 2011-11-02 21:26:22.504564906 -0400 @@ -0,0 +1,796 @@ +package org.apache.lucene.index.codecs.memory; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Comparator; +import java.util.Iterator; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; + +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PostingsConsumer; +import org.apache.lucene.index.codecs.TermStats; +import org.apache.lucene.index.codecs.TermsConsumer; +import org.apache.lucene.store.ByteArrayDataInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.RAMOutputStream; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.fst.Builder; +import org.apache.lucene.util.fst.ByteSequenceOutputs; +import org.apache.lucene.util.fst.BytesRefFSTEnum; +import org.apache.lucene.util.fst.FST; + +// TODO: would be nice to somehow allow this to act like +// InstantiatedIndex, by never writing to disk; ie you write +// to this Codec in RAM only and then when you open a reader +// it pulls the FST directly from what you wrote w/o going +// to disk. + +/** Stores terms & postings (docs, positions, payloads) in + * RAM, using an FST. + * + *

Note that this codec implements advance as a linear + * scan! This means if you store large fields in here, + * queries that rely on advance will (AND BooleanQuery, + * PhraseQuery) will be relatively slow! + * + *

NOTE: this codec cannot address more than ~2.1 GB + * of postings, because the underlying FST uses an int + * to address the underlying byte[]. + * + * @lucene.experimental */ + +// TODO: Maybe name this 'Cached' or something to reflect +// the reality that it is actually written to disk, but +// loads itself in ram? +public class MemoryPostingsFormat extends PostingsFormat { + + public MemoryPostingsFormat() { + super("Memory"); + } + + private static final boolean VERBOSE = false; + + private final static class TermsWriter extends TermsConsumer { + private final IndexOutput out; + private final FieldInfo field; + private final Builder builder; + private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton(); + private int termCount; + + public TermsWriter(IndexOutput out, FieldInfo field) { + this.out = out; + this.field = field; + builder = new Builder(FST.INPUT_TYPE.BYTE1, outputs); + } + + private class PostingsWriter extends PostingsConsumer { + private int lastDocID; + private int lastPos; + private int lastPayloadLen; + + // NOTE: not private so we don't pay access check at runtime: + int docCount; + RAMOutputStream buffer = new RAMOutputStream(); + + @Override + public void startDoc(int docID, int termDocFreq) throws IOException { + if (VERBOSE) System.out.println(" startDoc docID=" + docID + " freq=" + termDocFreq); + final int delta = docID - lastDocID; + assert docID == 0 || delta > 0; + lastDocID = docID; + docCount++; + + if (field.indexOptions == IndexOptions.DOCS_ONLY) { + buffer.writeVInt(delta); + } else if (termDocFreq == 1) { + buffer.writeVInt((delta<<1) | 1); + } else { + buffer.writeVInt(delta<<1); + assert termDocFreq > 0; + buffer.writeVInt(termDocFreq); + } + + lastPos = 0; + } + + @Override + public void addPosition(int pos, BytesRef payload) throws IOException { + assert payload == null || field.storePayloads; + + if (VERBOSE) System.out.println(" addPos pos=" + pos + " payload=" + payload); + + final int delta = pos - lastPos; + assert delta >= 0; + lastPos = pos; + + if (field.storePayloads) { + final int payloadLen = payload == null ? 0 : payload.length; + if (payloadLen != lastPayloadLen) { + lastPayloadLen = payloadLen; + buffer.writeVInt((delta<<1)|1); + buffer.writeVInt(payloadLen); + } else { + buffer.writeVInt(delta<<1); + } + + if (payloadLen > 0) { + buffer.writeBytes(payload.bytes, payload.offset, payloadLen); + } + } else { + buffer.writeVInt(delta); + } + } + + @Override + public void finishDoc() { + } + + public PostingsWriter reset() { + assert buffer.getFilePointer() == 0; + lastDocID = 0; + docCount = 0; + lastPayloadLen = 0; + return this; + } + } + + private final PostingsWriter postingsWriter = new PostingsWriter(); + + @Override + public PostingsConsumer startTerm(BytesRef text) { + if (VERBOSE) System.out.println(" startTerm term=" + text.utf8ToString()); + return postingsWriter.reset(); + } + + private final RAMOutputStream buffer2 = new RAMOutputStream(); + private final BytesRef spare = new BytesRef(); + private byte[] finalBuffer = new byte[128]; + + @Override + public void finishTerm(BytesRef text, TermStats stats) throws IOException { + + assert postingsWriter.docCount == stats.docFreq; + + assert buffer2.getFilePointer() == 0; + + buffer2.writeVInt(stats.docFreq); + if (field.indexOptions != IndexOptions.DOCS_ONLY) { + buffer2.writeVLong(stats.totalTermFreq-stats.docFreq); + } + int pos = (int) buffer2.getFilePointer(); + buffer2.writeTo(finalBuffer, 0); + buffer2.reset(); + + final int totalBytes = pos + (int) postingsWriter.buffer.getFilePointer(); + if (totalBytes > finalBuffer.length) { + finalBuffer = ArrayUtil.grow(finalBuffer, totalBytes); + } + postingsWriter.buffer.writeTo(finalBuffer, pos); + postingsWriter.buffer.reset(); + + spare.bytes = finalBuffer; + spare.length = totalBytes; + if (VERBOSE) { + System.out.println(" finishTerm term=" + text.utf8ToString() + " " + totalBytes + " bytes totalTF=" + stats.totalTermFreq); + for(int i=0;i 0) { + out.writeVInt(termCount); + out.writeVInt(field.number); + if (field.indexOptions != IndexOptions.DOCS_ONLY) { + out.writeVLong(sumTotalTermFreq); + } + out.writeVLong(sumDocFreq); + out.writeVInt(docCount); + builder.finish().save(out); + if (VERBOSE) System.out.println("finish field=" + field.name + " fp=" + out.getFilePointer()); + } + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + } + + private static String EXTENSION = "ram"; + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + + final String fileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, EXTENSION); + final IndexOutput out = state.directory.createOutput(fileName, state.context); + + return new FieldsConsumer() { + @Override + public TermsConsumer addField(FieldInfo field) { + if (VERBOSE) System.out.println("\naddField field=" + field.name); + return new TermsWriter(out, field); + } + + @Override + public void close() throws IOException { + // EOF marker: + try { + out.writeVInt(0); + } finally { + out.close(); + } + } + }; + } + + private final static class FSTDocsEnum extends DocsEnum { + private final IndexOptions indexOptions; + private final boolean storePayloads; + private byte[] buffer = new byte[16]; + private final ByteArrayDataInput in = new ByteArrayDataInput(buffer); + + private Bits liveDocs; + private int docUpto; + private int docID; + private int freq; + private int payloadLen; + private int numDocs; + + public FSTDocsEnum(IndexOptions indexOptions, boolean storePayloads) { + this.indexOptions = indexOptions; + this.storePayloads = storePayloads; + } + + public boolean canReuse(IndexOptions indexOptions, boolean storePayloads) { + return indexOptions == this.indexOptions && storePayloads == this.storePayloads; + } + + public FSTDocsEnum reset(BytesRef bufferIn, Bits liveDocs, int numDocs) { + assert numDocs > 0; + if (buffer.length < bufferIn.length - bufferIn.offset) { + buffer = ArrayUtil.grow(buffer, bufferIn.length - bufferIn.offset); + } + in.reset(buffer, 0, bufferIn.length - bufferIn.offset); + System.arraycopy(bufferIn.bytes, bufferIn.offset, buffer, 0, bufferIn.length - bufferIn.offset); + this.liveDocs = liveDocs; + docID = 0; + docUpto = 0; + payloadLen = 0; + this.numDocs = numDocs; + return this; + } + + @Override + public int nextDoc() { + while(true) { + if (VERBOSE) System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this); + if (docUpto == numDocs) { + if (VERBOSE) { + System.out.println(" END"); + } + return docID = NO_MORE_DOCS; + } + docUpto++; + if (indexOptions == IndexOptions.DOCS_ONLY) { + docID += in.readVInt(); + freq = 1; + } else { + final int code = in.readVInt(); + docID += code >>> 1; + if (VERBOSE) System.out.println(" docID=" + docID + " code=" + code); + if ((code & 1) != 0) { + freq = 1; + } else { + freq = in.readVInt(); + assert freq > 0; + } + + if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { + // Skip positions + for(int posUpto=0;posUpto 0) { + nextPosition(); + } + while(true) { + if (VERBOSE) System.out.println(" nextDoc cycle docUpto=" + docUpto + " numDocs=" + numDocs + " fp=" + in.getPosition() + " this=" + this); + if (docUpto == numDocs) { + if (VERBOSE) System.out.println(" END"); + return docID = NO_MORE_DOCS; + } + docUpto++; + + final int code = in.readVInt(); + docID += code >>> 1; + if ((code & 1) != 0) { + freq = 1; + } else { + freq = in.readVInt(); + assert freq > 0; + } + + if (liveDocs == null || liveDocs.get(docID)) { + pos = 0; + posPending = freq; + if (VERBOSE) System.out.println(" return docID=" + docID + " freq=" + freq); + return docID; + } + + // Skip positions + for(int posUpto=0;posUpto fstEnum; + private final ByteArrayDataInput buffer = new ByteArrayDataInput(); + private boolean didDecode; + + private int docFreq; + private long totalTermFreq; + private BytesRefFSTEnum.InputOutput current; + + public FSTTermsEnum(FieldInfo field, FST fst) { + this.field = field; + fstEnum = new BytesRefFSTEnum(fst); + } + + private void decodeMetaData() throws IOException { + if (!didDecode) { + buffer.reset(current.output.bytes, 0, current.output.length); + docFreq = buffer.readVInt(); + if (field.indexOptions != IndexOptions.DOCS_ONLY) { + totalTermFreq = docFreq + buffer.readVLong(); + } else { + totalTermFreq = -1; + } + current.output.offset = buffer.getPosition(); + if (VERBOSE) System.out.println(" df=" + docFreq + " totTF=" + totalTermFreq + " offset=" + buffer.getPosition() + " len=" + current.output.length); + didDecode = true; + } + } + + @Override + public boolean seekExact(BytesRef text, boolean useCache /* ignored */) throws IOException { + if (VERBOSE) System.out.println("te.seekExact text=" + field.name + ":" + text.utf8ToString() + " this=" + this); + current = fstEnum.seekExact(text); + didDecode = false; + return current != null; + } + + @Override + public SeekStatus seekCeil(BytesRef text, boolean useCache /* ignored */) throws IOException { + if (VERBOSE) System.out.println("te.seek text=" + field.name + ":" + text.utf8ToString() + " this=" + this); + current = fstEnum.seekCeil(text); + if (current == null) { + return SeekStatus.END; + } else { + if (VERBOSE) { + System.out.println(" got term=" + current.input.utf8ToString()); + for(int i=0;i getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public void seekExact(long ord) { + // NOTE: we could add this... + throw new UnsupportedOperationException(); + } + + @Override + public long ord() { + // NOTE: we could add this... + throw new UnsupportedOperationException(); + } + } + + private final static class TermsReader extends Terms { + + private final long sumTotalTermFreq; + private final long sumDocFreq; + private final int docCount; + private final int termCount; + private FST fst; + private final ByteSequenceOutputs outputs = ByteSequenceOutputs.getSingleton(); + private final FieldInfo field; + + public TermsReader(FieldInfos fieldInfos, IndexInput in, int termCount) throws IOException { + this.termCount = termCount; + final int fieldNumber = in.readVInt(); + field = fieldInfos.fieldInfo(fieldNumber); + if (field.indexOptions != IndexOptions.DOCS_ONLY) { + sumTotalTermFreq = in.readVLong(); + } else { + sumTotalTermFreq = -1; + } + sumDocFreq = in.readVLong(); + docCount = in.readVInt(); + + fst = new FST(in, outputs); + } + + @Override + public long getSumTotalTermFreq() { + return sumTotalTermFreq; + } + + @Override + public long getSumDocFreq() throws IOException { + return sumDocFreq; + } + + @Override + public int getDocCount() throws IOException { + return docCount; + } + + @Override + public long getUniqueTermCount() throws IOException { + return termCount; + } + + @Override + public TermsEnum iterator() { + return new FSTTermsEnum(field, fst); + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION); + final IndexInput in = state.dir.openInput(fileName, IOContext.READONCE); + + final SortedMap fields = new TreeMap(); + + try { + while(true) { + final int termCount = in.readVInt(); + if (termCount == 0) { + break; + } + final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount); + fields.put(termsReader.field.name, termsReader); + } + } finally { + in.close(); + } + + return new FieldsProducer() { + @Override + public FieldsEnum iterator() { + final Iterator iter = fields.values().iterator(); + + return new FieldsEnum() { + + private TermsReader current; + + @Override + public String next() { + current = iter.next(); + return current.field.name; + } + + public TermsEnum terms() { + return current.iterator(); + } + }; + } + + @Override + public Terms terms(String field) { + return fields.get(field); + } + + @Override + public void close() { + // Drop ref to FST: + for(TermsReader termsReader : fields.values()) { + termsReader.fst = null; + } + } + }; + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION)); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java 2011-10-10 14:34:12.708772412 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java 2011-10-25 11:35:27.928939265 -0400 @@ -29,7 +29,7 @@ * this convert field values into a Codec specific format during indexing. *

* The {@link PerDocConsumer} API is accessible through the - * {@link Codec} - API providing per field consumers and producers for inverted + * {@link PostingsFormat} - API providing per field consumers and producers for inverted * data (terms, postings) as well as per-document data. * * @lucene.experimental diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java 2011-09-30 20:57:07.298773589 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java 2011-10-25 11:35:27.928939265 -0400 @@ -28,7 +28,7 @@ * {@link PerDocConsumer} counterpart. *

* The {@link PerDocValues} API is accessible through the - * {@link Codec} - API providing per field consumers and producers for inverted + * {@link PostingsFormat} - API providing per field consumers and producers for inverted * data (terms, postings) as well as per-document data. * * @lucene.experimental diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/perfield/PerFieldPostingsFormat.java 2011-11-03 13:24:53.428535280 -0400 @@ -0,0 +1,341 @@ +package org.apache.lucene.index.codecs.perfield; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.TermsConsumer; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.util.IOUtils; + +/** + * Enables per field format support. + * + * @lucene.experimental + */ + +public abstract class PerFieldPostingsFormat extends PostingsFormat { + + public static final String PER_FIELD_EXTENSION = "per"; + public static final String PER_FIELD_NAME = "PerField40"; + + public static final int VERSION_START = 0; + public static final int VERSION_LATEST = VERSION_START; + + public PerFieldPostingsFormat() { + super(PER_FIELD_NAME); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) + throws IOException { + return new FieldsWriter(state); + } + + // NOTE: not private to avoid $accessN at runtime!! + static class FieldsConsumerAndID implements Closeable { + final FieldsConsumer fieldsConsumer; + final String segmentSuffix; + + public FieldsConsumerAndID(FieldsConsumer fieldsConsumer, String segmentSuffix) { + this.fieldsConsumer = fieldsConsumer; + this.segmentSuffix = segmentSuffix; + } + + @Override + public void close() throws IOException { + fieldsConsumer.close(); + } + }; + + private class FieldsWriter extends FieldsConsumer { + + private final Map formats = new IdentityHashMap(); + + /** Records all fields we wrote. */ + private final Map fieldToFormat = new HashMap(); + + private final SegmentWriteState segmentWriteState; + + public FieldsWriter(SegmentWriteState state) throws IOException { + segmentWriteState = state; + } + + @Override + public TermsConsumer addField(FieldInfo field) throws IOException { + final PostingsFormat format = getPostingsFormatForField(field.name); + if (format == null) { + throw new IllegalStateException("invalid null PostingsFormat for field=\"" + field.name + "\""); + } + + assert !fieldToFormat.containsKey(field.name); + fieldToFormat.put(field.name, format); + + FieldsConsumerAndID consumerAndId = formats.get(format); + if (consumerAndId == null) { + // First time we are seeing this format; assign + // next id and init it: + final String segmentSuffix = getFullSegmentSuffix(field.name, + segmentWriteState.segmentSuffix, + ""+formats.size()); + consumerAndId = new FieldsConsumerAndID(format.fieldsConsumer(new SegmentWriteState(segmentWriteState, segmentSuffix)), + segmentSuffix); + formats.put(format, consumerAndId); + } + + return consumerAndId.fieldsConsumer.addField(field); + } + + @Override + public void close() throws IOException { + + // Close all subs + IOUtils.close(formats.values()); + + // Write _X.per: maps field name -> format name and + // format name -> format id + final String mapFileName = IndexFileNames.segmentFileName(segmentWriteState.segmentName, segmentWriteState.segmentSuffix, PER_FIELD_EXTENSION); + final IndexOutput out = segmentWriteState.directory.createOutput(mapFileName, segmentWriteState.context); + boolean success = false; + try { + CodecUtil.writeHeader(out, PER_FIELD_NAME, VERSION_LATEST); + + // format name -> int id + out.writeVInt(formats.size()); + for(Map.Entry ent : formats.entrySet()) { + out.writeString(ent.getValue().segmentSuffix); + out.writeString(ent.getKey().getName()); + } + + // field name -> format name + out.writeVInt(fieldToFormat.size()); + for(Map.Entry ent : fieldToFormat.entrySet()) { + out.writeString(ent.getKey()); + out.writeString(ent.getValue().getName()); + } + + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + } else { + IOUtils.close(out); + } + } + } + } + + static String getFullSegmentSuffix(String fieldName, String outerSegmentSuffix, String segmentSuffix) { + if (outerSegmentSuffix.length() == 0) { + return segmentSuffix; + } else { + // TODO: support embedding; I think it should work but + // we need a test confirm to confirm + // return outerSegmentSuffix + "_" + segmentSuffix; + throw new IllegalStateException("cannot embed PerFieldPostingsFormat inside itself (field \"" + fieldName + "\" returned PerFieldPostingsFormat)"); + } + } + + private class FieldsReader extends FieldsProducer { + + private final Map fields = new TreeMap(); + private final Map formats = new IdentityHashMap(); + + public FieldsReader(final SegmentReadState readState) throws IOException { + + // Read _X.per and init each format: + boolean success = false; + try { + new VisitPerFieldFile(readState.dir, readState.segmentInfo.name, readState.segmentSuffix) { + @Override + protected void visitOneFormat(String segmentSuffix, PostingsFormat postingsFormat) throws IOException { + formats.put(postingsFormat, postingsFormat.fieldsProducer(new SegmentReadState(readState, segmentSuffix))); + } + + @Override + protected void visitOneField(String fieldName, PostingsFormat postingsFormat) throws IOException { + assert formats.containsKey(postingsFormat); + fields.put(fieldName, formats.get(postingsFormat)); + } + }; + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(formats.values()); + } + } + } + + private final class FieldsIterator extends FieldsEnum { + private final Iterator it; + private String current; + + public FieldsIterator() { + it = fields.keySet().iterator(); + } + + @Override + public String next() { + if (it.hasNext()) { + current = it.next(); + } else { + current = null; + } + + return current; + } + + @Override + public TermsEnum terms() throws IOException { + final Terms terms = fields.get(current).terms(current); + if (terms != null) { + return terms.iterator(); + } else { + return TermsEnum.EMPTY; + } + } + } + + @Override + public FieldsEnum iterator() throws IOException { + return new FieldsIterator(); + } + + @Override + public Terms terms(String field) throws IOException { + FieldsProducer fieldsProducer = fields.get(field); + return fieldsProducer == null ? null : fieldsProducer.terms(field); + } + + @Override + public void close() throws IOException { + IOUtils.close(formats.values()); + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) + throws IOException { + return new FieldsReader(state); + } + + private abstract class VisitPerFieldFile { + public VisitPerFieldFile(Directory dir, String segmentName, String outerSegmentSuffix) throws IOException { + final String mapFileName = IndexFileNames.segmentFileName(segmentName, outerSegmentSuffix, PER_FIELD_EXTENSION); + final IndexInput in = dir.openInput(mapFileName, IOContext.READONCE); + boolean success = false; + try { + CodecUtil.checkHeader(in, PER_FIELD_NAME, VERSION_START, VERSION_LATEST); + + // Read format name -> format id + final int formatCount = in.readVInt(); + for(int formatIDX=0;formatIDX " + postingsFormat); + if (postingsFormat == null) { + throw new IllegalStateException("unable to lookup PostingsFormat for name=\"" + formatName + "\": got null"); + } + + // Better be defined, because it was defined + // during indexing: + visitOneFormat(segmentSuffix, postingsFormat); + } + + // Read field name -> format name + final int fieldCount = in.readVInt(); + for(int fieldIDX=0;fieldIDX files) + throws IOException { + + final String mapFileName = IndexFileNames.segmentFileName(info.name, segmentSuffix, PER_FIELD_EXTENSION); + files.add(mapFileName); + + try { + new VisitPerFieldFile(dir, info.name, segmentSuffix) { + @Override + protected void visitOneFormat(String segmentSuffix, PostingsFormat format) throws IOException { + format.files(dir, info, segmentSuffix, files); + } + + @Override + protected void visitOneField(String field, PostingsFormat format) { + } + }; + } catch (FileNotFoundException fnfe) { + // TODO: this is somewhat shady... if we can't open + // the .per file then most likely someone is calling + // .files() after this segment was deleted, so, they + // wouldn't be able to do anything with the files even + // if we could return them, so we don't add any files + // in this case. + } + } + + // NOTE: only called during writing; for reading we read + // all we need from the index (ie we save the field -> + // format mapping) + public abstract PostingsFormat getPostingsFormatForField(String field); +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PostingsBaseFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsBaseFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PostingsBaseFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsBaseFormat.java 2011-11-03 13:24:53.432535280 -0400 @@ -0,0 +1,55 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.store.Directory; + +/** + * Provides a {@link PostingsReaderBase} and {@link + * PostingsWriterBase}. + * + * @lucene.experimental */ + +// TODO: find a better name; this defines the API that the +// terms dict impls use to talk to a postings impl. +// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer + +// can we clean this up and do this some other way? +// refactor some of these classes and use covariant return? +public abstract class PostingsBaseFormat { + + /** Unique name that's used to retrieve this codec when + * reading the index */ + public final String name; + + protected PostingsBaseFormat(String name) { + this.name = name; + } + + public abstract PostingsReaderBase postingsReaderBase(SegmentReadState state) throws IOException; + + public abstract PostingsWriterBase postingsWriterBase(SegmentWriteState state) throws IOException; + + public abstract void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException; +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsFormat.java 2011-11-02 21:48:19.160564228 -0400 @@ -0,0 +1,84 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.util.NamedSPILoader; + +import org.apache.lucene.store.Directory; + +/** @lucene.experimental */ +public abstract class PostingsFormat implements NamedSPILoader.NamedSPI { + + private static final NamedSPILoader loader = + new NamedSPILoader(PostingsFormat.class); + + public static final PostingsFormat[] EMPTY = new PostingsFormat[0]; + /** Unique name that's used to retrieve this format when + * reading the index. + */ + private final String name; + + protected PostingsFormat(String name) { + this.name = name; + } + + @Override + public String getName() { + return name; + } + + /** Writes a new segment */ + public abstract FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException; + + /** Reads a segment. NOTE: by the time this call + * returns, it must hold open any files it will need to + * use; else, those files may be deleted. */ + public abstract FieldsProducer fieldsProducer(SegmentReadState state) throws IOException; + + /** + * Gathers files associated with this segment + * + * @param dir the {@link Directory} this segment was written to + * @param segmentInfo the {@link SegmentInfo} for this segment + * @param segmentSuffix the format's suffix within this segment + * @param files the of files to add the codec files to. + */ + public abstract void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException; + + @Override + public String toString() { + return "PostingsFormat(name=" + name + ")"; + } + + /** looks up a format by name */ + public static PostingsFormat forName(String name) { + return loader.lookup(name); + } + + /** returns a list of all available format names */ + public static Set availablePostingsFormats() { + return loader.availableServices(); + } + +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java 2011-08-20 13:35:07.951460633 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java 2011-11-02 21:26:22.540564909 -0400 @@ -36,6 +36,9 @@ * time. * @lucene.experimental */ +// TODO: find a better name; this defines the API that the +// terms dict impls use to talk to a postings impl. +// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer public abstract class PostingsReaderBase implements Closeable { public abstract void init(IndexInput termsIn) throws IOException; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java 2011-08-20 13:35:07.951460633 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java 2011-11-02 21:26:22.544564907 -0400 @@ -27,6 +27,9 @@ * @lucene.experimental */ +// TODO: find a better name; this defines the API that the +// terms dict impls use to talk to a postings impl. +// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer public abstract class PostingsWriterBase extends PostingsConsumer implements Closeable { public abstract void start(IndexOutput termsOut) throws IOException; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java 2011-10-25 12:50:53.520936931 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,94 +0,0 @@ -package org.apache.lucene.index.codecs.preflex; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Set; -import java.io.IOException; - -import org.apache.lucene.store.Directory; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; - -/** Codec that reads the pre-flex-indexing postings - * format. It does not provide a writer because newly - * written segments should use StandardCodec. - * - * @deprecated (4.0) This is only used to read indexes created - * before 4.0. - * @lucene.experimental - */ -@Deprecated -public class PreFlexCodec extends Codec { - - /** Extension of terms file */ - public static final String TERMS_EXTENSION = "tis"; - - /** Extension of terms index file */ - public static final String TERMS_INDEX_EXTENSION = "tii"; - - /** Extension of freq postings file */ - public static final String FREQ_EXTENSION = "frq"; - - /** Extension of prox postings file */ - public static final String PROX_EXTENSION = "prx"; - - public PreFlexCodec() { - super("PreFlex"); - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - throw new IllegalArgumentException("this codec can only be used for reading"); - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor); - } - - @Override - public void files(Directory dir, SegmentInfo info, int id, Set files) throws IOException { - // preflex fields have no codec ID - we ignore it here - PreFlexFields.files(dir, info, files); - } - - @Override - public void getExtensions(Set extensions) { - extensions.add(FREQ_EXTENSION); - extensions.add(PROX_EXTENSION); - extensions.add(TERMS_EXTENSION); - extensions.add(TERMS_INDEX_EXTENSION); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return null; - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return null; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java 2011-10-25 12:50:53.520936931 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,1115 +0,0 @@ -package org.apache.lucene.index.codecs.preflex; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Collection; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.TreeMap; - -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.FieldsEnum; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.Terms; -import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.store.CompoundFileDirectory; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.UnicodeUtil; - -/** Exposes flex API on a pre-flex index, as a codec. - * @lucene.experimental - * @deprecated (4.0) - */ -@Deprecated -public class PreFlexFields extends FieldsProducer { - - private static final boolean DEBUG_SURROGATES = false; - - public TermInfosReader tis; - public final TermInfosReader tisNoIndex; - - public final IndexInput freqStream; - public final IndexInput proxStream; - final private FieldInfos fieldInfos; - private final SegmentInfo si; - final TreeMap fields = new TreeMap(); - final Map preTerms = new HashMap(); - private final Directory dir; - private final IOContext context; - private Directory cfsReader; - - public PreFlexFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, IOContext context, int indexDivisor) - throws IOException { - - si = info; - - // NOTE: we must always load terms index, even for - // "sequential" scan during merging, because what is - // sequential to merger may not be to TermInfosReader - // since we do the surrogates dance: - if (indexDivisor < 0) { - indexDivisor = -indexDivisor; - } - - boolean success = false; - try { - TermInfosReader r = new TermInfosReader(dir, info.name, fieldInfos, context, indexDivisor); - if (indexDivisor == -1) { - tisNoIndex = r; - } else { - tisNoIndex = null; - tis = r; - } - this.context = context; - this.fieldInfos = fieldInfos; - - // make sure that all index files have been read or are kept open - // so that if an index update removes them we'll still have them - freqStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.FREQ_EXTENSION), context); - boolean anyProx = false; - for (FieldInfo fi : fieldInfos) { - if (fi.isIndexed) { - fields.put(fi.name, fi); - preTerms.put(fi.name, new PreTerms(fi)); - if (fi.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { - anyProx = true; - } - } - } - - if (anyProx) { - proxStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.PROX_EXTENSION), context); - } else { - proxStream = null; - } - success = true; - } finally { - // With lock-less commits, it's entirely possible (and - // fine) to hit a FileNotFound exception above. In - // this case, we want to explicitly close any subset - // of things that were opened so that we don't have to - // wait for a GC to do so. - if (!success) { - close(); - } - } - this.dir = dir; - } - - // If this returns, we do the surrogates dance so that the - // terms are sorted by unicode sort order. This should be - // true when segments are used for "normal" searching; - // it's only false during testing, to create a pre-flex - // index, using the test-only PreFlexRW. - protected boolean sortTermsByUnicode() { - return true; - } - - static void files(Directory dir, SegmentInfo info, Collection files) throws IOException { - files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.TERMS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.TERMS_INDEX_EXTENSION)); - files.add(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.FREQ_EXTENSION)); - if (info.getHasProx()) { - // LUCENE-1739: for certain versions of 2.9-dev, - // hasProx would be incorrectly computed during - // indexing as true, and then stored into the segments - // file, when it should have been false. So we do the - // extra check, here: - final String prx = IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.PROX_EXTENSION); - if (dir.fileExists(prx)) { - files.add(prx); - } - } - } - - @Override - public FieldsEnum iterator() throws IOException { - return new PreFlexFieldsEnum(); - } - - @Override - public Terms terms(String field) { - return preTerms.get(field); - } - - @Override - public long getUniqueTermCount() throws IOException { - return getTermsDict().size(); - } - - synchronized private TermInfosReader getTermsDict() { - if (tis != null) { - return tis; - } else { - return tisNoIndex; - } - } - - @Override - public void close() throws IOException { - if (tis != null) { - tis.close(); - } - if (tisNoIndex != null) { - tisNoIndex.close(); - } - if (cfsReader != null) { - cfsReader.close(); - } - if (freqStream != null) { - freqStream.close(); - } - if (proxStream != null) { - proxStream.close(); - } - } - - private class PreFlexFieldsEnum extends FieldsEnum { - final Iterator it; - private final PreTermsEnum termsEnum; - FieldInfo current; - - public PreFlexFieldsEnum() throws IOException { - it = fields.values().iterator(); - termsEnum = new PreTermsEnum(); - } - - @Override - public String next() { - if (it.hasNext()) { - current = it.next(); - return current.name; - } else { - return null; - } - } - - @Override - public TermsEnum terms() throws IOException { - termsEnum.reset(current); - return termsEnum; - } - } - - private class PreTerms extends Terms { - final FieldInfo fieldInfo; - PreTerms(FieldInfo fieldInfo) { - this.fieldInfo = fieldInfo; - } - - @Override - public TermsEnum iterator() throws IOException { - PreTermsEnum termsEnum = new PreTermsEnum(); - termsEnum.reset(fieldInfo); - return termsEnum; - } - - @Override - public Comparator getComparator() { - // Pre-flex indexes always sorted in UTF16 order, but - // we remap on-the-fly to unicode order - if (sortTermsByUnicode()) { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } else { - return BytesRef.getUTF8SortedAsUTF16Comparator(); - } - } - - @Override - public long getUniqueTermCount() throws IOException { - return -1; - } - - @Override - public long getSumTotalTermFreq() { - return -1; - } - - @Override - public long getSumDocFreq() throws IOException { - return -1; - } - - @Override - public int getDocCount() throws IOException { - return -1; - } - } - - private class PreTermsEnum extends TermsEnum { - private SegmentTermEnum termEnum; - private FieldInfo fieldInfo; - private String internedFieldName; - private boolean skipNext; - private BytesRef current; - - private SegmentTermEnum seekTermEnum; - - private static final byte UTF8_NON_BMP_LEAD = (byte) 0xf0; - private static final byte UTF8_HIGH_BMP_LEAD = (byte) 0xee; - - // Returns true if the unicode char is "after" the - // surrogates in UTF16, ie >= U+E000 and <= U+FFFF: - private final boolean isHighBMPChar(byte[] b, int idx) { - return (b[idx] & UTF8_HIGH_BMP_LEAD) == UTF8_HIGH_BMP_LEAD; - } - - // Returns true if the unicode char in the UTF8 byte - // sequence starting at idx encodes a char outside of - // BMP (ie what would be a surrogate pair in UTF16): - private final boolean isNonBMPChar(byte[] b, int idx) { - return (b[idx] & UTF8_NON_BMP_LEAD) == UTF8_NON_BMP_LEAD; - } - - private final byte[] scratch = new byte[4]; - private final BytesRef prevTerm = new BytesRef(); - private final BytesRef scratchTerm = new BytesRef(); - private int newSuffixStart; - - // Swap in S, in place of E: - private boolean seekToNonBMP(SegmentTermEnum te, BytesRef term, int pos) throws IOException { - final int savLength = term.length; - - assert term.offset == 0; - - // The 3 bytes starting at downTo make up 1 - // unicode character: - assert isHighBMPChar(term.bytes, pos); - - // NOTE: we cannot make this assert, because - // AutomatonQuery legitimately sends us malformed UTF8 - // (eg the UTF8 bytes with just 0xee) - // assert term.length >= pos + 3: "term.length=" + term.length + " pos+3=" + (pos+3) + " byte=" + Integer.toHexString(term.bytes[pos]) + " term=" + term.toString(); - - // Save the bytes && length, since we need to - // restore this if seek "back" finds no matching - // terms - if (term.bytes.length < 4+pos) { - term.grow(4+pos); - } - - scratch[0] = term.bytes[pos]; - scratch[1] = term.bytes[pos+1]; - scratch[2] = term.bytes[pos+2]; - - term.bytes[pos] = (byte) 0xf0; - term.bytes[pos+1] = (byte) 0x90; - term.bytes[pos+2] = (byte) 0x80; - term.bytes[pos+3] = (byte) 0x80; - term.length = 4+pos; - - if (DEBUG_SURROGATES) { - System.out.println(" try seek term=" + UnicodeUtil.toHexString(term.utf8ToString())); - } - - // Seek "back": - getTermsDict().seekEnum(te, new Term(fieldInfo.name, term), true); - - // Test if the term we seek'd to in fact found a - // surrogate pair at the same position as the E: - Term t2 = te.term(); - - // Cannot be null (or move to next field) because at - // "worst" it'd seek to the same term we are on now, - // unless we are being called from seek - if (t2 == null || t2.field() != internedFieldName) { - return false; - } - - if (DEBUG_SURROGATES) { - System.out.println(" got term=" + UnicodeUtil.toHexString(t2.text())); - } - - // Now test if prefix is identical and we found - // a non-BMP char at the same position: - BytesRef b2 = t2.bytes(); - assert b2.offset == 0; - - boolean matches; - if (b2.length >= term.length && isNonBMPChar(b2.bytes, pos)) { - matches = true; - for(int i=0;i limit) { - - if (isHighBMPChar(prevTerm.bytes, downTo)) { - - if (DEBUG_SURROGATES) { - System.out.println(" found E pos=" + downTo + " vs len=" + prevTerm.length); - } - - if (seekToNonBMP(seekTermEnum, prevTerm, downTo)) { - // TODO: more efficient seek? - getTermsDict().seekEnum(termEnum, seekTermEnum.term(), true); - //newSuffixStart = downTo+4; - newSuffixStart = downTo; - scratchTerm.copy(termEnum.term().bytes()); - didSeek = true; - if (DEBUG_SURROGATES) { - System.out.println(" seek!"); - } - break; - } else { - if (DEBUG_SURROGATES) { - System.out.println(" no seek"); - } - } - } - - // Shorten prevTerm in place so that we don't redo - // this loop if we come back here: - if ((prevTerm.bytes[downTo] & 0xc0) == 0xc0 || (prevTerm.bytes[downTo] & 0x80) == 0) { - prevTerm.length = downTo; - } - - downTo--; - } - - return didSeek; - } - - // Look for seek type 3 ("pop"): if the delta from - // prev -> current was replacing an S with an E, - // we must now seek to beyond that E. This seek - // "finishes" the dance at this character - // position. - private boolean doPop() throws IOException { - - if (DEBUG_SURROGATES) { - System.out.println(" try pop"); - } - - assert newSuffixStart <= prevTerm.length; - assert newSuffixStart < scratchTerm.length || newSuffixStart == 0; - - if (prevTerm.length > newSuffixStart && - isNonBMPChar(prevTerm.bytes, newSuffixStart) && - isHighBMPChar(scratchTerm.bytes, newSuffixStart)) { - - // Seek type 2 -- put 0xFF at this position: - scratchTerm.bytes[newSuffixStart] = (byte) 0xff; - scratchTerm.length = newSuffixStart+1; - - if (DEBUG_SURROGATES) { - System.out.println(" seek to term=" + UnicodeUtil.toHexString(scratchTerm.utf8ToString()) + " " + scratchTerm.toString()); - } - - // TODO: more efficient seek? can we simply swap - // the enums? - getTermsDict().seekEnum(termEnum, new Term(fieldInfo.name, scratchTerm), true); - - final Term t2 = termEnum.term(); - - // We could hit EOF or different field since this - // was a seek "forward": - if (t2 != null && t2.field() == internedFieldName) { - - if (DEBUG_SURROGATES) { - System.out.println(" got term=" + UnicodeUtil.toHexString(t2.text()) + " " + t2.bytes()); - } - - final BytesRef b2 = t2.bytes(); - assert b2.offset == 0; - - - // Set newSuffixStart -- we can't use - // termEnum's since the above seek may have - // done no scanning (eg, term was precisely - // and index term, or, was in the term seek - // cache): - scratchTerm.copy(b2); - setNewSuffixStart(prevTerm, scratchTerm); - - return true; - } else if (newSuffixStart != 0 || scratchTerm.length != 0) { - if (DEBUG_SURROGATES) { - System.out.println(" got term=null (or next field)"); - } - newSuffixStart = 0; - scratchTerm.length = 0; - return true; - } - } - - return false; - } - - // Pre-flex indices store terms in UTF16 sort order, but - // certain queries require Unicode codepoint order; this - // method carefully seeks around surrogates to handle - // this impedance mismatch - - private void surrogateDance() throws IOException { - - if (!unicodeSortOrder) { - return; - } - - // We are invoked after TIS.next() (by UTF16 order) to - // possibly seek to a different "next" (by unicode - // order) term. - - // We scan only the "delta" from the last term to the - // current term, in UTF8 bytes. We look at 1) the bytes - // stripped from the prior term, and then 2) the bytes - // appended to that prior term's prefix. - - // We don't care about specific UTF8 sequences, just - // the "category" of the UTF16 character. Category S - // is a high/low surrogate pair (it non-BMP). - // Category E is any BMP char > UNI_SUR_LOW_END (and < - // U+FFFF). Category A is the rest (any unicode char - // <= UNI_SUR_HIGH_START). - - // The core issue is that pre-flex indices sort the - // characters as ASE, while flex must sort as AES. So - // when scanning, when we hit S, we must 1) seek - // forward to E and enum the terms there, then 2) seek - // back to S and enum all terms there, then 3) seek to - // after E. Three different seek points (1, 2, 3). - - // We can easily detect S in UTF8: if a byte has - // prefix 11110 (0xf0), then that byte and the - // following 3 bytes encode a single unicode codepoint - // in S. Similarly, we can detect E: if a byte has - // prefix 1110111 (0xee), then that byte and the - // following 2 bytes encode a single unicode codepoint - // in E. - - // Note that this is really a recursive process -- - // maybe the char at pos 2 needs to dance, but any - // point in its dance, suddenly pos 4 needs to dance - // so you must finish pos 4 before returning to pos - // 2. But then during pos 4's dance maybe pos 7 needs - // to dance, etc. However, despite being recursive, - // we don't need to hold any state because the state - // can always be derived by looking at prior term & - // current term. - - // TODO: can we avoid this copy? - if (termEnum.term() == null || termEnum.term().field() != internedFieldName) { - scratchTerm.length = 0; - } else { - scratchTerm.copy(termEnum.term().bytes()); - } - - if (DEBUG_SURROGATES) { - System.out.println(" dance"); - System.out.println(" prev=" + UnicodeUtil.toHexString(prevTerm.utf8ToString())); - System.out.println(" " + prevTerm.toString()); - System.out.println(" term=" + UnicodeUtil.toHexString(scratchTerm.utf8ToString())); - System.out.println(" " + scratchTerm.toString()); - } - - // This code assumes TermInfosReader/SegmentTermEnum - // always use BytesRef.offset == 0 - assert prevTerm.offset == 0; - assert scratchTerm.offset == 0; - - // Need to loop here because we may need to do multiple - // pops, and possibly a continue in the end, ie: - // - // cont - // pop, cont - // pop, pop, cont - // - // - - while(true) { - if (doContinue()) { - break; - } else { - if (!doPop()) { - break; - } - } - } - - if (DEBUG_SURROGATES) { - System.out.println(" finish bmp ends"); - } - - doPushes(); - } - - - // Look for seek type 1 ("push"): if the newly added - // suffix contains any S, we must try to seek to the - // corresponding E. If we find a match, we go there; - // else we keep looking for additional S's in the new - // suffix. This "starts" the dance, at this character - // position: - private void doPushes() throws IOException { - - int upTo = newSuffixStart; - if (DEBUG_SURROGATES) { - System.out.println(" try push newSuffixStart=" + newSuffixStart + " scratchLen=" + scratchTerm.length); - } - - while(upTo < scratchTerm.length) { - if (isNonBMPChar(scratchTerm.bytes, upTo) && - (upTo > newSuffixStart || - (upTo >= prevTerm.length || - (!isNonBMPChar(prevTerm.bytes, upTo) && - !isHighBMPChar(prevTerm.bytes, upTo))))) { - - // A non-BMP char (4 bytes UTF8) starts here: - assert scratchTerm.length >= upTo + 4; - - final int savLength = scratchTerm.length; - scratch[0] = scratchTerm.bytes[upTo]; - scratch[1] = scratchTerm.bytes[upTo+1]; - scratch[2] = scratchTerm.bytes[upTo+2]; - - scratchTerm.bytes[upTo] = UTF8_HIGH_BMP_LEAD; - scratchTerm.bytes[upTo+1] = (byte) 0x80; - scratchTerm.bytes[upTo+2] = (byte) 0x80; - scratchTerm.length = upTo+3; - - if (DEBUG_SURROGATES) { - System.out.println(" try seek 1 pos=" + upTo + " term=" + UnicodeUtil.toHexString(scratchTerm.utf8ToString()) + " " + scratchTerm.toString() + " len=" + scratchTerm.length); - } - - // Seek "forward": - // TODO: more efficient seek? - getTermsDict().seekEnum(seekTermEnum, new Term(fieldInfo.name, scratchTerm), true); - - scratchTerm.bytes[upTo] = scratch[0]; - scratchTerm.bytes[upTo+1] = scratch[1]; - scratchTerm.bytes[upTo+2] = scratch[2]; - scratchTerm.length = savLength; - - // Did we find a match? - final Term t2 = seekTermEnum.term(); - - if (DEBUG_SURROGATES) { - if (t2 == null) { - System.out.println(" hit term=null"); - } else { - System.out.println(" hit term=" + UnicodeUtil.toHexString(t2.text()) + " " + (t2==null? null:t2.bytes())); - } - } - - // Since this was a seek "forward", we could hit - // EOF or a different field: - boolean matches; - - if (t2 != null && t2.field() == internedFieldName) { - final BytesRef b2 = t2.bytes(); - assert b2.offset == 0; - if (b2.length >= upTo+3 && isHighBMPChar(b2.bytes, upTo)) { - matches = true; - for(int i=0;i BMP - upTo += 3; - - // NOTE: we keep iterating, now, since this - // can easily "recurse". Ie, after seeking - // forward at a certain char position, we may - // find another surrogate in our [new] suffix - // and must then do another seek (recurse) - } else { - upTo++; - } - } else { - upTo++; - } - } - } - - private boolean unicodeSortOrder; - - void reset(FieldInfo fieldInfo) throws IOException { - //System.out.println("pff.reset te=" + termEnum); - this.fieldInfo = fieldInfo; - internedFieldName = fieldInfo.name.intern(); - final Term term = new Term(internedFieldName); - if (termEnum == null) { - termEnum = getTermsDict().terms(term); - seekTermEnum = getTermsDict().terms(term); - //System.out.println(" term=" + termEnum.term()); - } else { - getTermsDict().seekEnum(termEnum, term, true); - } - skipNext = true; - - unicodeSortOrder = sortTermsByUnicode(); - - final Term t = termEnum.term(); - if (t != null && t.field() == internedFieldName) { - newSuffixStart = 0; - prevTerm.length = 0; - surrogateDance(); - } - } - - @Override - public Comparator getComparator() { - // Pre-flex indexes always sorted in UTF16 order, but - // we remap on-the-fly to unicode order - if (unicodeSortOrder) { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } else { - return BytesRef.getUTF8SortedAsUTF16Comparator(); - } - } - - @Override - public void seekExact(long ord) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public long ord() throws IOException { - throw new UnsupportedOperationException(); - } - - @Override - public SeekStatus seekCeil(BytesRef term, boolean useCache) throws IOException { - if (DEBUG_SURROGATES) { - System.out.println("TE.seek target=" + UnicodeUtil.toHexString(term.utf8ToString())); - } - skipNext = false; - final TermInfosReader tis = getTermsDict(); - final Term t0 = new Term(fieldInfo.name, term); - - assert termEnum != null; - - tis.seekEnum(termEnum, t0, useCache); - - final Term t = termEnum.term(); - - if (t != null && t.field() == internedFieldName && term.bytesEquals(t.bytes())) { - // If we found an exact match, no need to do the - // surrogate dance - if (DEBUG_SURROGATES) { - System.out.println(" seek exact match"); - } - current = t.bytes(); - return SeekStatus.FOUND; - } else if (t == null || t.field() != internedFieldName) { - - // TODO: maybe we can handle this like the next() - // into null? set term as prevTerm then dance? - - if (DEBUG_SURROGATES) { - System.out.println(" seek hit EOF"); - } - - // We hit EOF; try end-case surrogate dance: if we - // find an E, try swapping in S, backwards: - scratchTerm.copy(term); - - assert scratchTerm.offset == 0; - - for(int i=scratchTerm.length-1;i>=0;i--) { - if (isHighBMPChar(scratchTerm.bytes, i)) { - if (DEBUG_SURROGATES) { - System.out.println(" found E pos=" + i + "; try seek"); - } - - if (seekToNonBMP(seekTermEnum, scratchTerm, i)) { - - scratchTerm.copy(seekTermEnum.term().bytes()); - getTermsDict().seekEnum(termEnum, seekTermEnum.term(), useCache); - - newSuffixStart = 1+i; - - doPushes(); - - // Found a match - // TODO: faster seek? - current = termEnum.term().bytes(); - return SeekStatus.NOT_FOUND; - } - } - } - - if (DEBUG_SURROGATES) { - System.out.println(" seek END"); - } - - current = null; - return SeekStatus.END; - } else { - - // We found a non-exact but non-null term; this one - // is fun -- just treat it like next, by pretending - // requested term was prev: - prevTerm.copy(term); - - if (DEBUG_SURROGATES) { - System.out.println(" seek hit non-exact term=" + UnicodeUtil.toHexString(t.text())); - } - - final BytesRef br = t.bytes(); - assert br.offset == 0; - - setNewSuffixStart(term, br); - - surrogateDance(); - - final Term t2 = termEnum.term(); - if (t2 == null || t2.field() != internedFieldName) { - // PreFlex codec interns field names; verify: - assert t2 == null || !t2.field().equals(internedFieldName); - current = null; - return SeekStatus.END; - } else { - current = t2.bytes(); - assert !unicodeSortOrder || term.compareTo(current) < 0 : "term=" + UnicodeUtil.toHexString(term.utf8ToString()) + " vs current=" + UnicodeUtil.toHexString(current.utf8ToString()); - return SeekStatus.NOT_FOUND; - } - } - } - - private void setNewSuffixStart(BytesRef br1, BytesRef br2) { - final int limit = Math.min(br1.length, br2.length); - int lastStart = 0; - for(int i=0;i>> 1; // shift off low bit - if ((docCode & 1) != 0) // if low bit is set - freq = 1; // freq is one - else { - freq = freqStream.readVInt(); // else read freq - assert freq != 1; - } - } - - count++; - - if (liveDocs == null || liveDocs.get(doc)) { - break; - } - skippingDoc(); - } - return true; - } - - /** Optimized implementation. */ - public int read(final int[] docs, final int[] freqs) - throws IOException { - final int length = docs.length; - if (indexOptions == IndexOptions.DOCS_ONLY) { - return readNoTf(docs, freqs, length); - } else { - int i = 0; - while (i < length && count < df) { - // manually inlined call to next() for speed - final int docCode = freqStream.readVInt(); - doc += docCode >>> 1; // shift off low bit - if ((docCode & 1) != 0) // if low bit is set - freq = 1; // freq is one - else - freq = freqStream.readVInt(); // else read freq - count++; - - if (liveDocs == null || liveDocs.get(doc)) { - docs[i] = doc; - freqs[i] = freq; - ++i; - } - } - return i; - } - } - - private final int readNoTf(final int[] docs, final int[] freqs, final int length) throws IOException { - int i = 0; - while (i < length && count < df) { - // manually inlined call to next() for speed - doc += freqStream.readVInt(); - count++; - - if (liveDocs == null || liveDocs.get(doc)) { - docs[i] = doc; - // Hardware freq to 1 when term freqs were not - // stored in the index - freqs[i] = 1; - ++i; - } - } - return i; - } - - - /** Overridden by SegmentTermPositions to skip in prox stream. */ - protected void skipProx(long proxPointer, int payloadLength) throws IOException {} - - /** Optimized implementation. */ - public boolean skipTo(int target) throws IOException { - // don't skip if the target is close (within skipInterval docs away) - if ((target - skipInterval) >= doc && df >= skipInterval) { // optimized case - if (skipListReader == null) - skipListReader = new DefaultSkipListReader((IndexInput) freqStream.clone(), maxSkipLevels, skipInterval); // lazily clone - - if (!haveSkipped) { // lazily initialize skip stream - skipListReader.init(skipPointer, freqBasePointer, proxBasePointer, df, currentFieldStoresPayloads); - haveSkipped = true; - } - - int newCount = skipListReader.skipTo(target); - if (newCount > count) { - freqStream.seek(skipListReader.getFreqPointer()); - skipProx(skipListReader.getProxPointer(), skipListReader.getPayloadLength()); - - doc = skipListReader.getDoc(); - count = newCount; - } - } - - // done skipping, now just scan - do { - if (!next()) - return false; - } while (target > doc); - return true; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java 2011-10-24 12:30:31.656982074 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,225 +0,0 @@ -package org.apache.lucene.index.codecs.preflex; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.IndexFormatTooNewException; - -/** - * @deprecated (4.0) No longer used with flex indexing, except for - * reading old segments - * @lucene.experimental */ - -@Deprecated -public final class SegmentTermEnum implements Cloneable { - private IndexInput input; - FieldInfos fieldInfos; - long size; - long position = -1; - - // Changed strings to true utf8 with length-in-bytes not - // length-in-chars - public static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = -4; - - // NOTE: always change this if you switch to a new format! - // whenever you add a new format, make it 1 smaller (negative version logic)! - public static final int FORMAT_CURRENT = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; - - // when removing support for old versions, leave the last supported version here - public static final int FORMAT_MINIMUM = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; - - private TermBuffer termBuffer = new TermBuffer(); - private TermBuffer prevBuffer = new TermBuffer(); - private TermBuffer scanBuffer = new TermBuffer(); // used for scanning - - TermInfo termInfo = new TermInfo(); - - private int format; - private boolean isIndex = false; - long indexPointer = 0; - int indexInterval; - int skipInterval; - int newSuffixStart; - int maxSkipLevels; - private boolean first = true; - - SegmentTermEnum(IndexInput i, FieldInfos fis, boolean isi) - throws CorruptIndexException, IOException { - input = i; - fieldInfos = fis; - isIndex = isi; - maxSkipLevels = 1; // use single-level skip lists for formats > -3 - - int firstInt = input.readInt(); - if (firstInt >= 0) { - // original-format file, without explicit format version number - format = 0; - size = firstInt; - - // back-compatible settings - indexInterval = 128; - skipInterval = Integer.MAX_VALUE; // switch off skipTo optimization - } else { - // we have a format version number - format = firstInt; - - // check that it is a format we can understand - if (format > FORMAT_MINIMUM) - throw new IndexFormatTooOldException(null, format, FORMAT_MINIMUM, FORMAT_CURRENT); - if (format < FORMAT_CURRENT) - throw new IndexFormatTooNewException(null, format, FORMAT_MINIMUM, FORMAT_CURRENT); - - size = input.readLong(); // read the size - - indexInterval = input.readInt(); - skipInterval = input.readInt(); - maxSkipLevels = input.readInt(); - assert indexInterval > 0: "indexInterval=" + indexInterval + " is negative; must be > 0"; - assert skipInterval > 0: "skipInterval=" + skipInterval + " is negative; must be > 0"; - } - } - - @Override - protected Object clone() { - SegmentTermEnum clone = null; - try { - clone = (SegmentTermEnum) super.clone(); - } catch (CloneNotSupportedException e) {} - - clone.input = (IndexInput) input.clone(); - clone.termInfo = new TermInfo(termInfo); - - clone.termBuffer = (TermBuffer)termBuffer.clone(); - clone.prevBuffer = (TermBuffer)prevBuffer.clone(); - clone.scanBuffer = new TermBuffer(); - - return clone; - } - - final void seek(long pointer, long p, Term t, TermInfo ti) - throws IOException { - input.seek(pointer); - position = p; - termBuffer.set(t); - prevBuffer.reset(); - //System.out.println(" ste doSeek prev=" + prevBuffer.toTerm() + " this=" + this); - termInfo.set(ti); - first = p == -1; - } - - /** Increments the enumeration to the next element. True if one exists.*/ - public final boolean next() throws IOException { - prevBuffer.set(termBuffer); - //System.out.println(" ste setPrev=" + prev() + " this=" + this); - - if (position++ >= size - 1) { - termBuffer.reset(); - //System.out.println(" EOF"); - return false; - } - - termBuffer.read(input, fieldInfos); - newSuffixStart = termBuffer.newSuffixStart; - - termInfo.docFreq = input.readVInt(); // read doc freq - termInfo.freqPointer += input.readVLong(); // read freq pointer - termInfo.proxPointer += input.readVLong(); // read prox pointer - - if (termInfo.docFreq >= skipInterval) - termInfo.skipOffset = input.readVInt(); - - if (isIndex) - indexPointer += input.readVLong(); // read index pointer - - //System.out.println(" ste ret term=" + term()); - return true; - } - - /* Optimized scan, without allocating new terms. - * Return number of invocations to next(). - * - * NOTE: LUCENE-3183: if you pass Term("", "") here then this - * will incorrectly return before positioning the enum, - * and position will be -1; caller must detect this. */ - final int scanTo(Term term) throws IOException { - scanBuffer.set(term); - int count = 0; - if (first) { - // Always force initial next() in case term is - // Term("", "") - next(); - first = false; - count++; - } - while (scanBuffer.compareTo(termBuffer) > 0 && next()) { - count++; - } - return count; - } - - /** Returns the current Term in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - public final Term term() { - return termBuffer.toTerm(); - } - - /** Returns the previous Term enumerated. Initially null.*/ - final Term prev() { - return prevBuffer.toTerm(); - } - - /** Returns the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final TermInfo termInfo() { - return new TermInfo(termInfo); - } - - /** Sets the argument to the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final void termInfo(TermInfo ti) { - ti.set(termInfo); - } - - /** Returns the docFreq from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - public final int docFreq() { - return termInfo.docFreq; - } - - /* Returns the freqPointer from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final long freqPointer() { - return termInfo.freqPointer; - } - - /* Returns the proxPointer from the current TermInfo in the enumeration. - Initially invalid, valid after next() called for the first time.*/ - final long proxPointer() { - return termInfo.proxPointer; - } - - /** Closes the enumeration to further activity, freeing resources. */ - public final void close() throws IOException { - input.close(); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java 2011-07-17 06:27:33.818948135 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,219 +0,0 @@ -package org.apache.lucene.index.codecs.preflex; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.store.IndexInput; - -/** - * @lucene.experimental - * @deprecated (4.0) - */ -@Deprecated -public final class SegmentTermPositions -extends SegmentTermDocs { - private IndexInput proxStream; - private IndexInput proxStreamOrig; - private int proxCount; - private int position; - - // the current payload length - private int payloadLength; - // indicates whether the payload of the current position has - // been read from the proxStream yet - private boolean needToLoadPayload; - - // these variables are being used to remember information - // for a lazy skip - private long lazySkipPointer = -1; - private int lazySkipProxCount = 0; - - /* - SegmentTermPositions(SegmentReader p) { - super(p); - this.proxStream = null; // the proxStream will be cloned lazily when nextPosition() is called for the first time - } - */ - - public SegmentTermPositions(IndexInput freqStream, IndexInput proxStream, TermInfosReader tis, FieldInfos fieldInfos) { - super(freqStream, tis, fieldInfos); - this.proxStreamOrig = proxStream; // the proxStream will be cloned lazily when nextPosition() is called for the first time - } - - @Override - final void seek(TermInfo ti, Term term) throws IOException { - super.seek(ti, term); - if (ti != null) - lazySkipPointer = ti.proxPointer; - - lazySkipProxCount = 0; - proxCount = 0; - payloadLength = 0; - needToLoadPayload = false; - } - - @Override - public final void close() throws IOException { - super.close(); - if (proxStream != null) proxStream.close(); - } - - public final int nextPosition() throws IOException { - if (indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) - // This field does not store positions, payloads - return 0; - // perform lazy skips if necessary - lazySkip(); - proxCount--; - return position += readDeltaPosition(); - } - - private final int readDeltaPosition() throws IOException { - int delta = proxStream.readVInt(); - if (currentFieldStoresPayloads) { - // if the current field stores payloads then - // the position delta is shifted one bit to the left. - // if the LSB is set, then we have to read the current - // payload length - if ((delta & 1) != 0) { - payloadLength = proxStream.readVInt(); - } - delta >>>= 1; - needToLoadPayload = true; - } - return delta; - } - - @Override - protected final void skippingDoc() throws IOException { - // we remember to skip a document lazily - lazySkipProxCount += freq; - } - - @Override - public final boolean next() throws IOException { - // we remember to skip the remaining positions of the current - // document lazily - lazySkipProxCount += proxCount; - - if (super.next()) { // run super - proxCount = freq; // note frequency - position = 0; // reset position - return true; - } - return false; - } - - @Override - public final int read(final int[] docs, final int[] freqs) { - throw new UnsupportedOperationException("TermPositions does not support processing multiple documents in one call. Use TermDocs instead."); - } - - - /** Called by super.skipTo(). */ - @Override - protected void skipProx(long proxPointer, int payloadLength) throws IOException { - // we save the pointer, we might have to skip there lazily - lazySkipPointer = proxPointer; - lazySkipProxCount = 0; - proxCount = 0; - this.payloadLength = payloadLength; - needToLoadPayload = false; - } - - private void skipPositions(int n) throws IOException { - assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - for (int f = n; f > 0; f--) { // skip unread positions - readDeltaPosition(); - skipPayload(); - } - } - - private void skipPayload() throws IOException { - if (needToLoadPayload && payloadLength > 0) { - proxStream.seek(proxStream.getFilePointer() + payloadLength); - } - needToLoadPayload = false; - } - - // It is not always necessary to move the prox pointer - // to a new document after the freq pointer has been moved. - // Consider for example a phrase query with two terms: - // the freq pointer for term 1 has to move to document x - // to answer the question if the term occurs in that document. But - // only if term 2 also matches document x, the positions have to be - // read to figure out if term 1 and term 2 appear next - // to each other in document x and thus satisfy the query. - // So we move the prox pointer lazily to the document - // as soon as positions are requested. - private void lazySkip() throws IOException { - if (proxStream == null) { - // clone lazily - proxStream = (IndexInput)proxStreamOrig.clone(); - } - - // we might have to skip the current payload - // if it was not read yet - skipPayload(); - - if (lazySkipPointer != -1) { - proxStream.seek(lazySkipPointer); - lazySkipPointer = -1; - } - - if (lazySkipProxCount != 0) { - skipPositions(lazySkipProxCount); - lazySkipProxCount = 0; - } - } - - public int getPayloadLength() { - return payloadLength; - } - - public byte[] getPayload(byte[] data, int offset) throws IOException { - if (!needToLoadPayload) { - throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once."); - } - - // read payloads lazily - byte[] retArray; - int retOffset; - if (data == null || data.length - offset < payloadLength) { - // the array is too small to store the payload data, - // so we allocate a new one - retArray = new byte[payloadLength]; - retOffset = 0; - } else { - retArray = data; - retOffset = offset; - } - proxStream.readBytes(retArray, retOffset, payloadLength); - needToLoadPayload = false; - return retArray; - } - - public boolean isPayloadAvailable() { - return needToLoadPayload && payloadLength > 0; - } - -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java 2011-06-22 06:11:16.131822179 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,122 +0,0 @@ -package org.apache.lucene.index.codecs.preflex; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Comparator; - -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.FieldInfos; - -/** - * @lucene.experimental - * @deprecated (4.0) - */ -@Deprecated -final class TermBuffer implements Cloneable { - - private String field; - private Term term; // cached - - private BytesRef bytes = new BytesRef(10); - - // Cannot be -1 since (strangely) we write that - // fieldNumber into index for first indexed term: - private int currentFieldNumber = -2; - - private static final Comparator utf8AsUTF16Comparator = BytesRef.getUTF8SortedAsUTF16Comparator(); - - int newSuffixStart; // only valid right after .read is called - - public int compareTo(TermBuffer other) { - if (field == other.field) // fields are interned - // (only by PreFlex codec) - return utf8AsUTF16Comparator.compare(bytes, other.bytes); - else - return field.compareTo(other.field); - } - - public void read(IndexInput input, FieldInfos fieldInfos) - throws IOException { - this.term = null; // invalidate cache - newSuffixStart = input.readVInt(); - int length = input.readVInt(); - int totalLength = newSuffixStart + length; - if (bytes.bytes.length < totalLength) { - bytes.grow(totalLength); - } - bytes.length = totalLength; - input.readBytes(bytes.bytes, newSuffixStart, length); - final int fieldNumber = input.readVInt(); - if (fieldNumber != currentFieldNumber) { - currentFieldNumber = fieldNumber; - field = fieldInfos.fieldName(currentFieldNumber).intern(); - } else { - assert field.equals(fieldInfos.fieldName(fieldNumber)): "currentFieldNumber=" + currentFieldNumber + " field=" + field + " vs " + fieldInfos.fieldName(fieldNumber); - } - } - - public void set(Term term) { - if (term == null) { - reset(); - return; - } - bytes.copy(term.bytes()); - field = term.field().intern(); - currentFieldNumber = -1; - this.term = term; - } - - public void set(TermBuffer other) { - field = other.field; - currentFieldNumber = other.currentFieldNumber; - // dangerous to copy Term over, since the underlying - // BytesRef could subsequently be modified: - term = null; - bytes.copy(other.bytes); - } - - public void reset() { - field = null; - term = null; - currentFieldNumber= -1; - } - - public Term toTerm() { - if (field == null) // unset - return null; - - if (term == null) { - term = new Term(field, new BytesRef(bytes)); - } - - return term; - } - - @Override - protected Object clone() { - TermBuffer clone = null; - try { - clone = (TermBuffer)super.clone(); - } catch (CloneNotSupportedException e) {} - clone.bytes = new BytesRef(bytes); - return clone; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java 2011-05-08 10:52:19.620078710 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,63 +0,0 @@ -package org.apache.lucene.index.codecs.preflex; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** A TermInfo is the record of information stored for a - * term - * @deprecated (4.0) This class is no longer used in flexible - * indexing. */ - -@Deprecated -public class TermInfo { - /** The number of documents which contain the term. */ - public int docFreq = 0; - - public long freqPointer = 0; - public long proxPointer = 0; - public int skipOffset; - - public TermInfo() {} - - public TermInfo(int df, long fp, long pp) { - docFreq = df; - freqPointer = fp; - proxPointer = pp; - } - - public TermInfo(TermInfo ti) { - docFreq = ti.docFreq; - freqPointer = ti.freqPointer; - proxPointer = ti.proxPointer; - skipOffset = ti.skipOffset; - } - - public final void set(int docFreq, - long freqPointer, long proxPointer, int skipOffset) { - this.docFreq = docFreq; - this.freqPointer = freqPointer; - this.proxPointer = proxPointer; - this.skipOffset = skipOffset; - } - - public final void set(TermInfo ti) { - docFreq = ti.docFreq; - freqPointer = ti.freqPointer; - proxPointer = ti.proxPointer; - skipOffset = ti.skipOffset; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java 2011-10-27 17:03:30.968840102 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,252 +0,0 @@ -package org.apache.lucene.index.codecs.preflex; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.List; - -import org.apache.lucene.index.Term; -import org.apache.lucene.util.BitUtil; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.PagedBytes.PagedBytesDataInput; -import org.apache.lucene.util.PagedBytes.PagedBytesDataOutput; -import org.apache.lucene.util.PagedBytes; -import org.apache.lucene.util.packed.GrowableWriter; -import org.apache.lucene.util.packed.PackedInts; - -/** - * This stores a monotonically increasing set of pairs in an - * index segment. Pairs are accessed either by Term or by ordinal position the - * set. The Terms and TermInfo are actually serialized and stored into a byte - * array and pointers to the position of each are stored in a int array. - */ -class TermInfosReaderIndex { - - private static final int MAX_PAGE_BITS = 18; // 256 KB block - private Term[] fields; - private int totalIndexInterval; - private Comparator comparator = BytesRef.getUTF8SortedAsUTF16Comparator(); - private final PagedBytesDataInput dataInput; - private final PackedInts.Reader indexToDataOffset; - private final int indexSize; - private final int skipInterval; - - /** - * Loads the segment information at segment load time. - * - * @param indexEnum - * the term enum. - * @param indexDivisor - * the index divisor. - * @param tiiFileLength - * the size of the tii file, used to approximate the size of the - * buffer. - * @param totalIndexInterval - * the total index interval. - */ - TermInfosReaderIndex(SegmentTermEnum indexEnum, int indexDivisor, long tiiFileLength, int totalIndexInterval) throws IOException { - this.totalIndexInterval = totalIndexInterval; - indexSize = 1 + ((int) indexEnum.size - 1) / indexDivisor; - skipInterval = indexEnum.skipInterval; - // this is only an inital size, it will be GCed once the build is complete - long initialSize = (long) (tiiFileLength * 1.5) / indexDivisor; - PagedBytes dataPagedBytes = new PagedBytes(estimatePageBits(initialSize)); - PagedBytesDataOutput dataOutput = dataPagedBytes.getDataOutput(); - - GrowableWriter indexToTerms = new GrowableWriter(4, indexSize, false); - String currentField = null; - List fieldStrs = new ArrayList(); - int fieldCounter = -1; - for (int i = 0; indexEnum.next(); i++) { - Term term = indexEnum.term(); - if (currentField == null || !currentField.equals(term.field())) { - currentField = term.field(); - fieldStrs.add(currentField); - fieldCounter++; - } - TermInfo termInfo = indexEnum.termInfo(); - indexToTerms.set(i, dataOutput.getPosition()); - dataOutput.writeVInt(fieldCounter); - dataOutput.writeString(term.text()); - dataOutput.writeVInt(termInfo.docFreq); - if (termInfo.docFreq >= skipInterval) { - dataOutput.writeVInt(termInfo.skipOffset); - } - dataOutput.writeVLong(termInfo.freqPointer); - dataOutput.writeVLong(termInfo.proxPointer); - dataOutput.writeVLong(indexEnum.indexPointer); - for (int j = 1; j < indexDivisor; j++) { - if (!indexEnum.next()) { - break; - } - } - } - - fields = new Term[fieldStrs.size()]; - for (int i = 0; i < fields.length; i++) { - fields[i] = new Term(fieldStrs.get(i)); - } - - dataPagedBytes.freeze(true); - dataInput = dataPagedBytes.getDataInput(); - indexToDataOffset = indexToTerms.getMutable(); - } - - private static int estimatePageBits(long estSize) { - return Math.max(Math.min(64 - BitUtil.nlz(estSize), MAX_PAGE_BITS), 4); - } - - void seekEnum(SegmentTermEnum enumerator, int indexOffset) throws IOException { - PagedBytesDataInput input = (PagedBytesDataInput) dataInput.clone(); - - input.setPosition(indexToDataOffset.get(indexOffset)); - - // read the term - int fieldId = input.readVInt(); - Term field = fields[fieldId]; - Term term = new Term(field.field(), input.readString()); - - // read the terminfo - TermInfo termInfo = new TermInfo(); - termInfo.docFreq = input.readVInt(); - if (termInfo.docFreq >= skipInterval) { - termInfo.skipOffset = input.readVInt(); - } else { - termInfo.skipOffset = 0; - } - termInfo.freqPointer = input.readVLong(); - termInfo.proxPointer = input.readVLong(); - - long pointer = input.readVLong(); - - // perform the seek - enumerator.seek(pointer, ((long) indexOffset * totalIndexInterval) - 1, term, termInfo); - } - - /** - * Binary search for the given term. - * - * @param term - * the term to locate. - * @throws IOException - */ - int getIndexOffset(Term term) throws IOException { - int lo = 0; - int hi = indexSize - 1; - PagedBytesDataInput input = (PagedBytesDataInput) dataInput.clone(); - BytesRef scratch = new BytesRef(); - while (hi >= lo) { - int mid = (lo + hi) >>> 1; - int delta = compareTo(term, mid, input, scratch); - if (delta < 0) - hi = mid - 1; - else if (delta > 0) - lo = mid + 1; - else - return mid; - } - return hi; - } - - /** - * Gets the term at the given position. For testing. - * - * @param termIndex - * the position to read the term from the index. - * @return the term. - * @throws IOException - */ - Term getTerm(int termIndex) throws IOException { - PagedBytesDataInput input = (PagedBytesDataInput) dataInput.clone(); - input.setPosition(indexToDataOffset.get(termIndex)); - - // read the term - int fieldId = input.readVInt(); - Term field = fields[fieldId]; - return new Term(field.field(), input.readString()); - } - - /** - * Returns the number of terms. - * - * @return int. - */ - int length() { - return indexSize; - } - - /** - * The compares the given term against the term in the index specified by the - * term index. ie It returns negative N when term is less than index term; - * - * @param term - * the given term. - * @param termIndex - * the index of the of term to compare. - * @return int. - * @throws IOException - */ - int compareTo(Term term, int termIndex) throws IOException { - return compareTo(term, termIndex, (PagedBytesDataInput) dataInput.clone(), new BytesRef()); - } - - /** - * Compare the fields of the terms first, and if not equals return from - * compare. If equal compare terms. - * - * @param term - * the term to compare. - * @param termIndex - * the position of the term in the input to compare - * @param input - * the input buffer. - * @return int. - * @throws IOException - */ - private int compareTo(Term term, int termIndex, PagedBytesDataInput input, BytesRef reuse) throws IOException { - // if term field does not equal mid's field index, then compare fields - // else if they are equal, compare term's string values... - int c = compareField(term, termIndex, input); - if (c == 0) { - reuse.length = input.readVInt(); - reuse.grow(reuse.length); - input.readBytes(reuse.bytes, 0, reuse.length); - return comparator.compare(term.bytes(), reuse); - } - return c; - } - - /** - * Compares the fields before checking the text of the terms. - * - * @param term - * the given term. - * @param termIndex - * the term that exists in the data block. - * @param input - * the data block. - * @return int. - * @throws IOException - */ - private int compareField(Term term, int termIndex, PagedBytesDataInput input) throws IOException { - input.setPosition(indexToDataOffset.get(termIndex)); - return term.field().compareTo(fields[input.readVInt()].field()); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java 2011-10-28 10:32:09.324807688 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,347 +0,0 @@ -package org.apache.lucene.index.codecs.preflex; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Comparator; - -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.Term; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CloseableThreadLocal; -import org.apache.lucene.util.DoubleBarrelLRUCache; - -/** This stores a monotonically increasing set of pairs in a - * Directory. Pairs are accessed either by Term or by ordinal position the - * set - * @deprecated (4.0) This class has been replaced by - * FormatPostingsTermsDictReader, except for reading old segments. - * @lucene.experimental - */ -@Deprecated -public final class TermInfosReader { - private final Directory directory; - private final String segment; - private final FieldInfos fieldInfos; - - private final CloseableThreadLocal threadResources = new CloseableThreadLocal(); - private final SegmentTermEnum origEnum; - private final long size; - - private final TermInfosReaderIndex index; - private final int indexLength; - - private final int totalIndexInterval; - - private final static int DEFAULT_CACHE_SIZE = 1024; - - // Just adds term's ord to TermInfo - private final static class TermInfoAndOrd extends TermInfo { - final long termOrd; - public TermInfoAndOrd(TermInfo ti, long termOrd) { - super(ti); - assert termOrd >= 0; - this.termOrd = termOrd; - } - } - - private static class CloneableTerm extends DoubleBarrelLRUCache.CloneableKey { - Term term; - public CloneableTerm(Term t) { - this.term = t; - } - - @Override - public boolean equals(Object other) { - CloneableTerm t = (CloneableTerm) other; - return this.term.equals(t.term); - } - - @Override - public int hashCode() { - return term.hashCode(); - } - - @Override - public Object clone() { - return new CloneableTerm(term); - } - } - - private final DoubleBarrelLRUCache termsCache = new DoubleBarrelLRUCache(DEFAULT_CACHE_SIZE); - - /** - * Per-thread resources managed by ThreadLocal - */ - private static final class ThreadResources { - SegmentTermEnum termEnum; - } - - TermInfosReader(Directory dir, String seg, FieldInfos fis, IOContext context, int indexDivisor) - throws CorruptIndexException, IOException { - boolean success = false; - - if (indexDivisor < 1 && indexDivisor != -1) { - throw new IllegalArgumentException("indexDivisor must be -1 (don't load terms index) or greater than 0: got " + indexDivisor); - } - - try { - directory = dir; - segment = seg; - fieldInfos = fis; - - origEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_EXTENSION), - context), fieldInfos, false); - size = origEnum.size; - - - if (indexDivisor != -1) { - // Load terms index - totalIndexInterval = origEnum.indexInterval * indexDivisor; - - final String indexFileName = IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION); - final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(indexFileName, - context), fieldInfos, true); - - try { - index = new TermInfosReaderIndex(indexEnum, indexDivisor, dir.fileLength(indexFileName), totalIndexInterval); - indexLength = index.length(); - } finally { - indexEnum.close(); - } - } else { - // Do not load terms index: - totalIndexInterval = -1; - index = null; - indexLength = -1; - } - success = true; - } finally { - // With lock-less commits, it's entirely possible (and - // fine) to hit a FileNotFound exception above. In - // this case, we want to explicitly close any subset - // of things that were opened so that we don't have to - // wait for a GC to do so. - if (!success) { - close(); - } - } - } - - public int getSkipInterval() { - return origEnum.skipInterval; - } - - public int getMaxSkipLevels() { - return origEnum.maxSkipLevels; - } - - void close() throws IOException { - if (origEnum != null) - origEnum.close(); - threadResources.close(); - } - - /** Returns the number of term/value pairs in the set. */ - long size() { - return size; - } - - private ThreadResources getThreadResources() { - ThreadResources resources = threadResources.get(); - if (resources == null) { - resources = new ThreadResources(); - resources.termEnum = terms(); - threadResources.set(resources); - } - return resources; - } - - private static final Comparator legacyComparator = - BytesRef.getUTF8SortedAsUTF16Comparator(); - - private final int compareAsUTF16(Term term1, Term term2) { - if (term1.field().equals(term2.field())) { - return legacyComparator.compare(term1.bytes(), term2.bytes()); - } else { - return term1.field().compareTo(term2.field()); - } - } - - /** Returns the TermInfo for a Term in the set, or null. */ - TermInfo get(Term term) throws IOException { - return get(term, false); - } - - /** Returns the TermInfo for a Term in the set, or null. */ - private TermInfo get(Term term, boolean mustSeekEnum) throws IOException { - if (size == 0) return null; - - ensureIndexIsRead(); - TermInfoAndOrd tiOrd = termsCache.get(new CloneableTerm(term)); - ThreadResources resources = getThreadResources(); - - if (!mustSeekEnum && tiOrd != null) { - return tiOrd; - } - - return seekEnum(resources.termEnum, term, tiOrd, true); - } - - public void cacheCurrentTerm(SegmentTermEnum enumerator) { - termsCache.put(new CloneableTerm(enumerator.term()), - new TermInfoAndOrd(enumerator.termInfo, - enumerator.position)); - } - - TermInfo seekEnum(SegmentTermEnum enumerator, Term term, boolean useCache) throws IOException { - if (useCache) { - return seekEnum(enumerator, term, termsCache.get(new CloneableTerm(term)), useCache); - } else { - return seekEnum(enumerator, term, null, useCache); - } - } - - TermInfo seekEnum(SegmentTermEnum enumerator, Term term, TermInfoAndOrd tiOrd, boolean useCache) throws IOException { - if (size == 0) { - return null; - } - - // optimize sequential access: first try scanning cached enum w/o seeking - if (enumerator.term() != null // term is at or past current - && ((enumerator.prev() != null && compareAsUTF16(term, enumerator.prev())> 0) - || compareAsUTF16(term, enumerator.term()) >= 0)) { - int enumOffset = (int)(enumerator.position/totalIndexInterval)+1; - if (indexLength == enumOffset // but before end of block - || index.compareTo(term, enumOffset) < 0) { - // no need to seek - - final TermInfo ti; - int numScans = enumerator.scanTo(term); - if (enumerator.term() != null && compareAsUTF16(term, enumerator.term()) == 0) { - ti = enumerator.termInfo; - if (numScans > 1) { - // we only want to put this TermInfo into the cache if - // scanEnum skipped more than one dictionary entry. - // This prevents RangeQueries or WildcardQueries to - // wipe out the cache when they iterate over a large numbers - // of terms in order - if (tiOrd == null) { - if (useCache) { - termsCache.put(new CloneableTerm(term), new TermInfoAndOrd(ti, enumerator.position)); - } - } else { - assert sameTermInfo(ti, tiOrd, enumerator); - assert (int) enumerator.position == tiOrd.termOrd; - } - } - } else { - ti = null; - } - - return ti; - } - } - - // random-access: must seek - final int indexPos; - if (tiOrd != null) { - indexPos = (int) (tiOrd.termOrd / totalIndexInterval); - } else { - // Must do binary search: - indexPos = index.getIndexOffset(term); - } - - index.seekEnum(enumerator, indexPos); - enumerator.scanTo(term); - final TermInfo ti; - - if (enumerator.term() != null && compareAsUTF16(term, enumerator.term()) == 0) { - ti = enumerator.termInfo; - if (tiOrd == null) { - if (useCache) { - termsCache.put(new CloneableTerm(term), new TermInfoAndOrd(ti, enumerator.position)); - } - } else { - assert sameTermInfo(ti, tiOrd, enumerator); - assert enumerator.position == tiOrd.termOrd; - } - } else { - ti = null; - } - return ti; - } - - // called only from asserts - private boolean sameTermInfo(TermInfo ti1, TermInfo ti2, SegmentTermEnum enumerator) { - if (ti1.docFreq != ti2.docFreq) { - return false; - } - if (ti1.freqPointer != ti2.freqPointer) { - return false; - } - if (ti1.proxPointer != ti2.proxPointer) { - return false; - } - // skipOffset is only valid when docFreq >= skipInterval: - if (ti1.docFreq >= enumerator.skipInterval && - ti1.skipOffset != ti2.skipOffset) { - return false; - } - return true; - } - - private void ensureIndexIsRead() { - if (index == null) { - throw new IllegalStateException("terms index was not loaded when this reader was created"); - } - } - - /** Returns the position of a Term in the set or -1. */ - long getPosition(Term term) throws IOException { - if (size == 0) return -1; - - ensureIndexIsRead(); - int indexOffset = index.getIndexOffset(term); - - SegmentTermEnum enumerator = getThreadResources().termEnum; - index.seekEnum(enumerator, indexOffset); - - while(compareAsUTF16(term, enumerator.term()) > 0 && enumerator.next()) {} - - if (compareAsUTF16(term, enumerator.term()) == 0) - return enumerator.position; - else - return -1; - } - - /** Returns an enumeration of all the Terms and TermInfos in the set. */ - public SegmentTermEnum terms() { - return (SegmentTermEnum)origEnum.clone(); - } - - /** Returns an enumeration of terms starting at or after the named term. */ - public SegmentTermEnum terms(Term term) throws IOException { - get(term, true); - return (SegmentTermEnum)getThreadResources().termEnum.clone(); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/pulsing/Pulsing40PostingsFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/pulsing/Pulsing40PostingsFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/pulsing/Pulsing40PostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/pulsing/Pulsing40PostingsFormat.java 2011-11-01 21:10:35.916609908 -0400 @@ -0,0 +1,42 @@ +package org.apache.lucene.index.codecs.pulsing; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsBaseFormat; + +/** + * @lucene.experimental + */ +public class Pulsing40PostingsFormat extends PulsingPostingsFormat { + + /** Inlines docFreq=1 terms, otherwise uses the normal "Lucene40" format. */ + public Pulsing40PostingsFormat() { + this(1); + } + + /** Inlines docFreq=freqCutoff terms, otherwise uses the normal "Lucene40" format. */ + public Pulsing40PostingsFormat(int freqCutoff) { + this(freqCutoff, BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); + } + + /** Inlines docFreq=freqCutoff terms, otherwise uses the normal "Lucene40" format. */ + public Pulsing40PostingsFormat(int freqCutoff, int minBlockSize, int maxBlockSize) { + super("Pulsing40", new Lucene40PostingsBaseFormat(), freqCutoff, minBlockSize, maxBlockSize); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java 2011-09-30 20:57:07.258773589 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,155 +0,0 @@ -package org.apache.lucene.index.codecs.pulsing; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.BlockTreeTermsReader; -import org.apache.lucene.index.codecs.BlockTreeTermsWriter; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; -import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; -import org.apache.lucene.store.Directory; - -/** This codec "inlines" the postings for terms that have - * low docFreq. It wraps another codec, which is used for - * writing the non-inlined terms. - * - * Currently in only inlines docFreq=1 terms, and - * otherwise uses the normal "standard" codec. - * @lucene.experimental */ - -public class PulsingCodec extends Codec { - - private final int freqCutoff; - private final int minBlockSize; - private final int maxBlockSize; - - public PulsingCodec() { - this(1); - } - - public PulsingCodec(int freqCutoff) { - this(freqCutoff, BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); - } - - /** Terms with freq <= freqCutoff are inlined into terms - * dict. */ - public PulsingCodec(int freqCutoff, int minBlockSize, int maxBlockSize) { - super("Pulsing"); - this.freqCutoff = freqCutoff; - this.minBlockSize = minBlockSize; - assert minBlockSize > 1; - this.maxBlockSize = maxBlockSize; - } - - @Override - public String toString() { - return name + "(freqCutoff=" + freqCutoff + " minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")"; - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - // We wrap StandardPostingsWriter, but any PostingsWriterBase - // will work: - - PostingsWriterBase docsWriter = new StandardPostingsWriter(state); - - // Terms that have <= freqCutoff number of docs are - // "pulsed" (inlined): - PostingsWriterBase pulsingWriter = new PulsingPostingsWriter(freqCutoff, docsWriter); - - // Terms dict - boolean success = false; - try { - FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, minBlockSize, maxBlockSize); - success = true; - return ret; - } finally { - if (!success) { - pulsingWriter.close(); - } - } - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - - // We wrap StandardPostingsReader, but any StandardPostingsReader - // will work: - PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); - PostingsReaderBase pulsingReader = new PulsingPostingsReader(docsReader); - - boolean success = false; - try { - FieldsProducer ret = new BlockTreeTermsReader( - state.dir, state.fieldInfos, state.segmentInfo.name, - pulsingReader, - state.context, - state.codecId, - state.termsIndexDivisor); - success = true; - return ret; - } finally { - if (!success) { - pulsingReader.close(); - } - } - } - - public int getFreqCutoff() { - return freqCutoff; - } - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, codecID, files); - BlockTreeTermsReader.files(dir, segmentInfo, codecID, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files); - } - - @Override - public void getExtensions(Set extensions) { - StandardCodec.getStandardExtensions(extensions); - DefaultDocValuesConsumer.getExtensions(extensions); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsFormat.java 2011-11-02 21:26:22.480564906 -0400 @@ -0,0 +1,122 @@ +package org.apache.lucene.index.codecs.pulsing; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.BlockTreeTermsReader; +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; +import org.apache.lucene.index.codecs.PostingsBaseFormat; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.store.Directory; + +/** This postings format "inlines" the postings for terms that have + * low docFreq. It wraps another postings format, which is used for + * writing the non-inlined terms. + * + * @lucene.experimental */ + +public abstract class PulsingPostingsFormat extends PostingsFormat { + + private final int freqCutoff; + private final int minBlockSize; + private final int maxBlockSize; + private final PostingsBaseFormat wrappedPostingsBaseFormat; + + public PulsingPostingsFormat(String name, PostingsBaseFormat wrappedPostingsBaseFormat, int freqCutoff) { + this(name, wrappedPostingsBaseFormat, freqCutoff, BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); + } + + /** Terms with freq <= freqCutoff are inlined into terms + * dict. */ + public PulsingPostingsFormat(String name, PostingsBaseFormat wrappedPostingsBaseFormat, int freqCutoff, int minBlockSize, int maxBlockSize) { + super(name); + this.freqCutoff = freqCutoff; + this.minBlockSize = minBlockSize; + assert minBlockSize > 1; + this.maxBlockSize = maxBlockSize; + this.wrappedPostingsBaseFormat = wrappedPostingsBaseFormat; + } + + @Override + public String toString() { + return getName() + "(freqCutoff=" + freqCutoff + " minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")"; + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docsWriter = wrappedPostingsBaseFormat.postingsWriterBase(state); + + // Terms that have <= freqCutoff number of docs are + // "pulsed" (inlined): + PostingsWriterBase pulsingWriter = new PulsingPostingsWriter(freqCutoff, docsWriter); + + // Terms dict + boolean success = false; + try { + FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, minBlockSize, maxBlockSize); + success = true; + return ret; + } finally { + if (!success) { + pulsingWriter.close(); + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + + PostingsReaderBase docsReader = wrappedPostingsBaseFormat.postingsReaderBase(state); + PostingsReaderBase pulsingReader = new PulsingPostingsReader(docsReader); + + boolean success = false; + try { + FieldsProducer ret = new BlockTreeTermsReader( + state.dir, state.fieldInfos, state.segmentInfo.name, + pulsingReader, + state.context, + state.segmentSuffix, + state.termsIndexDivisor); + success = true; + return ret; + } finally { + if (!success) { + pulsingReader.close(); + } + } + } + + public int getFreqCutoff() { + return freqCutoff; + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + wrappedPostingsBaseFormat.files(dir, segmentInfo, segmentSuffix, files); + BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosFormat.java 2011-11-01 21:10:35.936609908 -0400 @@ -0,0 +1,37 @@ +package org.apache.lucene.index.codecs; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Expert: Controls the format of the segments file. + * Note, this isn't a per-segment file, if you change the format, other versions + * of lucene won't be able to read it, yackedy schmackedy + * + * @lucene.experimental + */ +// TODO: would be great to handle this situation better. +// ideally a custom implementation could implement two-phase commit differently, +// (e.g. atomic rename), and ideally all versions of lucene could still read it. +// but this is just reflecting reality as it is today... +// +// also, perhaps the name should change (to cover all global files like .fnx?) +// then again, maybe we can just remove that file... +public abstract class SegmentInfosFormat { + public abstract SegmentInfosReader getSegmentInfosReader(); + public abstract SegmentInfosWriter getSegmentInfosWriter(); +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java 2011-07-08 07:13:00.771820239 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java 2011-11-02 21:49:13.244564200 -0400 @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -33,9 +34,9 @@ * Read {@link SegmentInfos} data from a directory. * @param directory directory to read from * @param segmentsFileName name of the "segments_N" file - * @param codecs current codecs + * @param header input of "segments_N" file after reading preamble * @param infos empty instance to be populated with data * @throws IOException */ - public abstract void read(Directory directory, String segmentsFileName, CodecProvider codecs, SegmentInfos infos, IOContext context) throws IOException; + public abstract void read(Directory directory, String segmentsFileName, ChecksumIndexInput header, SegmentInfos infos, IOContext context) throws IOException; } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java 2011-07-08 07:13:00.771820239 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java 2011-11-02 21:46:04.900564297 -0400 @@ -42,13 +42,13 @@ * phase commit" operations as described above. * @throws IOException */ - public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, SegmentInfos infos, IOContext context) throws IOException; + public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, String codecID, SegmentInfos infos, IOContext context) throws IOException; /** * First phase of the two-phase commit - ensure that all output can be * successfully written out. * @param out an instance of {@link IndexOutput} returned from a previous - * call to {@link #writeInfos(Directory, String, SegmentInfos, IOContext)}. + * call to {@link #writeInfos(Directory, String, String, SegmentInfos, IOContext)}. * @throws IOException */ public abstract void prepareCommit(IndexOutput out) throws IOException; @@ -57,7 +57,7 @@ * Second phase of the two-phase commit. In this step the output should be * finalized and closed. * @param out an instance of {@link IndexOutput} returned from a previous - * call to {@link #writeInfos(Directory, String, SegmentInfos, IOContext)}. + * call to {@link #writeInfos(Directory, String, String, SegmentInfos, IOContext)}. * @throws IOException */ public abstract void finishCommit(IndexOutput out) throws IOException; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java 2011-10-07 11:55:34.558772788 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java 2011-11-02 21:26:22.524564906 -0400 @@ -47,12 +47,12 @@ } @SuppressWarnings("fallthrough") - public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, + public static void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException { FieldInfos fieldInfos = segmentInfo.getFieldInfos(); for (FieldInfo fieldInfo : fieldInfos) { - if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) { - String filename = docValuesId(segmentInfo.name, codecId, fieldInfo.number); + if (fieldInfo.hasDocValues()) { + String filename = docValuesId(segmentInfo.name, fieldInfo.number); switch (fieldInfo.getDocValues()) { case BYTES_FIXED_DEREF: case BYTES_VAR_DEREF: @@ -83,9 +83,4 @@ } } } - - public static void getExtensions(Set extensions) { - extensions.add(Writer.DATA_EXTENSION); - extensions.add(Writer.INDEX_EXTENSION); - } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java 2011-09-30 20:57:07.288773589 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java 2011-11-02 21:26:22.524564906 -0400 @@ -39,7 +39,7 @@ * {@link IndexDocValues} instances for this segment and codec. */ public SepDocValuesProducer(SegmentReadState state) throws IOException { - docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, state.dir, state.codecId, state.context); + docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, state.dir, state.context); } @Override diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java 2011-08-22 08:19:47.271460417 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java 2011-11-02 21:26:22.524564906 -0400 @@ -60,23 +60,23 @@ int maxSkipLevels; int skipMinimum; - public SepPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext context, IntStreamFactory intFactory, int codecId) throws IOException { + public SepPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext context, IntStreamFactory intFactory, String segmentSuffix) throws IOException { boolean success = false; try { - final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.DOC_EXTENSION); + final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.DOC_EXTENSION); docIn = intFactory.openInput(dir, docFileName, context); - skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.SKIP_EXTENSION), context); + skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.SKIP_EXTENSION), context); if (segmentInfo.getFieldInfos().hasFreq()) { - freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.FREQ_EXTENSION), context); + freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.FREQ_EXTENSION), context); } else { freqIn = null; } if (segmentInfo.getHasProx()) { - posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.POS_EXTENSION), context); - payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.PAYLOAD_EXTENSION), context); + posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.POS_EXTENSION), context); + payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.PAYLOAD_EXTENSION), context); } else { posIn = null; payloadIn = null; @@ -89,17 +89,17 @@ } } - public static void files(SegmentInfo segmentInfo, int codecId, Collection files) throws IOException { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.DOC_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.SKIP_EXTENSION)); + public static void files(SegmentInfo segmentInfo, String segmentSuffix, Collection files) throws IOException { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.DOC_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.SKIP_EXTENSION)); if (segmentInfo.getFieldInfos().hasFreq()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.FREQ_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.FREQ_EXTENSION)); } if (segmentInfo.getHasProx()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.POS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.PAYLOAD_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.POS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.PAYLOAD_EXTENSION)); } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java 2011-08-29 12:22:46.831459551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java 2011-11-02 21:26:22.524564906 -0400 @@ -116,27 +116,27 @@ try { this.skipInterval = skipInterval; this.skipMinimum = skipInterval; /* set to the same for now */ - final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, DOC_EXTENSION); + final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, DOC_EXTENSION); docOut = factory.createOutput(state.directory, docFileName, state.context); docIndex = docOut.index(); if (state.fieldInfos.hasFreq()) { - final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, FREQ_EXTENSION); + final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, FREQ_EXTENSION); freqOut = factory.createOutput(state.directory, frqFileName, state.context); freqIndex = freqOut.index(); } if (state.fieldInfos.hasProx()) { - final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, POS_EXTENSION); + final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, POS_EXTENSION); posOut = factory.createOutput(state.directory, posFileName, state.context); posIndex = posOut.index(); // TODO: -- only if at least one field stores payloads? - final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, PAYLOAD_EXTENSION); + final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, PAYLOAD_EXTENSION); payloadOut = state.directory.createOutput(payloadFileName, state.context); } - final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SKIP_EXTENSION); + final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, SKIP_EXTENSION); skipOut = state.directory.createOutput(skipFileName, state.context); totalNumDocs = state.numDocs; @@ -391,12 +391,4 @@ public void close() throws IOException { IOUtils.close(docOut, skipOut, freqOut, posOut, payloadOut); } - - public static void getExtensions(Set extensions) { - extensions.add(DOC_EXTENSION); - extensions.add(FREQ_EXTENSION); - extensions.add(SKIP_EXTENSION); - extensions.add(POS_EXTENSION); - extensions.add(PAYLOAD_EXTENSION); - } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java 2011-09-30 20:57:07.268773589 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,91 +0,0 @@ -package org.apache.lucene.index.codecs.simpletext; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.store.Directory; - -/** For debugging, curiosity, transparency only!! Do not - * use this codec in production. - * - *

This codec stores all postings data in a single - * human-readable text file (_N.pst). You can view this in - * any text editor, and even edit it to alter your index. - * - * @lucene.experimental */ -public class SimpleTextCodec extends Codec { - - public SimpleTextCodec() { - super("SimpleText"); - } - - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - return new SimpleTextFieldsWriter(state); - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - return new SimpleTextFieldsReader(state); - } - - /** Extension of freq postings file */ - static final String POSTINGS_EXTENSION = "pst"; - - static String getPostingsFileName(String segment, int id) { - return IndexFileNames.segmentFileName(segment, id, POSTINGS_EXTENSION); - } - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { - files.add(getPostingsFileName(segmentInfo.name, id)); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); - } - - @Override - public void getExtensions(Set extensions) { - extensions.add(POSTINGS_EXTENSION); - DefaultDocValuesConsumer.getExtensions(extensions); - } - - // TODO: would be great if these used a plain text impl - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java 2011-10-15 21:56:47.701365171 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java 2011-11-02 21:26:22.496564907 -0400 @@ -60,7 +60,7 @@ final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD; public SimpleTextFieldsReader(SegmentReadState state) throws IOException { - in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, state.codecId), state.context); + in = state.dir.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context); fieldInfos = state.fieldInfos; } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java 2011-09-15 13:19:31.196254551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java 2011-11-02 21:26:22.496564907 -0400 @@ -47,7 +47,7 @@ final static BytesRef PAYLOAD = new BytesRef(" payload "); public SimpleTextFieldsWriter(SegmentWriteState state) throws IOException { - final String fileName = SimpleTextCodec.getPostingsFileName(state.segmentName, state.codecId); + final String fileName = SimpleTextPostingsFormat.getPostingsFileName(state.segmentName, state.segmentSuffix); out = state.directory.createOutput(fileName, state.context); } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextPostingsFormat.java 2011-11-02 21:26:22.496564907 -0400 @@ -0,0 +1,67 @@ +package org.apache.lucene.index.codecs.simpletext; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.store.Directory; + +/** For debugging, curiosity, transparency only!! Do not + * use this codec in production. + * + *

This codec stores all postings data in a single + * human-readable text file (_N.pst). You can view this in + * any text editor, and even edit it to alter your index. + * + * @lucene.experimental */ +public class SimpleTextPostingsFormat extends PostingsFormat { + + public SimpleTextPostingsFormat() { + super("SimpleText"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + return new SimpleTextFieldsWriter(state); + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + return new SimpleTextFieldsReader(state); + } + + /** Extension of freq postings file */ + static final String POSTINGS_EXTENSION = "pst"; + + static String getPostingsFileName(String segment, String segmentSuffix) { + return IndexFileNames.segmentFileName(segment, segmentSuffix, POSTINGS_EXTENSION); + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + files.add(getPostingsFileName(segmentInfo.name, segmentSuffix)); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListReader.java 2011-05-08 10:52:19.600078710 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListReader.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,118 +0,0 @@ -package org.apache.lucene.index.codecs.standard; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.lucene.index.codecs.MultiLevelSkipListReader; -import org.apache.lucene.store.IndexInput; - -/** - * Implements the skip list reader for the default posting list format - * that stores positions and payloads. - * @lucene.experimental - */ -public class DefaultSkipListReader extends MultiLevelSkipListReader { - private boolean currentFieldStoresPayloads; - private long freqPointer[]; - private long proxPointer[]; - private int payloadLength[]; - - private long lastFreqPointer; - private long lastProxPointer; - private int lastPayloadLength; - - - public DefaultSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) { - super(skipStream, maxSkipLevels, skipInterval); - freqPointer = new long[maxSkipLevels]; - proxPointer = new long[maxSkipLevels]; - payloadLength = new int[maxSkipLevels]; - } - - public void init(long skipPointer, long freqBasePointer, long proxBasePointer, int df, boolean storesPayloads) { - super.init(skipPointer, df); - this.currentFieldStoresPayloads = storesPayloads; - lastFreqPointer = freqBasePointer; - lastProxPointer = proxBasePointer; - - Arrays.fill(freqPointer, freqBasePointer); - Arrays.fill(proxPointer, proxBasePointer); - Arrays.fill(payloadLength, 0); - } - - /** Returns the freq pointer of the doc to which the last call of - * {@link MultiLevelSkipListReader#skipTo(int)} has skipped. */ - public long getFreqPointer() { - return lastFreqPointer; - } - - /** Returns the prox pointer of the doc to which the last call of - * {@link MultiLevelSkipListReader#skipTo(int)} has skipped. */ - public long getProxPointer() { - return lastProxPointer; - } - - /** Returns the payload length of the payload stored just before - * the doc to which the last call of {@link MultiLevelSkipListReader#skipTo(int)} - * has skipped. */ - public int getPayloadLength() { - return lastPayloadLength; - } - - @Override - protected void seekChild(int level) throws IOException { - super.seekChild(level); - freqPointer[level] = lastFreqPointer; - proxPointer[level] = lastProxPointer; - payloadLength[level] = lastPayloadLength; - } - - @Override - protected void setLastSkipData(int level) { - super.setLastSkipData(level); - lastFreqPointer = freqPointer[level]; - lastProxPointer = proxPointer[level]; - lastPayloadLength = payloadLength[level]; - } - - - @Override - protected int readSkipData(int level, IndexInput skipStream) throws IOException { - int delta; - if (currentFieldStoresPayloads) { - // the current field stores payloads. - // if the doc delta is odd then we have - // to read the current payload length - // because it differs from the length of the - // previous payload - delta = skipStream.readVInt(); - if ((delta & 1) != 0) { - payloadLength[level] = skipStream.readVInt(); - } - delta >>>= 1; - } else { - delta = skipStream.readVInt(); - } - freqPointer[level] += skipStream.readVInt(); - proxPointer[level] += skipStream.readVInt(); - - return delta; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java 2011-05-08 10:52:19.600078710 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,128 +0,0 @@ -package org.apache.lucene.index.codecs.standard; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.index.codecs.MultiLevelSkipListWriter; - - -/** - * Implements the skip list writer for the default posting list format - * that stores positions and payloads. - * @lucene.experimental - */ -public class DefaultSkipListWriter extends MultiLevelSkipListWriter { - private int[] lastSkipDoc; - private int[] lastSkipPayloadLength; - private long[] lastSkipFreqPointer; - private long[] lastSkipProxPointer; - - private IndexOutput freqOutput; - private IndexOutput proxOutput; - - private int curDoc; - private boolean curStorePayloads; - private int curPayloadLength; - private long curFreqPointer; - private long curProxPointer; - - public DefaultSkipListWriter(int skipInterval, int numberOfSkipLevels, int docCount, IndexOutput freqOutput, IndexOutput proxOutput) { - super(skipInterval, numberOfSkipLevels, docCount); - this.freqOutput = freqOutput; - this.proxOutput = proxOutput; - - lastSkipDoc = new int[numberOfSkipLevels]; - lastSkipPayloadLength = new int[numberOfSkipLevels]; - lastSkipFreqPointer = new long[numberOfSkipLevels]; - lastSkipProxPointer = new long[numberOfSkipLevels]; - } - - /** - * Sets the values for the current skip data. - */ - public void setSkipData(int doc, boolean storePayloads, int payloadLength) { - this.curDoc = doc; - this.curStorePayloads = storePayloads; - this.curPayloadLength = payloadLength; - this.curFreqPointer = freqOutput.getFilePointer(); - if (proxOutput != null) - this.curProxPointer = proxOutput.getFilePointer(); - } - - @Override - public void resetSkip() { - super.resetSkip(); - Arrays.fill(lastSkipDoc, 0); - Arrays.fill(lastSkipPayloadLength, -1); // we don't have to write the first length in the skip list - Arrays.fill(lastSkipFreqPointer, freqOutput.getFilePointer()); - if (proxOutput != null) - Arrays.fill(lastSkipProxPointer, proxOutput.getFilePointer()); - } - - @Override - protected void writeSkipData(int level, IndexOutput skipBuffer) throws IOException { - // To efficiently store payloads in the posting lists we do not store the length of - // every payload. Instead we omit the length for a payload if the previous payload had - // the same length. - // However, in order to support skipping the payload length at every skip point must be known. - // So we use the same length encoding that we use for the posting lists for the skip data as well: - // Case 1: current field does not store payloads - // SkipDatum --> DocSkip, FreqSkip, ProxSkip - // DocSkip,FreqSkip,ProxSkip --> VInt - // DocSkip records the document number before every SkipInterval th document in TermFreqs. - // Document numbers are represented as differences from the previous value in the sequence. - // Case 2: current field stores payloads - // SkipDatum --> DocSkip, PayloadLength?, FreqSkip,ProxSkip - // DocSkip,FreqSkip,ProxSkip --> VInt - // PayloadLength --> VInt - // In this case DocSkip/2 is the difference between - // the current and the previous value. If DocSkip - // is odd, then a PayloadLength encoded as VInt follows, - // if DocSkip is even, then it is assumed that the - // current payload length equals the length at the previous - // skip point - if (curStorePayloads) { - int delta = curDoc - lastSkipDoc[level]; - if (curPayloadLength == lastSkipPayloadLength[level]) { - // the current payload length equals the length at the previous skip point, - // so we don't store the length again - skipBuffer.writeVInt(delta * 2); - } else { - // the payload length is different from the previous one. We shift the DocSkip, - // set the lowest bit and store the current payload length as VInt. - skipBuffer.writeVInt(delta * 2 + 1); - skipBuffer.writeVInt(curPayloadLength); - lastSkipPayloadLength[level] = curPayloadLength; - } - } else { - // current field does not store payloads - skipBuffer.writeVInt(curDoc - lastSkipDoc[level]); - } - skipBuffer.writeVInt((int) (curFreqPointer - lastSkipFreqPointer[level])); - skipBuffer.writeVInt((int) (curProxPointer - lastSkipProxPointer[level])); - - lastSkipDoc[level] = curDoc; - - lastSkipFreqPointer[level] = curFreqPointer; - lastSkipProxPointer[level] = curProxPointer; - } - -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java 2011-09-30 20:57:07.278773589 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,142 +0,0 @@ -package org.apache.lucene.index.codecs.standard; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.BlockTreeTermsReader; -import org.apache.lucene.index.codecs.BlockTreeTermsWriter; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.store.Directory; - -/** Default codec. - * @lucene.experimental */ -public class StandardCodec extends Codec { - - private final int minBlockSize; - private final int maxBlockSize; - - public StandardCodec() { - this(BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); - } - - public StandardCodec(int minBlockSize, int maxBlockSize) { - super("Standard"); - this.minBlockSize = minBlockSize; - assert minBlockSize > 1; - this.maxBlockSize = maxBlockSize; - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase docs = new StandardPostingsWriter(state); - - // TODO: should we make the terms index more easily - // pluggable? Ie so that this codec would record which - // index impl was used, and switch on loading? - // Or... you must make a new Codec for this? - boolean success = false; - try { - FieldsConsumer ret = new BlockTreeTermsWriter(state, docs, minBlockSize, maxBlockSize); - success = true; - return ret; - } finally { - if (!success) { - docs.close(); - } - } - } - - public final static int TERMS_CACHE_SIZE = 1024; - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); - - boolean success = false; - try { - FieldsProducer ret = new BlockTreeTermsReader( - state.dir, - state.fieldInfos, - state.segmentInfo.name, - postings, - state.context, - state.codecId, - state.termsIndexDivisor); - success = true; - return ret; - } finally { - if (!success) { - postings.close(); - } - } - } - - /** Extension of freq postings file */ - static final String FREQ_EXTENSION = "frq"; - - /** Extension of prox postings file */ - static final String PROX_EXTENSION = "prx"; - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int codecID, Set files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, codecID, files); - BlockTreeTermsReader.files(dir, segmentInfo, codecID, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, codecID, files); - } - - @Override - public void getExtensions(Set extensions) { - getStandardExtensions(extensions); - } - - public static void getStandardExtensions(Set extensions) { - extensions.add(FREQ_EXTENSION); - extensions.add(PROX_EXTENSION); - BlockTreeTermsReader.getExtensions(extensions); - DefaultDocValuesConsumer.getExtensions(extensions); - } - - @Override - public String toString() { - return name + "(minBlockSize=" + minBlockSize + " maxBlockSize=" + maxBlockSize + ")"; - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java 2011-08-20 13:35:07.941460633 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,874 +0,0 @@ -package org.apache.lucene.index.codecs.standard; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Collection; - -import org.apache.lucene.index.DocsAndPositionsEnum; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.TermState; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.BlockTermState; -import org.apache.lucene.store.ByteArrayDataInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; - -/** Concrete class that reads the current doc/freq/skip - * postings format. - * @lucene.experimental */ - -public class StandardPostingsReader extends PostingsReaderBase { - - private final IndexInput freqIn; - private final IndexInput proxIn; - // public static boolean DEBUG = BlockTreeTermsWriter.DEBUG; - - int skipInterval; - int maxSkipLevels; - int skipMinimum; - - // private String segment; - - public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext ioContext, int codecId) throws IOException { - freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.FREQ_EXTENSION), - ioContext); - // this.segment = segmentInfo.name; - if (segmentInfo.getHasProx()) { - boolean success = false; - try { - proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.PROX_EXTENSION), - ioContext); - success = true; - } finally { - if (!success) { - freqIn.close(); - } - } - } else { - proxIn = null; - } - } - - public static void files(Directory dir, SegmentInfo segmentInfo, int codecID, Collection files) throws IOException { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, StandardCodec.FREQ_EXTENSION)); - if (segmentInfo.getHasProx()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, StandardCodec.PROX_EXTENSION)); - } - } - - @Override - public void init(IndexInput termsIn) throws IOException { - - // Make sure we are talking to the matching past writer - CodecUtil.checkHeader(termsIn, StandardPostingsWriter.CODEC, - StandardPostingsWriter.VERSION_START, StandardPostingsWriter.VERSION_START); - - skipInterval = termsIn.readInt(); - maxSkipLevels = termsIn.readInt(); - skipMinimum = termsIn.readInt(); - } - - // Must keep final because we do non-standard clone - private final static class StandardTermState extends BlockTermState { - long freqOffset; - long proxOffset; - int skipOffset; - - // Only used by the "primary" TermState -- clones don't - // copy this (basically they are "transient"): - ByteArrayDataInput bytesReader; // TODO: should this NOT be in the TermState...? - byte[] bytes; - - @Override - public Object clone() { - StandardTermState other = new StandardTermState(); - other.copyFrom(this); - return other; - } - - @Override - public void copyFrom(TermState _other) { - super.copyFrom(_other); - StandardTermState other = (StandardTermState) _other; - freqOffset = other.freqOffset; - proxOffset = other.proxOffset; - skipOffset = other.skipOffset; - - // Do not copy bytes, bytesReader (else TermState is - // very heavy, ie drags around the entire block's - // byte[]). On seek back, if next() is in fact used - // (rare!), they will be re-read from disk. - } - - @Override - public String toString() { - return super.toString() + " freqFP=" + freqOffset + " proxFP=" + proxOffset + " skipOffset=" + skipOffset; - } - } - - @Override - public BlockTermState newTermState() { - return new StandardTermState(); - } - - @Override - public void close() throws IOException { - try { - if (freqIn != null) { - freqIn.close(); - } - } finally { - if (proxIn != null) { - proxIn.close(); - } - } - } - - /* Reads but does not decode the byte[] blob holding - metadata for the current terms block */ - @Override - public void readTermsBlock(IndexInput termsIn, FieldInfo fieldInfo, BlockTermState _termState) throws IOException { - final StandardTermState termState = (StandardTermState) _termState; - - final int len = termsIn.readVInt(); - - // if (DEBUG) System.out.println(" SPR.readTermsBlock bytes=" + len + " ts=" + _termState); - if (termState.bytes == null) { - termState.bytes = new byte[ArrayUtil.oversize(len, 1)]; - termState.bytesReader = new ByteArrayDataInput(); - } else if (termState.bytes.length < len) { - termState.bytes = new byte[ArrayUtil.oversize(len, 1)]; - } - - termsIn.readBytes(termState.bytes, 0, len); - termState.bytesReader.reset(termState.bytes, 0, len); - } - - @Override - public void nextTerm(FieldInfo fieldInfo, BlockTermState _termState) - throws IOException { - final StandardTermState termState = (StandardTermState) _termState; - // if (DEBUG) System.out.println("SPR: nextTerm seg=" + segment + " tbOrd=" + termState.termBlockOrd + " bytesReader.fp=" + termState.bytesReader.getPosition()); - final boolean isFirstTerm = termState.termBlockOrd == 0; - - if (isFirstTerm) { - termState.freqOffset = termState.bytesReader.readVLong(); - } else { - termState.freqOffset += termState.bytesReader.readVLong(); - } - /* - if (DEBUG) { - System.out.println(" dF=" + termState.docFreq); - System.out.println(" freqFP=" + termState.freqOffset); - } - */ - assert termState.freqOffset < freqIn.length(); - - if (termState.docFreq >= skipMinimum) { - termState.skipOffset = termState.bytesReader.readVInt(); - // if (DEBUG) System.out.println(" skipOffset=" + termState.skipOffset + " vs freqIn.length=" + freqIn.length()); - assert termState.freqOffset + termState.skipOffset < freqIn.length(); - } else { - // undefined - } - - if (fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { - if (isFirstTerm) { - termState.proxOffset = termState.bytesReader.readVLong(); - } else { - termState.proxOffset += termState.bytesReader.readVLong(); - } - // if (DEBUG) System.out.println(" proxFP=" + termState.proxOffset); - } - } - - @Override - public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse) throws IOException { - SegmentDocsEnum docsEnum; - if (reuse == null || !(reuse instanceof SegmentDocsEnum)) { - docsEnum = new SegmentDocsEnum(freqIn); - } else { - docsEnum = (SegmentDocsEnum) reuse; - if (docsEnum.startFreqIn != freqIn) { - // If you are using ParellelReader, and pass in a - // reused DocsEnum, it could have come from another - // reader also using standard codec - docsEnum = new SegmentDocsEnum(freqIn); - } - } - // if (DEBUG) System.out.println("SPR.docs ts=" + termState); - return docsEnum.reset(fieldInfo, (StandardTermState) termState, liveDocs); - } - - @Override - public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException { - if (fieldInfo.indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { - return null; - } - - // TODO: refactor - if (fieldInfo.storePayloads) { - SegmentDocsAndPositionsAndPayloadsEnum docsEnum; - if (reuse == null || !(reuse instanceof SegmentDocsAndPositionsAndPayloadsEnum)) { - docsEnum = new SegmentDocsAndPositionsAndPayloadsEnum(freqIn, proxIn); - } else { - docsEnum = (SegmentDocsAndPositionsAndPayloadsEnum) reuse; - if (docsEnum.startFreqIn != freqIn) { - // If you are using ParellelReader, and pass in a - // reused DocsEnum, it could have come from another - // reader also using standard codec - docsEnum = new SegmentDocsAndPositionsAndPayloadsEnum(freqIn, proxIn); - } - } - return docsEnum.reset(fieldInfo, (StandardTermState) termState, liveDocs); - } else { - SegmentDocsAndPositionsEnum docsEnum; - if (reuse == null || !(reuse instanceof SegmentDocsAndPositionsEnum)) { - docsEnum = new SegmentDocsAndPositionsEnum(freqIn, proxIn); - } else { - docsEnum = (SegmentDocsAndPositionsEnum) reuse; - if (docsEnum.startFreqIn != freqIn) { - // If you are using ParellelReader, and pass in a - // reused DocsEnum, it could have come from another - // reader also using standard codec - docsEnum = new SegmentDocsAndPositionsEnum(freqIn, proxIn); - } - } - return docsEnum.reset(fieldInfo, (StandardTermState) termState, liveDocs); - } - } - - // Decodes only docs - private class SegmentDocsEnum extends DocsEnum { - final IndexInput freqIn; - final IndexInput startFreqIn; - - boolean omitTF; // does current field omit term freq? - boolean storePayloads; // does current field store payloads? - - int limit; // number of docs in this posting - int ord; // how many docs we've read - int doc; // doc we last read - int freq; // freq we last read - - Bits liveDocs; - - long freqOffset; - int skipOffset; - - boolean skipped; - DefaultSkipListReader skipper; - - public SegmentDocsEnum(IndexInput freqIn) throws IOException { - startFreqIn = freqIn; - this.freqIn = (IndexInput) freqIn.clone(); - } - - public SegmentDocsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException { - omitTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY; - if (omitTF) { - freq = 1; - } - storePayloads = fieldInfo.storePayloads; - this.liveDocs = liveDocs; - freqOffset = termState.freqOffset; - skipOffset = termState.skipOffset; - - // TODO: for full enum case (eg segment merging) this - // seek is unnecessary; maybe we can avoid in such - // cases - freqIn.seek(termState.freqOffset); - limit = termState.docFreq; - assert limit > 0; - ord = 0; - doc = 0; - // if (DEBUG) System.out.println(" sde limit=" + limit + " freqFP=" + freqOffset); - - skipped = false; - - return this; - } - - @Override - public int nextDoc() throws IOException { - //if (DEBUG) System.out.println(" stpr.nextDoc seg=" + segment + " fp=" + freqIn.getFilePointer()); - while(true) { - if (ord == limit) { - //if (DEBUG) System.out.println(" return doc=" + NO_MORE_DOCS); - return doc = NO_MORE_DOCS; - } - - ord++; - - // Decode next doc/freq pair - final int code = freqIn.readVInt(); - // if (DEBUG) System.out.println(" code=" + code); - if (omitTF) { - doc += code; - } else { - doc += code >>> 1; // shift off low bit - if ((code & 1) != 0) { // if low bit is set - freq = 1; // freq is one - } else { - freq = freqIn.readVInt(); // else read freq - } - } - - if (liveDocs == null || liveDocs.get(doc)) { - break; - } - } - - //if (DEBUG) System.out.println(" stpr.nextDoc return doc=" + doc); - return doc; - } - - @Override - public int read() throws IOException { - - final int[] docs = bulkResult.docs.ints; - final int[] freqs = bulkResult.freqs.ints; - int i = 0; - final int length = docs.length; - while (i < length && ord < limit) { - ord++; - // manually inlined call to next() for speed - final int code = freqIn.readVInt(); - if (omitTF) { - doc += code; - } else { - doc += code >>> 1; // shift off low bit - if ((code & 1) != 0) { // if low bit is set - freq = 1; // freq is one - } else { - freq = freqIn.readVInt(); // else read freq - } - } - - if (liveDocs == null || liveDocs.get(doc)) { - docs[i] = doc; - freqs[i] = freq; - ++i; - } - } - - return i; - } - - @Override - public int docID() { - return doc; - } - - @Override - public int freq() { - return freq; - } - - @Override - public int advance(int target) throws IOException { - - if ((target - skipInterval) >= doc && limit >= skipMinimum) { - - // There are enough docs in the posting to have - // skip data, and it isn't too close. - - if (skipper == null) { - // This is the first time this enum has ever been used for skipping -- do lazy init - skipper = new DefaultSkipListReader((IndexInput) freqIn.clone(), maxSkipLevels, skipInterval); - } - - if (!skipped) { - - // This is the first time this posting has - // skipped since reset() was called, so now we - // load the skip data for this posting - - skipper.init(freqOffset + skipOffset, - freqOffset, 0, - limit, storePayloads); - - skipped = true; - } - - final int newOrd = skipper.skipTo(target); - - if (newOrd > ord) { - // Skipper moved - - ord = newOrd; - doc = skipper.getDoc(); - freqIn.seek(skipper.getFreqPointer()); - } - } - - // scan for the rest: - do { - nextDoc(); - } while (target > doc); - - return doc; - } - } - - // Decodes docs & positions. payloads are not present. - private class SegmentDocsAndPositionsEnum extends DocsAndPositionsEnum { - final IndexInput startFreqIn; - private final IndexInput freqIn; - private final IndexInput proxIn; - - int limit; // number of docs in this posting - int ord; // how many docs we've read - int doc; // doc we last read - int freq; // freq we last read - int position; - - Bits liveDocs; - - long freqOffset; - int skipOffset; - long proxOffset; - - int posPendingCount; - - boolean skipped; - DefaultSkipListReader skipper; - private long lazyProxPointer; - - public SegmentDocsAndPositionsEnum(IndexInput freqIn, IndexInput proxIn) throws IOException { - startFreqIn = freqIn; - this.freqIn = (IndexInput) freqIn.clone(); - this.proxIn = (IndexInput) proxIn.clone(); - } - - public SegmentDocsAndPositionsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException { - assert fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - assert !fieldInfo.storePayloads; - - this.liveDocs = liveDocs; - - // TODO: for full enum case (eg segment merging) this - // seek is unnecessary; maybe we can avoid in such - // cases - freqIn.seek(termState.freqOffset); - lazyProxPointer = termState.proxOffset; - - limit = termState.docFreq; - assert limit > 0; - - ord = 0; - doc = 0; - position = 0; - - skipped = false; - posPendingCount = 0; - - freqOffset = termState.freqOffset; - proxOffset = termState.proxOffset; - skipOffset = termState.skipOffset; - // if (DEBUG) System.out.println("StandardR.D&PE reset seg=" + segment + " limit=" + limit + " freqFP=" + freqOffset + " proxFP=" + proxOffset); - - return this; - } - - @Override - public int nextDoc() throws IOException { - // if (DEBUG) System.out.println("SPR.nextDoc seg=" + segment + " freqIn.fp=" + freqIn.getFilePointer()); - while(true) { - if (ord == limit) { - // if (DEBUG) System.out.println(" return END"); - return doc = NO_MORE_DOCS; - } - - ord++; - - // Decode next doc/freq pair - final int code = freqIn.readVInt(); - - doc += code >>> 1; // shift off low bit - if ((code & 1) != 0) { // if low bit is set - freq = 1; // freq is one - } else { - freq = freqIn.readVInt(); // else read freq - } - posPendingCount += freq; - - if (liveDocs == null || liveDocs.get(doc)) { - break; - } - } - - position = 0; - - // if (DEBUG) System.out.println(" return doc=" + doc); - return doc; - } - - @Override - public int docID() { - return doc; - } - - @Override - public int freq() { - return freq; - } - - @Override - public int advance(int target) throws IOException { - - //System.out.println("StandardR.D&PE advance target=" + target); - - if ((target - skipInterval) >= doc && limit >= skipMinimum) { - - // There are enough docs in the posting to have - // skip data, and it isn't too close - - if (skipper == null) { - // This is the first time this enum has ever been used for skipping -- do lazy init - skipper = new DefaultSkipListReader((IndexInput) freqIn.clone(), maxSkipLevels, skipInterval); - } - - if (!skipped) { - - // This is the first time this posting has - // skipped, since reset() was called, so now we - // load the skip data for this posting - - skipper.init(freqOffset+skipOffset, - freqOffset, proxOffset, - limit, false); - - skipped = true; - } - - final int newOrd = skipper.skipTo(target); - - if (newOrd > ord) { - // Skipper moved - ord = newOrd; - doc = skipper.getDoc(); - freqIn.seek(skipper.getFreqPointer()); - lazyProxPointer = skipper.getProxPointer(); - posPendingCount = 0; - position = 0; - } - } - - // Now, linear scan for the rest: - do { - nextDoc(); - } while (target > doc); - - return doc; - } - - @Override - public int nextPosition() throws IOException { - - if (lazyProxPointer != -1) { - proxIn.seek(lazyProxPointer); - lazyProxPointer = -1; - } - - // scan over any docs that were iterated without their positions - if (posPendingCount > freq) { - position = 0; - while(posPendingCount != freq) { - if ((proxIn.readByte() & 0x80) == 0) { - posPendingCount--; - } - } - } - - position += proxIn.readVInt(); - - posPendingCount--; - - assert posPendingCount >= 0: "nextPosition() was called too many times (more than freq() times) posPendingCount=" + posPendingCount; - - return position; - } - - /** Returns the payload at this position, or null if no - * payload was indexed. */ - @Override - public BytesRef getPayload() throws IOException { - throw new IOException("No payloads exist for this field!"); - } - - @Override - public boolean hasPayload() { - return false; - } - } - - // Decodes docs & positions & payloads - private class SegmentDocsAndPositionsAndPayloadsEnum extends DocsAndPositionsEnum { - final IndexInput startFreqIn; - private final IndexInput freqIn; - private final IndexInput proxIn; - - int limit; // number of docs in this posting - int ord; // how many docs we've read - int doc; // doc we last read - int freq; // freq we last read - int position; - - Bits liveDocs; - - long freqOffset; - int skipOffset; - long proxOffset; - - int posPendingCount; - int payloadLength; - boolean payloadPending; - - boolean skipped; - DefaultSkipListReader skipper; - private BytesRef payload; - private long lazyProxPointer; - - public SegmentDocsAndPositionsAndPayloadsEnum(IndexInput freqIn, IndexInput proxIn) throws IOException { - startFreqIn = freqIn; - this.freqIn = (IndexInput) freqIn.clone(); - this.proxIn = (IndexInput) proxIn.clone(); - } - - public SegmentDocsAndPositionsAndPayloadsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException { - assert fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS; - assert fieldInfo.storePayloads; - if (payload == null) { - payload = new BytesRef(); - payload.bytes = new byte[1]; - } - - this.liveDocs = liveDocs; - - // TODO: for full enum case (eg segment merging) this - // seek is unnecessary; maybe we can avoid in such - // cases - freqIn.seek(termState.freqOffset); - lazyProxPointer = termState.proxOffset; - - limit = termState.docFreq; - ord = 0; - doc = 0; - position = 0; - - skipped = false; - posPendingCount = 0; - payloadPending = false; - - freqOffset = termState.freqOffset; - proxOffset = termState.proxOffset; - skipOffset = termState.skipOffset; - //System.out.println("StandardR.D&PE reset seg=" + segment + " limit=" + limit + " freqFP=" + freqOffset + " proxFP=" + proxOffset + " this=" + this); - - return this; - } - - @Override - public int nextDoc() throws IOException { - while(true) { - if (ord == limit) { - //System.out.println("StandardR.D&PE seg=" + segment + " nextDoc return doc=END"); - return doc = NO_MORE_DOCS; - } - - ord++; - - // Decode next doc/freq pair - final int code = freqIn.readVInt(); - - doc += code >>> 1; // shift off low bit - if ((code & 1) != 0) { // if low bit is set - freq = 1; // freq is one - } else { - freq = freqIn.readVInt(); // else read freq - } - posPendingCount += freq; - - if (liveDocs == null || liveDocs.get(doc)) { - break; - } - } - - position = 0; - - //System.out.println("StandardR.D&PE nextDoc seg=" + segment + " return doc=" + doc); - return doc; - } - - @Override - public int docID() { - return doc; - } - - @Override - public int freq() { - return freq; - } - - @Override - public int advance(int target) throws IOException { - - //System.out.println("StandardR.D&PE advance seg=" + segment + " target=" + target + " this=" + this); - - if ((target - skipInterval) >= doc && limit >= skipMinimum) { - - // There are enough docs in the posting to have - // skip data, and it isn't too close - - if (skipper == null) { - // This is the first time this enum has ever been used for skipping -- do lazy init - skipper = new DefaultSkipListReader((IndexInput) freqIn.clone(), maxSkipLevels, skipInterval); - } - - if (!skipped) { - - // This is the first time this posting has - // skipped, since reset() was called, so now we - // load the skip data for this posting - //System.out.println(" init skipper freqOffset=" + freqOffset + " skipOffset=" + skipOffset + " vs len=" + freqIn.length()); - skipper.init(freqOffset+skipOffset, - freqOffset, proxOffset, - limit, true); - - skipped = true; - } - - final int newOrd = skipper.skipTo(target); - - if (newOrd > ord) { - // Skipper moved - ord = newOrd; - doc = skipper.getDoc(); - freqIn.seek(skipper.getFreqPointer()); - lazyProxPointer = skipper.getProxPointer(); - posPendingCount = 0; - position = 0; - payloadPending = false; - payloadLength = skipper.getPayloadLength(); - } - } - - // Now, linear scan for the rest: - do { - nextDoc(); - } while (target > doc); - - return doc; - } - - @Override - public int nextPosition() throws IOException { - - if (lazyProxPointer != -1) { - proxIn.seek(lazyProxPointer); - lazyProxPointer = -1; - } - - if (payloadPending && payloadLength > 0) { - // payload of last position as never retrieved -- skip it - proxIn.seek(proxIn.getFilePointer() + payloadLength); - payloadPending = false; - } - - // scan over any docs that were iterated without their positions - while(posPendingCount > freq) { - - final int code = proxIn.readVInt(); - - if ((code & 1) != 0) { - // new payload length - payloadLength = proxIn.readVInt(); - assert payloadLength >= 0; - } - - assert payloadLength != -1; - proxIn.seek(proxIn.getFilePointer() + payloadLength); - - posPendingCount--; - position = 0; - payloadPending = false; - //System.out.println("StandardR.D&PE skipPos"); - } - - // read next position - if (payloadPending && payloadLength > 0) { - // payload wasn't retrieved for last position - proxIn.seek(proxIn.getFilePointer()+payloadLength); - } - - final int code = proxIn.readVInt(); - if ((code & 1) != 0) { - // new payload length - payloadLength = proxIn.readVInt(); - assert payloadLength >= 0; - } - assert payloadLength != -1; - - payloadPending = true; - position += code >>> 1; - - posPendingCount--; - - assert posPendingCount >= 0: "nextPosition() was called too many times (more than freq() times) posPendingCount=" + posPendingCount; - - //System.out.println("StandardR.D&PE nextPos return pos=" + position); - return position; - } - - /** Returns the payload at this position, or null if no - * payload was indexed. */ - @Override - public BytesRef getPayload() throws IOException { - assert lazyProxPointer == -1; - assert posPendingCount < freq; - if (!payloadPending) { - throw new IOException("Either no payload exists at this term position or an attempt was made to load it more than once."); - } - if (payloadLength > payload.bytes.length) { - payload.grow(payloadLength); - } - - proxIn.readBytes(payload.bytes, 0, payloadLength); - payload.length = payloadLength; - payloadPending = false; - - return payload; - } - - @Override - public boolean hasPayload() { - return payloadPending && payloadLength > 0; - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java 2011-08-20 13:35:07.941460633 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,325 +0,0 @@ -package org.apache.lucene.index.codecs.standard; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** Consumes doc & freq, writing them using the current - * index file format */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.DocsEnum; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.TermStats; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.RAMOutputStream; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CodecUtil; - -/** @lucene.experimental */ -public final class StandardPostingsWriter extends PostingsWriterBase { - final static String CODEC = "StandardPostingsWriter"; - - //private static boolean DEBUG = BlockTreeTermsWriter.DEBUG; - - // Increment version to change it: - final static int VERSION_START = 0; - final static int VERSION_CURRENT = VERSION_START; - - final IndexOutput freqOut; - final IndexOutput proxOut; - final DefaultSkipListWriter skipListWriter; - /** Expert: The fraction of TermDocs entries stored in skip tables, - * used to accelerate {@link DocsEnum#advance(int)}. Larger values result in - * smaller indexes, greater acceleration, but fewer accelerable cases, while - * smaller values result in bigger indexes, less acceleration and more - * accelerable cases. More detailed experiments would be useful here. */ - static final int DEFAULT_SKIP_INTERVAL = 16; - final int skipInterval; - - /** - * Expert: minimum docFreq to write any skip data at all - */ - final int skipMinimum; - - /** Expert: The maximum number of skip levels. Smaller values result in - * slightly smaller indexes, but slower skipping in big posting lists. - */ - final int maxSkipLevels = 10; - final int totalNumDocs; - IndexOutput termsOut; - - IndexOptions indexOptions; - boolean storePayloads; - // Starts a new term - long freqStart; - long proxStart; - FieldInfo fieldInfo; - int lastPayloadLength; - int lastPosition; - - // private String segment; - - public StandardPostingsWriter(SegmentWriteState state) throws IOException { - this(state, DEFAULT_SKIP_INTERVAL); - } - - public StandardPostingsWriter(SegmentWriteState state, int skipInterval) throws IOException { - super(); - this.skipInterval = skipInterval; - this.skipMinimum = skipInterval; /* set to the same for now */ - // this.segment = state.segmentName; - String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.FREQ_EXTENSION); - freqOut = state.directory.createOutput(fileName, state.context); - if (state.fieldInfos.hasProx()) { - // At least one field does not omit TF, so create the - // prox file - fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION); - proxOut = state.directory.createOutput(fileName, state.context); - } else { - // Every field omits TF so we will write no prox file - proxOut = null; - } - - totalNumDocs = state.numDocs; - - skipListWriter = new DefaultSkipListWriter(skipInterval, - maxSkipLevels, - state.numDocs, - freqOut, - proxOut); - } - - @Override - public void start(IndexOutput termsOut) throws IOException { - this.termsOut = termsOut; - CodecUtil.writeHeader(termsOut, CODEC, VERSION_CURRENT); - termsOut.writeInt(skipInterval); // write skipInterval - termsOut.writeInt(maxSkipLevels); // write maxSkipLevels - termsOut.writeInt(skipMinimum); // write skipMinimum - } - - @Override - public void startTerm() { - freqStart = freqOut.getFilePointer(); - //if (DEBUG) System.out.println("SPW: startTerm freqOut.fp=" + freqStart); - if (proxOut != null) { - proxStart = proxOut.getFilePointer(); - // force first payload to write its length - lastPayloadLength = -1; - } - skipListWriter.resetSkip(); - } - - // Currently, this instance is re-used across fields, so - // our parent calls setField whenever the field changes - @Override - public void setField(FieldInfo fieldInfo) { - //System.out.println("SPW: setField"); - /* - if (BlockTreeTermsWriter.DEBUG && fieldInfo.name.equals("id")) { - DEBUG = true; - } else { - DEBUG = false; - } - */ - this.fieldInfo = fieldInfo; - indexOptions = fieldInfo.indexOptions; - storePayloads = fieldInfo.storePayloads; - //System.out.println(" set init blockFreqStart=" + freqStart); - //System.out.println(" set init blockProxStart=" + proxStart); - } - - int lastDocID; - int df; - - /** Adds a new doc in this term. If this returns null - * then we just skip consuming positions/payloads. */ - @Override - public void startDoc(int docID, int termDocFreq) throws IOException { - // if (DEBUG) System.out.println("SPW: startDoc seg=" + segment + " docID=" + docID + " tf=" + termDocFreq + " freqOut.fp=" + freqOut.getFilePointer()); - - final int delta = docID - lastDocID; - - if (docID < 0 || (df > 0 && delta <= 0)) { - throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )"); - } - - if ((++df % skipInterval) == 0) { - skipListWriter.setSkipData(lastDocID, storePayloads, lastPayloadLength); - skipListWriter.bufferSkip(df); - } - - assert docID < totalNumDocs: "docID=" + docID + " totalNumDocs=" + totalNumDocs; - - lastDocID = docID; - if (indexOptions == IndexOptions.DOCS_ONLY) { - freqOut.writeVInt(delta); - } else if (1 == termDocFreq) { - freqOut.writeVInt((delta<<1) | 1); - } else { - freqOut.writeVInt(delta<<1); - freqOut.writeVInt(termDocFreq); - } - - lastPosition = 0; - } - - /** Add a new position & payload */ - @Override - public void addPosition(int position, BytesRef payload) throws IOException { - //if (DEBUG) System.out.println("SPW: addPos pos=" + position + " payload=" + (payload == null ? "null" : (payload.length + " bytes")) + " proxFP=" + proxOut.getFilePointer()); - assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS: "invalid indexOptions: " + indexOptions; - assert proxOut != null; - - final int delta = position - lastPosition; - - assert delta >= 0: "position=" + position + " lastPosition=" + lastPosition; // not quite right (if pos=0 is repeated twice we don't catch it) - - lastPosition = position; - - if (storePayloads) { - final int payloadLength = payload == null ? 0 : payload.length; - - if (payloadLength != lastPayloadLength) { - lastPayloadLength = payloadLength; - proxOut.writeVInt((delta<<1)|1); - proxOut.writeVInt(payloadLength); - } else { - proxOut.writeVInt(delta << 1); - } - - if (payloadLength > 0) { - proxOut.writeBytes(payload.bytes, payload.offset, payloadLength); - } - } else { - proxOut.writeVInt(delta); - } - } - - @Override - public void finishDoc() { - } - - private static class PendingTerm { - public final long freqStart; - public final long proxStart; - public final int skipOffset; - - public PendingTerm(long freqStart, long proxStart, int skipOffset) { - this.freqStart = freqStart; - this.proxStart = proxStart; - this.skipOffset = skipOffset; - } - } - - private final List pendingTerms = new ArrayList(); - - /** Called when we are done adding docs to this term */ - @Override - public void finishTerm(TermStats stats) throws IOException { - - // if (DEBUG) System.out.println("SPW: finishTerm seg=" + segment + " freqStart=" + freqStart); - assert stats.docFreq > 0; - - // TODO: wasteful we are counting this (counting # docs - // for this term) in two places? - assert stats.docFreq == df; - - final int skipOffset; - if (df >= skipMinimum) { - skipOffset = (int) (skipListWriter.writeSkip(freqOut)-freqStart); - } else { - skipOffset = -1; - } - - pendingTerms.add(new PendingTerm(freqStart, proxStart, skipOffset)); - - lastDocID = 0; - df = 0; - } - - private final RAMOutputStream bytesWriter = new RAMOutputStream(); - - @Override - public void flushTermsBlock(int start, int count) throws IOException { - //if (DEBUG) System.out.println("SPW: flushTermsBlock start=" + start + " count=" + count + " left=" + (pendingTerms.size()-count) + " pendingTerms.size()=" + pendingTerms.size()); - - if (count == 0) { - termsOut.writeByte((byte) 0); - return; - } - - assert start <= pendingTerms.size(); - assert count <= start; - - final int limit = pendingTerms.size() - start + count; - final PendingTerm firstTerm = pendingTerms.get(limit - count); - // First term in block is abs coded: - bytesWriter.writeVLong(firstTerm.freqStart); - - if (firstTerm.skipOffset != -1) { - assert firstTerm.skipOffset > 0; - bytesWriter.writeVInt(firstTerm.skipOffset); - } - if (indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) { - bytesWriter.writeVLong(firstTerm.proxStart); - } - long lastFreqStart = firstTerm.freqStart; - long lastProxStart = firstTerm.proxStart; - for(int idx=limit-count+1; idxtrue iff the last committed version differs from the * current version, otherwise false * @@ -198,7 +184,6 @@ private final SortedMap byNumber = new TreeMap(); private final HashMap byName = new HashMap(); private final FieldNumberBiMap globalFieldNumbers; - private final SegmentCodecsBuilder segmentCodecsBuilder; // First used in 2.9; prior to 2.9 there was no format header public static final int FORMAT_START = -2; @@ -230,16 +215,15 @@ /** * Creates a new {@link FieldInfos} instance with a private - * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} and a default {@link SegmentCodecsBuilder} - * initialized with {@link CodecProvider#getDefault()}. + * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} *

* Note: this ctor should not be used during indexing use * {@link FieldInfos#FieldInfos(FieldInfos)} or - * {@link FieldInfos#FieldInfos(FieldNumberBiMap,org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder)} + * {@link FieldInfos#FieldInfos(FieldNumberBiMap)} * instead. */ public FieldInfos() { - this(new FieldNumberBiMap(), SegmentCodecsBuilder.create(CodecProvider.getDefault())); + this(new FieldNumberBiMap()); } /** @@ -249,7 +233,7 @@ * @see #isReadOnly() */ FieldInfos(FieldInfos other) { - this(other.globalFieldNumbers, other.segmentCodecsBuilder); + this(other.globalFieldNumbers); } /** @@ -257,9 +241,8 @@ * If the {@link FieldNumberBiMap} is null this instance will be read-only. * @see #isReadOnly() */ - FieldInfos(FieldNumberBiMap globalFieldNumbers, SegmentCodecsBuilder segmentCodecsBuilder) { + FieldInfos(FieldNumberBiMap globalFieldNumbers) { this.globalFieldNumbers = globalFieldNumbers; - this.segmentCodecsBuilder = segmentCodecsBuilder; } /** @@ -273,7 +256,7 @@ * @throws IOException */ public FieldInfos(Directory d, String name) throws IOException { - this((FieldNumberBiMap)null, null); // use null here to make this FIs Read-Only + this((FieldNumberBiMap)null); // use null here to make this FIs Read-Only final IndexInput input = d.openInput(name, IOContext.READONCE); try { read(input, name); @@ -309,7 +292,7 @@ */ @Override synchronized public Object clone() { - FieldInfos fis = new FieldInfos(globalFieldNumbers, segmentCodecsBuilder); + FieldInfos fis = new FieldInfos(globalFieldNumbers); fis.format = format; fis.hasFreq = hasFreq; fis.hasProx = hasProx; @@ -468,7 +451,6 @@ if (globalFieldNumbers == null) { throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos"); } - assert segmentCodecsBuilder != null : "SegmentCodecsBuilder is set to null but FieldInfos is not read-only"; FieldInfo fi = fieldInfo(name); if (fi == null) { final int fieldNumber = nextFieldNumber(name, preferredFieldNumber); @@ -477,9 +459,6 @@ fi.update(isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions); fi.setDocValues(docValues); } - if ((fi.isIndexed || fi.hasDocValues()) && fi.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) { - segmentCodecsBuilder.tryAddAndSet(fi); - } version++; return fi; } @@ -569,22 +548,6 @@ } return false; } - - /** - * Builds the {@link SegmentCodecs} mapping for this {@link FieldInfos} instance. - * @param clearBuilder true iff the internal {@link SegmentCodecsBuilder} must be cleared otherwise false - */ - public SegmentCodecs buildSegmentCodecs(boolean clearBuilder) { - if (globalFieldNumbers == null) { - throw new IllegalStateException("FieldInfos are read-only no SegmentCodecs available"); - } - assert segmentCodecsBuilder != null; - final SegmentCodecs segmentCodecs = segmentCodecsBuilder.build(); - if (clearBuilder) { - segmentCodecsBuilder.clear(); - } - return segmentCodecs; - } public void write(Directory d, String name) throws IOException { IndexOutput output = d.createOutput(name, IOContext.READONCE); @@ -628,7 +591,6 @@ bits |= OMIT_POSITIONS; output.writeString(fi.name); output.writeInt(fi.number); - output.writeInt(fi.getCodecId()); output.writeByte(bits); final byte b; @@ -698,9 +660,7 @@ for (int i = 0; i < size; i++) { String name = input.readString(); - // if this is a previous format codec 0 will be preflex! final int fieldNumber = format <= FORMAT_FLEX? input.readInt():i; - final int codecId = format <= FORMAT_FLEX? input.readInt():0; byte bits = input.readByte(); boolean isIndexed = (bits & IS_INDEXED) != 0; boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0; @@ -781,8 +741,7 @@ throw new IllegalStateException("unhandled indexValues type " + b); } } - final FieldInfo addInternal = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType); - addInternal.setCodecId(codecId); + addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType); } if (input.getFilePointer() != input.length()) { @@ -804,7 +763,7 @@ if (isReadOnly()) { return this; } - final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null, null); + final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null); for (FieldInfo fieldInfo : this) { FieldInfo clone = (FieldInfo) (fieldInfo).clone(); roFis.putInternal(clone); @@ -814,5 +773,14 @@ } return roFis; } - + + public boolean anyDocValuesFields() { + for (FieldInfo fi : this) { + if (fi.hasDocValues()) { + return true; + } + } + + return false; + } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java 2011-08-29 12:22:46.861459551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java 2011-10-26 22:26:18.900874635 -0400 @@ -57,7 +57,7 @@ // Sort by field name CollectionUtil.quickSort(allFields); - final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state); + final FieldsConsumer consumer = state.codec.postingsFormat().fieldsConsumer(state); boolean success = false; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java lucene-2621/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java 2011-09-08 17:01:39.526036271 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java 2011-11-01 21:10:35.944609906 -0400 @@ -18,7 +18,6 @@ */ import java.io.FileNotFoundException; -import java.io.FilenameFilter; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; @@ -29,7 +28,6 @@ import java.util.List; import java.util.Map; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NoSuchDirectoryException; import org.apache.lucene.util.CollectionUtil; @@ -122,8 +120,6 @@ infoStream.println("IFD [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message); } - private final FilenameFilter indexFilenameFilter; - // called only from assert private boolean locked() { return writer == null || Thread.holdsLock(writer); @@ -138,7 +134,7 @@ * @throws IOException if there is a low-level IO error */ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, - PrintStream infoStream, CodecProvider codecs, IndexWriter writer) throws CorruptIndexException, IOException { + PrintStream infoStream, IndexWriter writer) throws CorruptIndexException, IOException { this.infoStream = infoStream; this.writer = writer; @@ -154,7 +150,6 @@ // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.getGeneration(); - indexFilenameFilter = new IndexFileNameFilter(codecs); CommitPoint currentCommitPoint = null; String[] files = null; @@ -167,7 +162,7 @@ for (String fileName : files) { - if ((indexFilenameFilter.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) { + if ((IndexFileNameFilter.INSTANCE.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: getRefCount(fileName); @@ -180,9 +175,9 @@ if (infoStream != null) { message("init: load commit \"" + fileName + "\""); } - SegmentInfos sis = new SegmentInfos(codecs); + SegmentInfos sis = new SegmentInfos(); try { - sis.read(directory, fileName, codecs); + sis.read(directory, fileName); } catch (FileNotFoundException e) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth @@ -253,9 +248,9 @@ // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: - SegmentInfos sis = new SegmentInfos(codecs); + SegmentInfos sis = new SegmentInfos(); try { - sis.read(directory, currentSegmentsFile, codecs); + sis.read(directory, currentSegmentsFile); } catch (IOException e) { throw new CorruptIndexException("failed to locate current segments_N file"); } @@ -373,7 +368,7 @@ for(int i=0;iThis does not accept *-write.lock files. * * @lucene.internal */ public class IndexFileNameFilter implements FilenameFilter { - private final HashSet extensions; - - public IndexFileNameFilter(CodecProvider codecs) { - extensions = new HashSet(); - for (String ext : IndexFileNames.INDEX_EXTENSIONS) { - extensions.add(ext); - } - if (codecs != null) { - for(String ext : codecs.getAllExtensions()) { - extensions.add(ext); - } - } + public static final FilenameFilter INSTANCE = new IndexFileNameFilter(); + + private IndexFileNameFilter() { } + // Approximate match for files that seem to be Lucene + // index files. This can easily over-match, ie if some + // app names a file _foo_bar.go: + private final Pattern luceneFilePattern = Pattern.compile("^_[a-z0-9]+(_[a-z0-9]+)?\\.[a-z0-9]+$"); + /* (non-Javadoc) * @see java.io.FilenameFilter#accept(java.io.File, java.lang.String) */ public boolean accept(File dir, String name) { - int i = name.lastIndexOf('.'); - if (i != -1) { - String extension = name.substring(1+i); - if (extensions.contains(extension)) { - return true; - } else if (extension.startsWith("f") && - extension.matches("f\\d+")) { - return true; - } else if (extension.startsWith("s") && - extension.matches("s\\d+")) { - return true; - } + if (name.lastIndexOf('.') != -1) { + // Has an extension + return luceneFilePattern.matcher(name).matches(); } else { - if (name.startsWith(IndexFileNames.SEGMENTS)) return true; + // No extension -- only segments_N file; + return name.startsWith(IndexFileNames.SEGMENTS); } - return false; } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexFileNames.java lucene-2621/lucene/src/java/org/apache/lucene/index/IndexFileNames.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexFileNames.java 2011-08-20 13:34:24.551460633 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/IndexFileNames.java 2011-11-03 13:43:22.052534709 -0400 @@ -19,7 +19,9 @@ import java.util.regex.Pattern; -import org.apache.lucene.index.codecs.Codec; // for javadocs +import org.apache.lucene.index.codecs.PostingsFormat; // for javadocs + +// TODO: put all files under codec and remove all the static extensions here /** * This class contains useful constants representing filenames and extensions @@ -31,7 +33,7 @@ * {@link #segmentFileName(String, String, String) segmentFileName}). * *

NOTE: extensions used by codecs are not - * listed here. You must interact with the {@link Codec} + * listed here. You must interact with the {@link PostingsFormat} * directly. * * @lucene.internal @@ -188,20 +190,20 @@ * NOTE: .<ext> is added to the result file name only if * ext is not empty. *

- * NOTE: _<name> is added to the result file name only if - * name is not empty. + * NOTE: _<segmentSuffix> is added to the result file name only if + * it's not the empty string *

* NOTE: all custom files should be named using this method, or * otherwise some structures may fail to handle them properly (such as if they * are added to compound files). */ - public static String segmentFileName(String segmentName, String name, String ext) { - if (ext.length() > 0 || name.length() > 0) { + public static String segmentFileName(String segmentName, String segmentSuffix, String ext) { + if (ext.length() > 0 || segmentSuffix.length() > 0) { assert !ext.startsWith("."); - StringBuilder sb = new StringBuilder(segmentName.length() + 2 + name.length() + ext.length()); + StringBuilder sb = new StringBuilder(segmentName.length() + 2 + segmentSuffix.length() + ext.length()); sb.append(segmentName); - if (name.length() > 0) { - sb.append('_').append(name); + if (segmentSuffix.length() > 0) { + sb.append('_').append(segmentSuffix); } if (ext.length() > 0) { sb.append('.').append(ext); @@ -212,11 +214,6 @@ } } - /** Sugar for passing "" + name instead */ - public static String segmentFileName(String segmentName, int name, String ext) { - return segmentFileName(segmentName, ""+name, ext); - } - /** * Returns true if the given filename ends with the given extension. One * should provide a pure extension, without '.'. diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java lucene-2621/lucene/src/java/org/apache/lucene/index/IndexReader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexReader.java 2011-11-03 08:31:17.328544356 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/IndexReader.java 2011-11-03 13:27:29.764535198 -0400 @@ -28,8 +28,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.DocumentStoredFieldVisitor; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.values.IndexDocValues; import org.apache.lucene.search.FieldCache; // javadocs @@ -312,7 +311,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory) throws CorruptIndexException, IOException { - return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR); } /** Returns an IndexReader reading the index in the given @@ -326,9 +325,9 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } - + /** * Open a near real time IndexReader from the {@link org.apache.lucene.index.IndexWriter}. * @@ -363,7 +362,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } /** Expert: returns an IndexReader reading the index in @@ -381,7 +380,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } /** Expert: returns an IndexReader reading the index in @@ -409,7 +408,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, null); + return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor); } /** Expert: returns an IndexReader reading the index in @@ -429,7 +428,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } /** Expert: returns an IndexReader reading the index in @@ -462,78 +461,11 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, null); - } - - /** Expert: returns an IndexReader reading the index in - * the given Directory, with a custom {@link - * IndexDeletionPolicy}, and specified {@link CodecProvider}. - * You should pass readOnly=true, since it gives much - * better concurrent performance, unless you intend to do - * write operations (delete documents or change norms) - * with the reader. - * @param directory the index directory - * @param deletionPolicy a custom deletion policy (only used - * if you use this reader to perform deletes or to set - * norms); see {@link IndexWriter} for details. - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader - * @param termInfosIndexDivisor Subsamples which indexed - * terms are loaded into RAM. This has the same effect as {@link - * IndexWriterConfig#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1. Set this - * to -1 to skip loading the terms index entirely. - * @param codecs CodecProvider to use when opening index - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, codecs); + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor); } - /** Expert: returns an IndexReader reading the index in - * the given Directory, using a specific commit and with - * a custom {@link IndexDeletionPolicy} and specified - * {@link CodecProvider}. You should pass readOnly=true, since - * it gives much better concurrent performance, unless - * you intend to do write operations (delete documents or - * change norms) with the reader. - - * @param commit the specific {@link IndexCommit} to open; - * see {@link IndexReader#listCommits} to list all commits - * in a directory - * @param deletionPolicy a custom deletion policy (only used - * if you use this reader to perform deletes or to set - * norms); see {@link IndexWriter} for details. - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader - * @param termInfosIndexDivisor Subsamples which indexed - * terms are loaded into RAM. This has the same effect as {@link - * IndexWriterConfig#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1. Set this - * to -1 to skip loading the terms index entirely. - * @param codecs CodecProvider to use when opening index - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs); - } - - private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor, - CodecProvider codecs) throws CorruptIndexException, IOException { - if (codecs == null) { - codecs = CodecProvider.getDefault(); - } - return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs); + private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { + return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor); } /** @@ -767,20 +699,6 @@ } }.run()).longValue(); } - - /** - * Reads version number from segments files. The version number is - * initialized with a timestamp and then increased by one for each change of - * the index. - * - * @param directory where the index resides. - * @return version number. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static long getCurrentVersion(Directory directory) throws CorruptIndexException, IOException { - return getCurrentVersion(directory, CodecProvider.getDefault()); - } /** * Reads version number from segments files. The version number is @@ -788,34 +706,14 @@ * the index. * * @param directory where the index resides. - * @param codecs the {@link CodecProvider} holding all {@link Codec}s required to open the index * @return version number. * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static long getCurrentVersion(Directory directory, CodecProvider codecs) throws CorruptIndexException, IOException { - return SegmentInfos.readCurrentVersion(directory, codecs); - } - - /** - * Reads commitUserData, previously passed to {@link - * IndexWriter#commit(Map)}, from current index - * segments file. This will return null if {@link - * IndexWriter#commit(Map)} has never been called for - * this index. - * - * @param directory where the index resides. - * @return commit userData. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - * - * @see #getCommitUserData() - */ - public static Map getCommitUserData(Directory directory) throws CorruptIndexException, IOException { - return getCommitUserData(directory, CodecProvider.getDefault()); + public static long getCurrentVersion(Directory directory) throws CorruptIndexException, IOException { + return SegmentInfos.readCurrentVersion(directory); } - /** * Reads commitUserData, previously passed to {@link * IndexWriter#commit(Map)}, from current index @@ -824,15 +722,14 @@ * this index. * * @param directory where the index resides. - * @param codecs the {@link CodecProvider} provider holding all {@link Codec}s required to open the index * @return commit userData. * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error * * @see #getCommitUserData() */ - public static Map getCommitUserData(Directory directory, CodecProvider codecs) throws CorruptIndexException, IOException { - return SegmentInfos.readCurrentUserData(directory, codecs); + public static Map getCommitUserData(Directory directory) throws CorruptIndexException, IOException { + return SegmentInfos.readCurrentUserData(directory); } /** @@ -982,22 +879,6 @@ return true; } catch (IOException ioe) { return false; - } - } - - /** - * Returns true if an index exists at the specified directory. - * @param directory the directory to check for an index - * @param codecProvider provides a CodecProvider in case the index uses non-core codecs - * @return true if an index exists; false otherwise - * @throws IOException if there is a problem with accessing the index - */ - public static boolean indexExists(Directory directory, CodecProvider codecProvider) throws IOException { - try { - new SegmentInfos().read(directory, codecProvider); - return true; - } catch (IOException ioe) { - return false; } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java lucene-2621/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java 2011-07-24 19:00:25.772328885 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/IndexUpgrader.java 2011-11-01 21:10:35.940609908 -0400 @@ -114,12 +114,12 @@ } public void upgrade() throws IOException { - if (!IndexReader.indexExists(dir, iwc.getCodecProvider())) { + if (!IndexReader.indexExists(dir)) { throw new IndexNotFoundException(dir.toString()); } if (!deletePriorCommits) { - final Collection commits = DirectoryReader.listCommits(dir, iwc.getCodecProvider()); + final Collection commits = DirectoryReader.listCommits(dir); if (commits.size() > 1) { throw new IllegalArgumentException("This tool was invoked to not delete prior commit points, but the following commits were found: " + commits); } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java lucene-2621/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java 2011-09-15 13:19:31.226254551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java 2011-11-01 21:10:35.944609906 -0400 @@ -20,7 +20,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.util.Version; @@ -121,7 +121,7 @@ private volatile int maxBufferedDocs; private volatile IndexingChain indexingChain; private volatile IndexReaderWarmer mergedSegmentWarmer; - private volatile CodecProvider codecProvider; + private volatile Codec codec; private volatile MergePolicy mergePolicy; private volatile DocumentsWriterPerThreadPool indexerThreadPool; private volatile boolean readerPooling; @@ -158,7 +158,7 @@ maxBufferedDocs = DEFAULT_MAX_BUFFERED_DOCS; indexingChain = DocumentsWriterPerThread.defaultIndexingChain; mergedSegmentWarmer = null; - codecProvider = CodecProvider.getDefault(); + codec = Codec.getDefault(); if (matchVersion.onOrAfter(Version.LUCENE_32)) { mergePolicy = new TieredMergePolicy(); } else { @@ -521,17 +521,17 @@ return this; } - /** Set the CodecProvider. See {@link CodecProvider}. + /** Set the Codec. See {@link Codec}. * *

Only takes effect when IndexWriter is first created. */ - public IndexWriterConfig setCodecProvider(CodecProvider codecProvider) { - this.codecProvider = codecProvider; + public IndexWriterConfig setCodec(Codec codec) { + this.codec = codec; return this; } - /** Returns the current merged segment warmer. See {@link IndexReaderWarmer}. */ - public CodecProvider getCodecProvider() { - return codecProvider; + /** Returns the current Codec. See {@link Codec}. */ + public Codec getCodec() { + return codec; } @@ -694,7 +694,7 @@ sb.append("ramBufferSizeMB=").append(ramBufferSizeMB).append("\n"); sb.append("maxBufferedDocs=").append(maxBufferedDocs).append("\n"); sb.append("mergedSegmentWarmer=").append(mergedSegmentWarmer).append("\n"); - sb.append("codecProvider=").append(codecProvider).append("\n"); + sb.append("codec=").append(codec).append("\n"); sb.append("mergePolicy=").append(mergePolicy).append("\n"); sb.append("indexerThreadPool=").append(indexerThreadPool).append("\n"); sb.append("readerPooling=").append(readerPooling).append("\n"); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/IndexWriter.java 2011-10-23 17:07:48.681018015 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/IndexWriter.java 2011-11-03 13:24:53.440535280 -0400 @@ -39,8 +39,7 @@ import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor; -import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.Query; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.CompoundFileDirectory; @@ -375,7 +374,7 @@ // just like we do when loading segments_N synchronized(this) { maybeApplyDeletes(applyAllDeletes); - r = new DirectoryReader(this, segmentInfos, codecs, applyAllDeletes); + r = new DirectoryReader(this, segmentInfos, applyAllDeletes); if (infoStream != null) { message("return reader version=" + r.getVersion() + " reader=" + r); } @@ -802,7 +801,7 @@ infoStream.println("IW " + messageID + " [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message); } - CodecProvider codecs; + final Codec codec; // for writing new segments /** * Constructs a new IndexWriter per the settings given in conf. @@ -837,7 +836,7 @@ mergePolicy = conf.getMergePolicy(); mergePolicy.setIndexWriter(this); mergeScheduler = conf.getMergeScheduler(); - codecs = conf.getCodecProvider(); + codec = conf.getCodec(); bufferedDeletesStream = new BufferedDeletesStream(messageID); bufferedDeletesStream.setInfoStream(infoStream); @@ -862,7 +861,7 @@ // If index is too old, reading the segments will throw // IndexFormatTooOldException. - segmentInfos = new SegmentInfos(codecs); + segmentInfos = new SegmentInfos(); try { if (create) { // Try to read first. This is to allow create @@ -870,7 +869,7 @@ // searching. In this case we write the next // segments_N file with no segments: try { - segmentInfos.read(directory, codecs); + segmentInfos.read(directory); segmentInfos.clear(); } catch (IOException e) { // Likely this means it's a fresh directory @@ -881,7 +880,7 @@ changeCount++; segmentInfos.changed(); } else { - segmentInfos.read(directory, codecs); + segmentInfos.read(directory); IndexCommit commit = conf.getIndexCommit(); if (commit != null) { @@ -892,8 +891,8 @@ // points. if (commit.getDirectory() != directory) throw new IllegalArgumentException("IndexCommit's directory doesn't match my directory"); - SegmentInfos oldInfos = new SegmentInfos(codecs); - oldInfos.read(directory, commit.getSegmentsFileName(), codecs); + SegmentInfos oldInfos = new SegmentInfos(); + oldInfos.read(directory, commit.getSegmentsFileName()); segmentInfos.replace(oldInfos); changeCount++; segmentInfos.changed(); @@ -906,7 +905,7 @@ // start with previous field numbers, but new FieldInfos globalFieldNumberMap = segmentInfos.getOrLoadGlobalFieldNumberMap(directory); - docWriter = new DocumentsWriter(config, directory, this, globalFieldNumberMap, bufferedDeletesStream); + docWriter = new DocumentsWriter(codec, config, directory, this, globalFieldNumberMap, bufferedDeletesStream); docWriter.setInfoStream(infoStream); // Default deleter (for backwards compatibility) is @@ -914,8 +913,7 @@ synchronized(this) { deleter = new IndexFileDeleter(directory, conf.getIndexDeletionPolicy(), - segmentInfos, infoStream, - codecs, this); + segmentInfos, infoStream, this); } if (deleter.startingCommitDeleted) { @@ -2149,6 +2147,7 @@ */ public synchronized void deleteAll() throws IOException { ensureOpen(); + boolean success = false; try { // Abort any running merges @@ -2170,10 +2169,11 @@ // Mark that the index has changed ++changeCount; segmentInfos.changed(); + success = true; } catch (OutOfMemoryError oom) { handleOOM(oom, "deleteAll"); } finally { - if (infoStream != null) { + if (!success && infoStream != null) { message("hit exception during deleteAll"); } } @@ -2476,8 +2476,8 @@ if (infoStream != null) { message("addIndexes: process directory " + dir); } - SegmentInfos sis = new SegmentInfos(codecs); // read infos from dir - sis.read(dir, codecs); + SegmentInfos sis = new SegmentInfos(); // read infos from dir + sis.read(dir); final Set dsFilesCopied = new HashSet(); final Map dsNames = new HashMap(); for (SegmentInfo info : sis) { @@ -2567,7 +2567,7 @@ // abortable so that IW.close(false) is able to stop it SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, null, payloadProcessorProvider, - globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)), context); + new FieldInfos(globalFieldNumberMap), codec, context); for (IndexReader reader : readers) // add new indexes merger.add(reader); @@ -2575,7 +2575,7 @@ final FieldInfos fieldInfos = merger.fieldInfos(); SegmentInfo info = new SegmentInfo(mergedName, docCount, directory, - false, merger.getSegmentCodecs(), + false, merger.getCodec(), fieldInfos); setDiagnostics(info, "addIndexes(IndexReader...)"); @@ -2591,7 +2591,7 @@ // Now create the compound file if needed if (useCompoundFile) { - merger.createCompoundFile(mergedName + ".cfs", info, context); + merger.createCompoundFile(IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_EXTENSION), info, context); // delete new non cfs files directly: they were never // registered with IFD @@ -2916,7 +2916,7 @@ try { if (infoStream != null) message("commit: pendingCommit != null"); - pendingCommit.finishCommit(directory); + pendingCommit.finishCommit(directory, codec); if (infoStream != null) message("commit: wrote segments file \"" + pendingCommit.getCurrentSegmentFileName() + "\""); lastCommitChangeCount = pendingCommitChangeCount; @@ -3459,7 +3459,7 @@ // Bind a new segment name here so even with // ConcurrentMergePolicy we keep deterministic segment // names. - merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs))); + merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, new FieldInfos(globalFieldNumberMap)); // Lock order: IW -> BD final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments); @@ -3633,7 +3633,7 @@ IOContext context = new IOContext(merge.getMergeInfo()); SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge, - payloadProcessorProvider, merge.info.getFieldInfos(), context); + payloadProcessorProvider, merge.info.getFieldInfos(), codec, context); if (infoStream != null) { message("merging " + merge.segString(directory) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors()); @@ -3679,10 +3679,10 @@ mergedDocCount = merge.info.docCount = merger.merge(); // Record which codec was used to write the segment - merge.info.setSegmentCodecs(merger.getSegmentCodecs()); + merge.info.setCodec(merger.getCodec()); if (infoStream != null) { - message("merge segmentCodecs=" + merger.getSegmentCodecs()); + message("merge codecs=" + merger.getCodec()); message("merge store matchedCount=" + merger.getMatchedSubReaderCount() + " vs " + merge.readers.size()); } anyNonBulkMerges |= merger.getAnyNonBulkMerges(); @@ -3975,7 +3975,7 @@ // Exception here means nothing is prepared // (this method unwinds everything it did on // an exception) - toSync.prepareCommit(directory); + toSync.prepareCommit(directory, codec); pendingCommitSet = true; pendingCommit = toSync; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java lucene-2621/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java 2011-09-08 17:01:39.526036271 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/PerDocWriteState.java 2011-11-02 21:26:22.548564906 -0400 @@ -34,41 +34,37 @@ public final String segmentName; public final FieldInfos fieldInfos; public final Counter bytesUsed; - public final SegmentCodecs segmentCodecs; - public final int codecId; + public final String segmentSuffix; public final IOContext context; - PerDocWriteState(PrintStream infoStream, Directory directory, + public PerDocWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos, Counter bytesUsed, - int codecId, IOContext context) { + String segmentSuffix, IOContext context) { this.infoStream = infoStream; this.directory = directory; this.segmentName = segmentName; this.fieldInfos = fieldInfos; - this.segmentCodecs = fieldInfos.buildSegmentCodecs(false); - this.codecId = codecId; + this.segmentSuffix = segmentSuffix; this.bytesUsed = bytesUsed; this.context = context; } - PerDocWriteState(SegmentWriteState state) { + public PerDocWriteState(SegmentWriteState state) { infoStream = state.infoStream; directory = state.directory; - segmentCodecs = state.segmentCodecs; segmentName = state.segmentName; fieldInfos = state.fieldInfos; - codecId = state.codecId; + segmentSuffix = state.segmentSuffix; bytesUsed = Counter.newCounter(); context = state.context; } - PerDocWriteState(PerDocWriteState state, int codecId) { + public PerDocWriteState(PerDocWriteState state, String segmentSuffix) { this.infoStream = state.infoStream; this.directory = state.directory; this.segmentName = state.segmentName; this.fieldInfos = state.fieldInfos; - this.segmentCodecs = state.segmentCodecs; - this.codecId = codecId; + this.segmentSuffix = segmentSuffix; this.bytesUsed = state.bytesUsed; this.context = state.context; } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java lucene-2621/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java 2011-08-29 12:22:46.861459551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,286 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; -import java.util.TreeSet; - -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.index.codecs.TermsConsumer; -import org.apache.lucene.index.codecs.DocValuesConsumer; -import org.apache.lucene.index.values.IndexDocValues; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.util.IOUtils; - -/** - * Enables native per field codec support. This class selects the codec used to - * write a field depending on the provided {@link SegmentCodecs}. For each field - * seen it resolves the codec based on the {@link FieldInfo#codecId} which is - * only valid during a segment merge. See {@link SegmentCodecs} javadoc for - * details. - * - * @lucene.internal - */ -final class PerFieldCodecWrapper extends Codec { - private final SegmentCodecs segmentCodecs; - - PerFieldCodecWrapper(SegmentCodecs segmentCodecs) { - super("PerField"); - this.segmentCodecs = segmentCodecs; - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) - throws IOException { - return new FieldsWriter(state); - } - - private class FieldsWriter extends FieldsConsumer { - private final ArrayList consumers = new ArrayList(); - - public FieldsWriter(SegmentWriteState state) throws IOException { - assert segmentCodecs == state.segmentCodecs; - final Codec[] codecs = segmentCodecs.codecs; - for (int i = 0; i < codecs.length; i++) { - boolean success = false; - try { - consumers.add(codecs[i].fieldsConsumer(new SegmentWriteState(state, i))); - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(consumers); - } - } - } - } - - @Override - public TermsConsumer addField(FieldInfo field) throws IOException { - assert field.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID; - final FieldsConsumer fields = consumers.get(field.getCodecId()); - return fields.addField(field); - } - - @Override - public void close() throws IOException { - IOUtils.close(consumers); - } - } - - private class FieldsReader extends FieldsProducer { - - private final Set fields = new TreeSet(); - private final Map codecs = new HashMap(); - - public FieldsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo si, - IOContext context, int indexDivisor) throws IOException { - - final Map producers = new HashMap(); - boolean success = false; - try { - for (FieldInfo fi : fieldInfos) { - if (fi.isIndexed) { - fields.add(fi.name); - assert fi.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID; - Codec codec = segmentCodecs.codecs[fi.getCodecId()]; - if (!producers.containsKey(codec)) { - producers.put(codec, codec.fieldsProducer(new SegmentReadState(dir, - si, fieldInfos, context, indexDivisor, fi.getCodecId()))); - } - codecs.put(fi.name, producers.get(codec)); - } - } - success = true; - } finally { - if (!success) { - // If we hit exception (eg, IOE because writer was - // committing, or, for any other reason) we must - // go back and close all FieldsProducers we opened: - IOUtils.closeWhileHandlingException(producers.values()); - } - } - } - - - private final class FieldsIterator extends FieldsEnum { - private final Iterator it; - private String current; - - public FieldsIterator() { - it = fields.iterator(); - } - - @Override - public String next() { - if (it.hasNext()) { - current = it.next(); - } else { - current = null; - } - - return current; - } - - @Override - public TermsEnum terms() throws IOException { - Terms terms = codecs.get(current).terms(current); - if (terms != null) { - return terms.iterator(); - } else { - return TermsEnum.EMPTY; - } - } - } - - @Override - public FieldsEnum iterator() throws IOException { - return new FieldsIterator(); - } - - @Override - public Terms terms(String field) throws IOException { - FieldsProducer fields = codecs.get(field); - return fields == null ? null : fields.terms(field); - } - - @Override - public void close() throws IOException { - IOUtils.close(codecs.values()); - } - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) - throws IOException { - return new FieldsReader(state.dir, state.fieldInfos, state.segmentInfo, - state.context, state.termsIndexDivisor); - } - - @Override - public void files(Directory dir, SegmentInfo info, int codecId, Set files) - throws IOException { - // ignore codecid since segmentCodec will assign it per codec - segmentCodecs.files(dir, info, files); - } - - @Override - public void getExtensions(Set extensions) { - for (Codec codec : segmentCodecs.codecs) { - codec.getExtensions(extensions); - } - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new PerDocConsumers(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new PerDocProducers(state.dir, state.fieldInfos, state.segmentInfo, - state.context, state.termsIndexDivisor); - } - - private final class PerDocProducers extends PerDocValues { - private final TreeMap codecs = new TreeMap(); - - public PerDocProducers(Directory dir, FieldInfos fieldInfos, SegmentInfo si, - IOContext context, int indexDivisor) throws IOException { - final Map producers = new HashMap(); - boolean success = false; - try { - for (FieldInfo fi : fieldInfos) { - if (fi.hasDocValues()) { - assert fi.getCodecId() != FieldInfo.UNASSIGNED_CODEC_ID; - Codec codec = segmentCodecs.codecs[fi.getCodecId()]; - if (!producers.containsKey(codec)) { - producers.put(codec, codec.docsProducer(new SegmentReadState(dir, - si, fieldInfos, context, indexDivisor, fi.getCodecId()))); - } - codecs.put(fi.name, producers.get(codec)); - } - } - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(producers.values()); - } - } - } - - @Override - public Collection fields() { - return codecs.keySet(); - } - @Override - public IndexDocValues docValues(String field) throws IOException { - final PerDocValues perDocProducer = codecs.get(field); - if (perDocProducer == null) { - return null; - } - return perDocProducer.docValues(field); - } - - public void close() throws IOException { - IOUtils.close(codecs.values()); - } - } - - private final class PerDocConsumers extends PerDocConsumer { - private final PerDocConsumer[] consumers; - private final Codec[] codecs; - private final PerDocWriteState state; - - public PerDocConsumers(PerDocWriteState state) throws IOException { - assert segmentCodecs == state.segmentCodecs; - this.state = state; - codecs = segmentCodecs.codecs; - consumers = new PerDocConsumer[codecs.length]; - } - - public void close() throws IOException { - IOUtils.close(consumers); - } - - @Override - public DocValuesConsumer addValuesField(FieldInfo field) throws IOException { - final int codecId = field.getCodecId(); - assert codecId != FieldInfo.UNASSIGNED_CODEC_ID; - PerDocConsumer perDoc = consumers[codecId]; - if (perDoc == null) { - perDoc = codecs[codecId].docsConsumer(new PerDocWriteState(state, codecId)); - assert perDoc != null; - consumers[codecId] = perDoc; - } - return perDoc.addValuesField(field); - } - - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java 2011-10-25 12:50:53.544936931 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentCodecs.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,167 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.IdentityHashMap; -import java.util.Map; -import java.util.Set; - -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; - -/** - * SegmentCodecs maintains an ordered list of distinct codecs used within a - * segment. Within a segment on codec is used to write multiple fields while - * each field could be written by a different codec. To enable codecs per field - * within a single segment we need to record the distinct codecs and map them to - * each field present in the segment. SegmentCodecs is created together with - * {@link SegmentWriteState} for each flush and is maintained in the - * corresponding {@link SegmentInfo} until it is committed. - *

- * During indexing {@link FieldInfos} uses {@link SegmentCodecsBuilder} to incrementally - * build the {@link SegmentCodecs} mapping. Once a segment is flushed - * DocumentsWriter creates a {@link SegmentCodecs} instance from - * {@link FieldInfos#buildSegmentCodecs(boolean)} The {@link FieldInfo#codecId} - * assigned by {@link SegmentCodecsBuilder} refers to the codecs ordinal - * maintained inside {@link SegmentCodecs}. This ord is later used to get the - * right codec when the segment is opened in a reader.The {@link Codec} returned - * from {@link SegmentCodecs#codec()} in turn uses {@link SegmentCodecs} - * internal structure to select and initialize the right codec for a fields when - * it is written. - *

- * Once a flush succeeded the {@link SegmentCodecs} is maintained inside the - * {@link SegmentInfo} for the flushed segment it was created for. - * {@link SegmentInfo} writes the name of each codec in {@link SegmentCodecs} - * for each segment and maintains the order. Later if a segment is opened by a - * reader this mapping is deserialized and used to create the codec per field. - * - * - * @lucene.internal - */ -final class SegmentCodecs implements Cloneable { - /** - * internal structure to map codecs to fields - don't modify this from outside - * of this class! - */ - final Codec[] codecs; - final CodecProvider provider; - private final Codec codec; - - SegmentCodecs(CodecProvider provider, IndexInput input) throws IOException { - this(provider, read(input, provider)); - } - - SegmentCodecs(CodecProvider provider, Codec... codecs) { - this.provider = provider; - this.codecs = codecs; - if (codecs.length == 1 && codecs[0] instanceof PreFlexCodec) { - this.codec = codecs[0]; // hack for backwards break... don't wrap the codec in preflex - } else { - this.codec = new PerFieldCodecWrapper(this); - } - } - - Codec codec() { - return codec; - } - - void write(IndexOutput out) throws IOException { - out.writeVInt(codecs.length); - for (Codec codec : codecs) { - out.writeString(codec.name); - } - } - - private static Codec[] read(IndexInput in, CodecProvider provider) throws IOException { - final int size = in.readVInt(); - final ArrayList list = new ArrayList(); - for (int i = 0; i < size; i++) { - final String codecName = in.readString(); - final Codec lookup = provider.lookup(codecName); - list.add(i, lookup); - } - return list.toArray(Codec.EMPTY); - } - - void files(Directory dir, SegmentInfo info, Set files) - throws IOException { - final Codec[] codecArray = codecs; - for (int i = 0; i < codecArray.length; i++) { - codecArray[i].files(dir, info, i, files); - } - - } - - @Override - public String toString() { - return "SegmentCodecs [codecs=" + Arrays.toString(codecs) + ", provider=" + provider + "]"; - } - - /** - * Used in {@link FieldInfos} to incrementally build the codec ID mapping for - * {@link FieldInfo} instances. - *

- * Note: this class is not thread-safe - *

- * @see FieldInfo#getCodecId() - */ - final static class SegmentCodecsBuilder { - private final Map codecRegistry = new IdentityHashMap(); - private final ArrayList codecs = new ArrayList(); - private final CodecProvider provider; - - private SegmentCodecsBuilder(CodecProvider provider) { - this.provider = provider; - } - - static SegmentCodecsBuilder create(CodecProvider provider) { - return new SegmentCodecsBuilder(provider); - } - - SegmentCodecsBuilder tryAddAndSet(FieldInfo fi) { - if (fi.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) { - final Codec fieldCodec = provider.lookup(provider - .getFieldCodec(fi.name)); - Integer ord = codecRegistry.get(fieldCodec); - if (ord == null) { - ord = Integer.valueOf(codecs.size()); - codecRegistry.put(fieldCodec, ord); - codecs.add(fieldCodec); - } - fi.setCodecId(ord.intValue()); - } - return this; - } - - SegmentCodecs build() { - return new SegmentCodecs(provider, codecs.toArray(Codec.EMPTY)); - } - - SegmentCodecsBuilder clear() { - codecRegistry.clear(); - codecs.clear(); - return this; - } - } -} \ No newline at end of file diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java 2011-10-07 11:55:34.578772788 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java 2011-10-26 22:36:09.276874332 -0400 @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FieldsReader; import org.apache.lucene.index.codecs.PerDocValues; @@ -68,7 +69,7 @@ } segment = si.name; - final SegmentCodecs segmentCodecs = si.getSegmentCodecs(); + final Codec codec = si.getCodec(); this.context = context; this.dir = dir; @@ -85,12 +86,12 @@ fieldInfos = si.getFieldInfos(); this.termsIndexDivisor = termsIndexDivisor; - final Codec codec = segmentCodecs.codec(); + final PostingsFormat format = codec.postingsFormat(); final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, context, termsIndexDivisor); // Ask codec for its Fields - fields = codec.fieldsProducer(segmentReadState); + fields = format.fieldsProducer(segmentReadState); assert fields != null; - perDocProducer = codec.docsProducer(segmentReadState); + perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState); success = true; } finally { if (!success) { @@ -165,7 +166,7 @@ } final String storesSegment = si.getDocStoreSegment(); - fieldsReaderOrig = si.getSegmentCodecs().provider.fieldsReader(storeDir, storesSegment, fieldInfos, context, + fieldsReaderOrig = si.getCodec().fieldsFormat().fieldsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount); // Verify two sources of "maxDoc" agree: diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentInfo.java 2011-10-15 21:56:47.717365171 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentInfo.java 2011-11-01 23:40:37.532605269 -0400 @@ -28,7 +28,6 @@ import java.util.Set; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; @@ -97,7 +96,7 @@ private FieldInfos fieldInfos; - private SegmentCodecs segmentCodecs; + private Codec codec; private Map diagnostics; @@ -116,7 +115,7 @@ private long fieldInfosVersion; public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, - SegmentCodecs segmentCodecs, FieldInfos fieldInfos) { + Codec codec, FieldInfos fieldInfos) { this.name = name; this.docCount = docCount; this.dir = dir; @@ -124,7 +123,7 @@ this.isCompoundFile = isCompoundFile; this.docStoreOffset = -1; this.docStoreSegment = name; - this.segmentCodecs = segmentCodecs; + this.codec = codec; delCount = 0; version = Constants.LUCENE_MAIN_VERSION; this.fieldInfos = fieldInfos; @@ -156,7 +155,7 @@ } isCompoundFile = src.isCompoundFile; delCount = src.delCount; - segmentCodecs = src.segmentCodecs; + codec = src.codec; } void setDiagnostics(Map diagnostics) { @@ -177,7 +176,7 @@ * @param format format of the segments info file * @param input input handle to read segment info from */ - public SegmentInfo(Directory dir, int format, IndexInput input, CodecProvider codecs) throws IOException { + public SegmentInfo(Directory dir, int format, IndexInput input) throws IOException { this.dir = dir; if (format <= DefaultSegmentInfosWriter.FORMAT_3_1) { version = input.readString(); @@ -221,13 +220,13 @@ hasProx = input.readByte(); + // System.out.println(Thread.currentThread().getName() + ": si.read hasProx=" + hasProx + " seg=" + name); + // note: if the codec is not available: Codec.forName will throw an exception. if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { - segmentCodecs = new SegmentCodecs(codecs, input); + codec = Codec.forName(input.readString()); } else { - // codec ID on FieldInfo is 0 so it will simply use the first codec available - // TODO what todo if preflex is not available in the provider? register it or fail? - segmentCodecs = new SegmentCodecs(codecs, new Codec[] { codecs.lookup("PreFlex")}); + codec = Codec.forName("Lucene3x"); } diagnostics = input.readStringStringMap(); @@ -350,7 +349,7 @@ @Override public Object clone() { - final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, segmentCodecs, + final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, codec, fieldInfos == null ? null : (FieldInfos) fieldInfos.clone()); si.docStoreOffset = docStoreOffset; si.docStoreSegment = docStoreSegment; @@ -573,7 +572,7 @@ output.writeByte((byte) (isCompoundFile ? YES : NO)); output.writeInt(delCount); output.writeByte((byte) (hasProx)); - segmentCodecs.write(output); + output.writeString(codec.getName()); output.writeStringStringMap(diagnostics); output.writeByte((byte) (hasVectors)); } @@ -583,16 +582,16 @@ } /** Can only be called once. */ - public void setSegmentCodecs(SegmentCodecs segmentCodecs) { - assert this.segmentCodecs == null; - if (segmentCodecs == null) { + public void setCodec(Codec codec) { + assert this.codec == null; + if (codec == null) { throw new IllegalArgumentException("segmentCodecs must be non-null"); } - this.segmentCodecs = segmentCodecs; + this.codec = codec; } - SegmentCodecs getSegmentCodecs() { - return segmentCodecs; + Codec getCodec() { + return codec; } private void addIfExists(Set files, String fileName) throws IOException { @@ -628,7 +627,7 @@ for(String ext : IndexFileNames.NON_STORE_INDEX_EXTENSIONS) { addIfExists(fileSet, IndexFileNames.segmentFileName(name, "", ext)); } - segmentCodecs.files(dir, this, fileSet); + codec.files(dir, this, fileSet); } if (docStoreOffset != -1) { diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentInfos.java lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentInfos.java 2011-08-29 12:22:46.851459551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentInfos.java 2011-11-02 11:34:20.976583210 -0400 @@ -32,10 +32,11 @@ import java.util.Set; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.index.codecs.SegmentInfosReader; import org.apache.lucene.index.codecs.SegmentInfosWriter; +import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -83,8 +84,6 @@ // there was an IOException that had interrupted a commit public Map userData = Collections.emptyMap(); // Opaque Map that user can specify during IndexWriter.commit - - private CodecProvider codecs; private int format; @@ -95,20 +94,14 @@ private transient List cachedUnmodifiableList; private transient Set cachedUnmodifiableSet; + private Codec codecFormat; + /** * If non-null, information about loading segments_N files * will be printed here. @see #setInfoStream. */ private static PrintStream infoStream = null; - public SegmentInfos() { - this(CodecProvider.getDefault()); - } - - public SegmentInfos(CodecProvider codecs) { - this.codecs = codecs; - } - public void setFormat(int format) { this.format = format; } @@ -197,7 +190,7 @@ * since this file might belong to more than one segment (global map) and * could otherwise easily be confused with a per-segment file. */ - return IndexFileNames.segmentFileName(""+ version, "", IndexFileNames.GLOBAL_FIELD_NUM_MAP_EXTENSION); + return IndexFileNames.segmentFileName("_"+ version, "", IndexFileNames.GLOBAL_FIELD_NUM_MAP_EXTENSION); } /** @@ -241,9 +234,7 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public final void read(Directory directory, String segmentFileName, - CodecProvider codecs) throws CorruptIndexException, IOException { - this.codecs = codecs; + public final void read(Directory directory, String segmentFileName) throws CorruptIndexException, IOException { boolean success = false; // Clear any previous segments: @@ -253,12 +244,40 @@ lastGeneration = generation; + // TODO: scary to have default impl reopen the file... but to make it a bit more flexible, + // maybe we could use a plain indexinput here... could default impl rewind/wrap with checksumII, + // and any checksumming is then up to implementation? + ChecksumIndexInput input = null; try { - SegmentInfosReader infosReader = codecs.getSegmentInfosReader(); - infosReader.read(directory, segmentFileName, codecs, this, IOContext.READ); + input = new ChecksumIndexInput(directory.openInput(segmentFileName, IOContext.READ)); + final int format = input.readInt(); + setFormat(format); + + // check that it is a format we can understand + if (format > DefaultSegmentInfosWriter.FORMAT_MINIMUM) + throw new IndexFormatTooOldException(segmentFileName, format, + DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); + if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT) + throw new IndexFormatTooNewException(segmentFileName, format, + DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); + + if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { + codecFormat = Codec.forName(input.readString()); + } else { + codecFormat = Codec.forName("Lucene3x"); + } + SegmentInfosReader infosReader = codecFormat.segmentInfosFormat().getSegmentInfosReader(); + infosReader.read(directory, segmentFileName, input, this, IOContext.READ); + final long checksumNow = input.getChecksum(); + final long checksumThen = input.readLong(); + if (checksumNow != checksumThen) + throw new CorruptIndexException("checksum mismatch in segments file"); success = true; } finally { + if (input != null) { + input.close(); + } if (!success) { // Clear any segment infos we had loaded so we // have a clean slate on retry: @@ -267,25 +286,14 @@ } } - /** - * This version of read uses the retry logic (for lock-less - * commits) to find the right segments file to load. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ public final void read(Directory directory) throws CorruptIndexException, IOException { - read(directory, CodecProvider.getDefault()); - } - - public final void read(Directory directory, final CodecProvider codecs) throws CorruptIndexException, IOException { generation = lastGeneration = -1; - this.codecs = codecs; new FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - read(directory, segmentFileName, codecs); + read(directory, segmentFileName); return null; } }.run(); @@ -297,7 +305,7 @@ // before finishCommit is called IndexOutput pendingSegnOutput; - private void write(Directory directory) throws IOException { + private void write(Directory directory, Codec codec) throws IOException { String segmentFileName = getNextSegmentFileName(); final String globalFieldMapFile; @@ -322,8 +330,8 @@ boolean success = false; try { - SegmentInfosWriter infosWriter = codecs.getSegmentInfosWriter(); - segnOutput = infosWriter.writeInfos(directory, segmentFileName, this, IOContext.DEFAULT); + SegmentInfosWriter infosWriter = codec.segmentInfosFormat().getSegmentInfosWriter(); + segnOutput = infosWriter.writeInfos(directory, segmentFileName, codec.getName(), this, IOContext.DEFAULT); infosWriter.prepareCommit(segnOutput); pendingSegnOutput = segnOutput; success = true; @@ -380,7 +388,7 @@ sis.cachedUnmodifiableList = null; sis.cachedUnmodifiableSet = null; for(final SegmentInfo info : this) { - assert info.getSegmentCodecs() != null; + assert info.getCodec() != null; // dont directly access segments, use add method!!! sis.add((SegmentInfo) info.clone()); } @@ -409,7 +417,7 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static long readCurrentVersion(Directory directory, final CodecProvider codecs) + public static long readCurrentVersion(Directory directory) throws CorruptIndexException, IOException { // Fully read the segments file: this ensures that it's @@ -417,8 +425,8 @@ // IndexWriter.prepareCommit has been called (but not // yet commit), then the reader will still see itself as // current: - SegmentInfos sis = new SegmentInfos(codecs); - sis.read(directory, codecs); + SegmentInfos sis = new SegmentInfos(); + sis.read(directory); return sis.version; } @@ -427,10 +435,10 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static Map readCurrentUserData(Directory directory, CodecProvider codecs) + public static Map readCurrentUserData(Directory directory) throws CorruptIndexException, IOException { - SegmentInfos sis = new SegmentInfos(codecs); - sis.read(directory, codecs); + SegmentInfos sis = new SegmentInfos(); + sis.read(directory); return sis.getUserData(); } @@ -808,10 +816,10 @@ * method if changes have been made to this {@link SegmentInfos} instance *

**/ - final void prepareCommit(Directory dir) throws IOException { + final void prepareCommit(Directory dir, Codec codec) throws IOException { if (pendingSegnOutput != null) throw new IllegalStateException("prepareCommit was already called"); - write(dir); + write(dir, codec); } private final long writeGlobalFieldMap(FieldNumberBiMap map, Directory dir, String name) throws IOException { @@ -882,12 +890,12 @@ return files; } - final void finishCommit(Directory dir) throws IOException { + final void finishCommit(Directory dir, Codec codec) throws IOException { if (pendingSegnOutput == null) throw new IllegalStateException("prepareCommit was not called"); boolean success = false; try { - SegmentInfosWriter infosWriter = codecs.getSegmentInfosWriter(); + SegmentInfosWriter infosWriter = codec.segmentInfosFormat().getSegmentInfosWriter(); infosWriter.finishCommit(pendingSegnOutput); pendingSegnOutput = null; success = true; @@ -958,9 +966,9 @@ * method if changes have been made to this {@link SegmentInfos} instance *

**/ - final void commit(Directory dir) throws IOException { - prepareCommit(dir); - finishCommit(dir); + final void commit(Directory dir, Codec codec) throws IOException { + prepareCommit(dir, codec); + finishCommit(dir, codec); } public String toString(Directory directory) { @@ -1106,7 +1114,7 @@ if (cloneChildren) { final List list = new ArrayList(size()); for(final SegmentInfo info : this) { - assert info.getSegmentCodecs() != null; + assert info.getCodec() != null; list.add((SegmentInfo) info.clone()); } return list; @@ -1120,6 +1128,14 @@ this.addAll(infos); } + /** + * Returns the codec used to decode this SegmentInfos from disk + * @lucene.internal + */ + Codec codecFormat() { + return codecFormat; + } + /** Returns an unmodifiable {@link Iterator} of contained segments in order. */ // @Override (comment out until Java 6) public Iterator iterator() { diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentMerger.java 2011-11-01 18:41:12.500614523 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentMerger.java 2011-10-29 16:37:24.996751886 -0400 @@ -73,7 +73,7 @@ private IOContext context; - SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, IOContext context) { + SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, Codec codec, IOContext context) { this.payloadProcessorProvider = payloadProcessorProvider; directory = dir; segment = name; @@ -89,6 +89,7 @@ }; } this.termIndexInterval = termIndexInterval; + this.codec = codec; this.context = context; } @@ -254,12 +255,11 @@ fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.DOC_VALUES), false); } } - final SegmentCodecs codecInfo = fieldInfos.buildSegmentCodecs(false); int docCount = 0; setMatchingSegmentReaders(); - final FieldsWriter fieldsWriter = codecInfo.provider.fieldsWriter(directory, segment, context); + final FieldsWriter fieldsWriter = codec.fieldsFormat().fieldsWriter(directory, segment, context); try { int idx = 0; for (MergeState.IndexReaderAndLiveDocs reader : readers) { @@ -293,7 +293,7 @@ // entering the index. See LUCENE-1282 for // details. throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption"); - segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo, null, context); + segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codec, null, context); return docCount; } @@ -493,9 +493,9 @@ } } - SegmentCodecs getSegmentCodecs() { + Codec getCodec() { assert segmentWriteState != null; - return segmentWriteState.segmentCodecs; + return segmentWriteState.codec; } private final void mergeTerms() throws CorruptIndexException, IOException { @@ -566,8 +566,8 @@ mergeState.dirPayloadProcessor[i] = payloadProcessorProvider.getDirProcessor(reader.reader.directory()); } } - codec = segmentWriteState.segmentCodecs.codec(); - final FieldsConsumer consumer = codec.fieldsConsumer(segmentWriteState); + + final FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(segmentWriteState); boolean success = false; try { consumer.merge(mergeState, @@ -584,7 +584,7 @@ } private void mergePerDoc() throws IOException { - final PerDocConsumer docsConsumer = codec + final PerDocConsumer docsConsumer = codec.docValuesFormat() .docsConsumer(new PerDocWriteState(segmentWriteState)); // TODO: remove this check when 3.x indexes are no longer supported // (3.x indexes don't have docvalues) diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentReadState.java lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentReadState.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentReadState.java 2011-07-08 07:13:00.781820239 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentReadState.java 2011-11-02 21:26:22.548564906 -0400 @@ -35,11 +35,11 @@ // that must do so), then it should negate this value to // get the app's terms divisor: public int termsIndexDivisor; - public final int codecId; + public final String segmentSuffix; public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, IOContext context, int termsIndexDivisor) { - this(dir, info, fieldInfos, context, termsIndexDivisor, -1); + this(dir, info, fieldInfos, context, termsIndexDivisor, ""); } public SegmentReadState(Directory dir, @@ -47,12 +47,22 @@ FieldInfos fieldInfos, IOContext context, int termsIndexDivisor, - int codecId) { + String segmentSuffix) { this.dir = dir; this.segmentInfo = info; this.fieldInfos = fieldInfos; this.context = context; this.termsIndexDivisor = termsIndexDivisor; - this.codecId = codecId; + this.segmentSuffix = segmentSuffix; } -} \ No newline at end of file + + public SegmentReadState(SegmentReadState other, + String newSegmentSuffix) { + this.dir = other.dir; + this.segmentInfo = other.segmentInfo; + this.fieldInfos = other.fieldInfos; + this.context = other.context; + this.termsIndexDivisor = other.termsIndexDivisor; + this.segmentSuffix = newSegmentSuffix; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java 2011-07-08 07:13:00.791820239 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java 2011-11-02 21:26:22.548564906 -0400 @@ -19,6 +19,7 @@ import java.io.PrintStream; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.BitVector; @@ -43,8 +44,8 @@ // Lazily created: public BitVector liveDocs; - final SegmentCodecs segmentCodecs; - public final int codecId; + public final Codec codec; + public final String segmentSuffix; /** Expert: The fraction of terms in the "dictionary" which should be stored * in RAM. Smaller values use more memory, but make searching slightly @@ -56,7 +57,7 @@ public final IOContext context; public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos, - int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, BufferedDeletes segDeletes, IOContext context) { + int numDocs, int termIndexInterval, Codec codec, BufferedDeletes segDeletes, IOContext context) { this.infoStream = infoStream; this.segDeletes = segDeletes; this.directory = directory; @@ -64,24 +65,24 @@ this.fieldInfos = fieldInfos; this.numDocs = numDocs; this.termIndexInterval = termIndexInterval; - this.segmentCodecs = segmentCodecs; - codecId = -1; + this.codec = codec; + segmentSuffix = ""; this.context = context; } /** - * Create a shallow {@link SegmentWriteState} copy final a codec ID + * Create a shallow {@link SegmentWriteState} copy final a format ID */ - SegmentWriteState(SegmentWriteState state, int codecId) { + public SegmentWriteState(SegmentWriteState state, String segmentSuffix) { infoStream = state.infoStream; directory = state.directory; segmentName = state.segmentName; fieldInfos = state.fieldInfos; numDocs = state.numDocs; termIndexInterval = state.termIndexInterval; - segmentCodecs = state.segmentCodecs; context = state.context; - this.codecId = codecId; + codec = state.codec; + this.segmentSuffix = segmentSuffix; segDeletes = state.segDeletes; } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java lucene-2621/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java 2011-10-07 11:55:34.568772788 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java 2011-11-01 21:10:35.940609908 -0400 @@ -19,7 +19,7 @@ import java.io.IOException; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldsWriter; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.ArrayUtil; @@ -35,12 +35,12 @@ int freeCount; final DocumentsWriterPerThread.DocState docState; - final CodecProvider codecProvider; + final Codec codec; public StoredFieldsWriter(DocumentsWriterPerThread docWriter) { this.docWriter = docWriter; this.docState = docWriter.docState; - this.codecProvider = docWriter.codecProvider; + this.codec = docWriter.codec; } private int numStoredFields; @@ -81,7 +81,7 @@ private synchronized void initFieldsWriter(IOContext context) throws IOException { if (fieldsWriter == null) { - fieldsWriter = codecProvider.fieldsWriter(docWriter.directory, docWriter.getSegment(), context); + fieldsWriter = codec.fieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegment(), context); lastDocID = 0; } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/values/Bytes.java lucene-2621/lucene/src/java/org/apache/lucene/index/values/Bytes.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/values/Bytes.java 2011-11-02 12:14:39.788581961 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/values/Bytes.java 2011-11-03 13:43:22.044534709 -0400 @@ -61,6 +61,9 @@ * @lucene.experimental */ public final class Bytes { + + static final String DV_SEGMENT_SUFFIX = "dv"; + // TODO - add bulk copy where possible private Bytes() { /* don't instantiate! */ } @@ -244,7 +247,7 @@ if (datOut == null) { boolean success = false; try { - datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", + datOut = dir.createOutput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, DATA_EXTENSION), context); CodecUtil.writeHeader(datOut, codecName, version); success = true; @@ -269,7 +272,7 @@ boolean success = false; try { if (idxOut == null) { - idxOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", + idxOut = dir.createOutput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, INDEX_EXTENSION), context); CodecUtil.writeHeader(idxOut, codecName, version); } @@ -307,10 +310,10 @@ @Override public void files(Collection files) throws IOException { assert datOut != null; - files.add(IndexFileNames.segmentFileName(id, "", DATA_EXTENSION)); + files.add(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, DATA_EXTENSION)); if (idxOut != null) { // called after flush - so this must be initialized // if needed or present - final String idxFile = IndexFileNames.segmentFileName(id, "", + final String idxFile = IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, INDEX_EXTENSION); files.add(idxFile); } @@ -334,11 +337,11 @@ IndexInput indexIn = null; boolean success = false; try { - dataIn = dir.openInput(IndexFileNames.segmentFileName(id, "", + dataIn = dir.openInput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, Writer.DATA_EXTENSION), context); version = CodecUtil.checkHeader(dataIn, codecName, maxVersion, maxVersion); if (doIndex) { - indexIn = dir.openInput(IndexFileNames.segmentFileName(id, "", + indexIn = dir.openInput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, Writer.INDEX_EXTENSION), context); final int version2 = CodecUtil.checkHeader(indexIn, codecName, maxVersion, maxVersion); @@ -494,8 +497,7 @@ protected void releaseResources() { hash.close(); - bytesUsed - .addAndGet((-docToEntry.length) * RamUsageEstimator.NUM_BYTES_INT); + bytesUsed.addAndGet((-docToEntry.length) * RamUsageEstimator.NUM_BYTES_INT); docToEntry = null; } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java lucene-2621/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java 2011-10-15 21:56:47.669365171 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java 2011-11-01 21:10:35.888609908 -0400 @@ -24,8 +24,7 @@ import org.apache.lucene.index.Fields; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.util.BytesRef; /** @@ -40,13 +39,11 @@ * performance {@link IndexDocValues} should be consumed per-segment just like * IndexReader. *

- * {@link IndexDocValues} are fully integrated into the {@link Codec} API. - * Custom implementations can be exposed on a per field basis via - * {@link CodecProvider}. + * {@link IndexDocValues} are fully integrated into the {@link DocValuesFormat} API. * * @see ValueType for limitations and default implementation documentation * @see IndexDocValuesField for adding values to the index - * @see Codec#docsConsumer(org.apache.lucene.index.PerDocWriteState) for + * @see DocValuesFormat#docsConsumer(org.apache.lucene.index.PerDocWriteState) for * customization * @lucene.experimental */ diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java lucene-2621/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java 2011-10-07 11:55:34.528772788 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java 2011-11-03 13:43:22.044534709 -0400 @@ -156,7 +156,7 @@ protected PackedIntsReader(Directory dir, String id, int numDocs, IOContext context) throws IOException { datIn = dir.openInput( - IndexFileNames.segmentFileName(id, "", Writer.DATA_EXTENSION), + IndexFileNames.segmentFileName(id, Bytes.DV_SEGMENT_SUFFIX, Writer.DATA_EXTENSION), context); this.numDocs = numDocs; boolean success = false; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/values/ValueType.java lucene-2621/lucene/src/java/org/apache/lucene/index/values/ValueType.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/index/values/ValueType.java 2011-10-07 11:55:34.528772788 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/index/values/ValueType.java 2011-11-02 21:50:33.324564159 -0400 @@ -17,7 +17,7 @@ * limitations under the License. */ -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.util.BytesRef; @@ -27,7 +27,7 @@ * ValueType specifies the {@link IndexDocValues} type for a * certain field. A ValueType only defines the data type for a field * while the actual implementation used to encode and decode the values depends - * on the the {@link Codec#docsConsumer} and {@link Codec#docsProducer} methods. + * on the the {@link DocValuesFormat#docsConsumer} and {@link DocValuesFormat#docsProducer} methods. * * @lucene.experimental */ diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java lucene-2621/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java 2011-08-29 12:22:46.871459551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java 2011-11-02 21:26:22.552564906 -0400 @@ -106,7 +106,9 @@ numEntries); for (int i = 0; i < numEntries; i++) { final FileEntry fileEntry = new FileEntry(); - mapping.put(input.readString(), fileEntry); + final String id = input.readString(); + assert !mapping.containsKey(id): "id=" + id + " was written multiple times in the CFS"; + mapping.put(id, fileEntry); fileEntry.offset = input.readLong(); fileEntry.length = input.readLong(); } @@ -170,6 +172,9 @@ entry = new FileEntry(); entry.offset = offset; + + assert !entries.containsKey(id); + entries.put(id, entry); } @@ -271,7 +276,7 @@ public long fileLength(String name) throws IOException { ensureOpen(); if (this.writer != null) { - return writer.fileLenght(name); + return writer.fileLength(name); } FileEntry e = entries.get(IndexFileNames.stripSegmentName(name)); if (e == null) @@ -323,4 +328,9 @@ } }; } + + @Override + public String toString() { + return "CompoundFileDirectory(file=\"" + fileName + "\" in dir=" + directory + ")"; + } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java lucene-2621/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java 2011-08-29 12:22:46.871459551 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java 2011-11-03 13:43:22.056534709 -0400 @@ -22,9 +22,11 @@ import java.io.IOException; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.Map; import java.util.Queue; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.index.IndexFileNames; @@ -85,6 +87,7 @@ private final Directory directory; private final Map entries = new HashMap(); + private final Set seenIDs = new HashSet(); // all entries that are written to a sep. file but not yet moved into CFS private final Queue pendingEntries = new LinkedList(); private boolean closed = false; @@ -238,6 +241,9 @@ final FileEntry entry = new FileEntry(); entry.file = name; entries.put(name, entry); + final String id = IndexFileNames.stripSegmentName(name); + assert !seenIDs.contains(id): "file=\"" + name + "\" maps to id=\"" + id + "\", which was already written"; + seenIDs.add(id); final DirectCFSIndexOutput out; if (outputTaken.compareAndSet(false, true)) { out = new DirectCFSIndexOutput(dataOut, entry, false); @@ -284,7 +290,7 @@ } } - long fileLenght(String name) throws IOException { + long fileLength(String name) throws IOException { FileEntry fileEntry = entries.get(name); if (fileEntry == null) { throw new FileNotFoundException(name + " does not exist"); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/util/CodecUtil.java lucene-2621/lucene/src/java/org/apache/lucene/util/CodecUtil.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/util/CodecUtil.java 2011-05-08 10:52:19.900078710 -0400 +++ lucene-2621/lucene/src/java/org/apache/lucene/util/CodecUtil.java 2011-10-28 12:47:18.288803511 -0400 @@ -35,7 +35,7 @@ private final static int CODEC_MAGIC = 0x3fd76c17; - public static DataOutput writeHeader(DataOutput out, String codec, int version) + public static void writeHeader(DataOutput out, String codec, int version) throws IOException { BytesRef bytes = new BytesRef(codec); if (bytes.length != codec.length() || bytes.length >= 128) { @@ -44,8 +44,6 @@ out.writeInt(CODEC_MAGIC); out.writeString(codec); out.writeInt(version); - - return out; } public static int headerLength(String codec) { diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/java/org/apache/lucene/util/NamedSPILoader.java lucene-2621/lucene/src/java/org/apache/lucene/util/NamedSPILoader.java --- lucene-clean-trunk/lucene/src/java/org/apache/lucene/util/NamedSPILoader.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/java/org/apache/lucene/util/NamedSPILoader.java 2011-11-01 21:10:35.948609905 -0400 @@ -0,0 +1,72 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Collections; +import java.util.Iterator; +import java.util.Map; +import java.util.LinkedHashMap; +import java.util.Set; +import java.util.ServiceLoader; + +/** + * Helper class for loading named SPIs from classpath (e.g. Codec, PostingsFormat). + * @lucene.internal + */ +public final class NamedSPILoader implements Iterable { + + private final Map services; + private final Class clazz; + + public NamedSPILoader(Class clazz) { + this.clazz = clazz; + final ServiceLoader loader = ServiceLoader.load(clazz); + final LinkedHashMap services = new LinkedHashMap(); + for (final S service : loader) { + final String name = service.getName(); + // only add the first one for each name, later services will be ignored + // this allows to place services before others in classpath to make + // them used instead of others + if (!services.containsKey(name)) { + services.put(name, service); + } + } + this.services = Collections.unmodifiableMap(services); + } + + public S lookup(String name) { + final S service = services.get(name); + if (service != null) return service; + throw new IllegalArgumentException("A SPI class of type "+clazz.getName()+" with name '"+name+"' does not exist. "+ + "You need to add the corresponding JAR file supporting this SPI to your classpath."+ + "The current classpath supports the following names: "+availableServices()); + } + + public Set availableServices() { + return services.keySet(); + } + + public Iterator iterator() { + return services.values().iterator(); + } + + public static interface NamedSPI { + String getName(); + } + +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java 2011-07-08 07:13:00.551820239 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java 2011-10-25 11:38:39.424939167 -0400 @@ -27,7 +27,7 @@ public void testSimpleIntBlocks() throws Exception { Directory dir = newDirectory(); - IntStreamFactory f = new MockFixedIntBlockCodec(128).getIntFactory(); + IntStreamFactory f = new MockFixedIntBlockPostingsFormat(128).getIntFactory(); IntIndexOutput out = f.createOutput(dir, "test", newIOContext(random)); for(int i=0;i<11777;i++) { @@ -49,7 +49,7 @@ public void testEmptySimpleIntBlocks() throws Exception { Directory dir = newDirectory(); - IntStreamFactory f = new MockFixedIntBlockCodec(128).getIntFactory(); + IntStreamFactory f = new MockFixedIntBlockPostingsFormat(128).getIntFactory(); IntIndexOutput out = f.createOutput(dir, "test", newIOContext(random)); // write no ints diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestImpersonation.java lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestImpersonation.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestImpersonation.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestImpersonation.java 2011-11-02 13:00:34.108580544 -0400 @@ -0,0 +1,34 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; +import org.apache.lucene.util.LuceneTestCase; + +/** + * Test that the SPI magic is returning "PreFlexRWCodec" for Lucene3x + * + * @lucene.experimental + */ +public class TestImpersonation extends LuceneTestCase { + public void test() throws Exception { + Codec codec = Codec.forName("Lucene3x"); + assertTrue(codec instanceof PreFlexRWCodec); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestSurrogates.java lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestSurrogates.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestSurrogates.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/lucene3x/TestSurrogates.java 2011-11-02 13:00:40.708580540 -0400 @@ -0,0 +1,354 @@ +package org.apache.lucene.index.codecs.lucene3x; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.*; +import org.apache.lucene.document.*; +import org.apache.lucene.analysis.*; +import org.apache.lucene.index.*; +import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; +import org.apache.lucene.util.*; + +import java.util.*; +import java.io.IOException; + +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestSurrogates extends LuceneTestCase { + /** we will manually instantiate preflex-rw here */ + @BeforeClass + public static void beforeClass() { + LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true; + } + + private static String makeDifficultRandomUnicodeString(Random r) { + final int end = r.nextInt(20); + if (end == 0) { + // allow 0 length + return ""; + } + final char[] buffer = new char[end]; + for (int i = 0; i < end; i++) { + int t = r.nextInt(5); + + if (0 == t && i < end - 1) { + // hi + buffer[i++] = (char) (0xd800 + r.nextInt(2)); + // lo + buffer[i] = (char) (0xdc00 + r.nextInt(2)); + } else if (t <= 3) { + buffer[i] = (char) ('a' + r.nextInt(2)); + } else if (4 == t) { + buffer[i] = (char) (0xe000 + r.nextInt(2)); + } + } + + return new String(buffer, 0, end); + } + + private String toHexString(Term t) { + return t.field() + ":" + UnicodeUtil.toHexString(t.text()); + } + + private String getRandomString(Random r) { + String s; + if (r.nextInt(5) == 1) { + if (r.nextInt(3) == 1) { + s = makeDifficultRandomUnicodeString(r); + } else { + s = _TestUtil.randomUnicodeString(r); + } + } else { + s = _TestUtil.randomRealisticUnicodeString(r); + } + return s; + } + + private static class SortTermAsUTF16Comparator implements Comparator { + private static final Comparator legacyComparator = + BytesRef.getUTF8SortedAsUTF16Comparator(); + + public int compare(Term term1, Term term2) { + if (term1.field().equals(term2.field())) { + return legacyComparator.compare(term1.bytes(), term2.bytes()); + } else { + return term1.field().compareTo(term2.field()); + } + } + } + + private static final SortTermAsUTF16Comparator termAsUTF16Comparator = new SortTermAsUTF16Comparator(); + + // single straight enum + private void doTestStraightEnum(List fieldTerms, IndexReader reader, int uniqueTermCount) throws IOException { + + if (VERBOSE) { + System.out.println("\nTEST: top now enum reader=" + reader); + } + FieldsEnum fieldsEnum = MultiFields.getFields(reader).iterator(); + + { + // Test straight enum: + String field; + int termCount = 0; + while((field = fieldsEnum.next()) != null) { + TermsEnum termsEnum = fieldsEnum.terms(); + BytesRef text; + BytesRef lastText = null; + while((text = termsEnum.next()) != null) { + Term exp = fieldTerms.get(termCount); + if (VERBOSE) { + System.out.println(" got term=" + field + ":" + UnicodeUtil.toHexString(text.utf8ToString())); + System.out.println(" exp=" + exp.field() + ":" + UnicodeUtil.toHexString(exp.text().toString())); + System.out.println(); + } + if (lastText == null) { + lastText = new BytesRef(text); + } else { + assertTrue(lastText.compareTo(text) < 0); + lastText.copy(text); + } + assertEquals(exp.field(), field); + assertEquals(exp.bytes(), text); + termCount++; + } + if (VERBOSE) { + System.out.println(" no more terms for field=" + field); + } + } + assertEquals(uniqueTermCount, termCount); + } + } + + // randomly seeks to term that we know exists, then next's + // from there + private void doTestSeekExists(Random r, List fieldTerms, IndexReader reader) throws IOException { + + final Map tes = new HashMap(); + + // Test random seek to existing term, then enum: + if (VERBOSE) { + System.out.println("\nTEST: top now seek"); + } + + int num = atLeast(100); + for (int iter = 0; iter < num; iter++) { + + // pick random field+term + int spot = r.nextInt(fieldTerms.size()); + Term term = fieldTerms.get(spot); + String field = term.field(); + + if (VERBOSE) { + System.out.println("TEST: exist seek field=" + field + " term=" + UnicodeUtil.toHexString(term.text())); + } + + // seek to it + TermsEnum te = tes.get(field); + if (te == null) { + te = MultiFields.getTerms(reader, field).iterator(); + tes.put(field, te); + } + + if (VERBOSE) { + System.out.println(" done get enum"); + } + + // seek should find the term + assertEquals(TermsEnum.SeekStatus.FOUND, + te.seekCeil(term.bytes())); + + // now .next() this many times: + int ct = _TestUtil.nextInt(r, 5, 100); + for(int i=0;i= fieldTerms.size()) { + break; + } + term = fieldTerms.get(1+spot+i); + if (!term.field().equals(field)) { + assertNull(te.next()); + break; + } else { + BytesRef t = te.next(); + + if (VERBOSE) { + System.out.println(" got term=" + (t == null ? null : UnicodeUtil.toHexString(t.utf8ToString()))); + System.out.println(" exp=" + UnicodeUtil.toHexString(term.text().toString())); + } + + assertEquals(term.bytes(), t); + } + } + } + } + + private void doTestSeekDoesNotExist(Random r, int numField, List fieldTerms, Term[] fieldTermsArray, IndexReader reader) throws IOException { + + final Map tes = new HashMap(); + + if (VERBOSE) { + System.out.println("TEST: top random seeks"); + } + + { + int num = atLeast(100); + for (int iter = 0; iter < num; iter++) { + + // seek to random spot + String field = ("f" + r.nextInt(numField)).intern(); + Term tx = new Term(field, getRandomString(r)); + + int spot = Arrays.binarySearch(fieldTermsArray, tx); + + if (spot < 0) { + if (VERBOSE) { + System.out.println("TEST: non-exist seek to " + field + ":" + UnicodeUtil.toHexString(tx.text())); + } + + // term does not exist: + TermsEnum te = tes.get(field); + if (te == null) { + te = MultiFields.getTerms(reader, field).iterator(); + tes.put(field, te); + } + + if (VERBOSE) { + System.out.println(" got enum"); + } + + spot = -spot - 1; + + if (spot == fieldTerms.size() || !fieldTerms.get(spot).field().equals(field)) { + assertEquals(TermsEnum.SeekStatus.END, te.seekCeil(tx.bytes())); + } else { + assertEquals(TermsEnum.SeekStatus.NOT_FOUND, te.seekCeil(tx.bytes())); + + if (VERBOSE) { + System.out.println(" got term=" + UnicodeUtil.toHexString(te.term().utf8ToString())); + System.out.println(" exp term=" + UnicodeUtil.toHexString(fieldTerms.get(spot).text())); + } + + assertEquals(fieldTerms.get(spot).bytes(), + te.term()); + + // now .next() this many times: + int ct = _TestUtil.nextInt(r, 5, 100); + for(int i=0;i= fieldTerms.size()) { + break; + } + Term term = fieldTerms.get(1+spot+i); + if (!term.field().equals(field)) { + assertNull(te.next()); + break; + } else { + BytesRef t = te.next(); + + if (VERBOSE) { + System.out.println(" got term=" + (t == null ? null : UnicodeUtil.toHexString(t.utf8ToString()))); + System.out.println(" exp=" + UnicodeUtil.toHexString(term.text().toString())); + } + + assertEquals(term.bytes(), t); + } + } + + } + } + } + } + } + + + @Test + public void testSurrogatesOrder() throws Exception { + Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random, + dir, + newIndexWriterConfig( TEST_VERSION_CURRENT, + new MockAnalyzer(random)).setCodec(new PreFlexRWCodec())); + + final int numField = _TestUtil.nextInt(random, 2, 5); + + int uniqueTermCount = 0; + + int tc = 0; + + List fieldTerms = new ArrayList(); + + for(int f=0;f uniqueTerms = new HashSet(); + + for(int i=0;i sampleTerms; + + /** we will manually instantiate preflex-rw here */ + @BeforeClass + public static void beforeClass() { + LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE = true; + } + + @Override + public void setUp() throws Exception { + super.setUp(); + indexDivisor = _TestUtil.nextInt(random, 1, 10); + directory = newDirectory(); + termIndexInterval = populate(directory); + + IndexReader r0 = IndexReader.open(directory); + SegmentReader r = (SegmentReader) r0.getSequentialSubReaders()[0]; + String segment = r.getSegmentName(); + r.close(); + + FieldInfos fieldInfos = new FieldInfos(directory, IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELD_INFOS_EXTENSION)); + String segmentFileName = IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION); + long tiiFileLength = directory.fileLength(segmentFileName); + IndexInput input = directory.openInput(segmentFileName, newIOContext(random)); + termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", Lucene3xPostingsFormat.TERMS_EXTENSION), newIOContext(random)), fieldInfos, false); + int totalIndexInterval = termEnum.indexInterval * indexDivisor; + + SegmentTermEnum indexEnum = new SegmentTermEnum(input, fieldInfos, true); + index = new TermInfosReaderIndex(indexEnum, indexDivisor, tiiFileLength, totalIndexInterval); + indexEnum.close(); + input.close(); + + reader = IndexReader.open(directory); + sampleTerms = sample(reader,1000); + + } + + @Override + public void tearDown() throws Exception { + termEnum.close(); + reader.close(); + directory.close(); + super.tearDown(); + } + + public void testSeekEnum() throws CorruptIndexException, IOException { + int indexPosition = 3; + SegmentTermEnum clone = (SegmentTermEnum) termEnum.clone(); + Term term = findTermThatWouldBeAtIndex(clone, indexPosition); + SegmentTermEnum enumerator = clone; + index.seekEnum(enumerator, indexPosition); + assertEquals(term, enumerator.term()); + clone.close(); + } + + public void testCompareTo() throws IOException { + Term term = new Term("field" + random.nextInt(NUMBER_OF_FIELDS) ,getText()); + for (int i = 0; i < index.length(); i++) { + Term t = index.getTerm(i); + int compareTo = term.compareTo(t); + assertEquals(compareTo, index.compareTo(term, i)); + } + } + + public void testRandomSearchPerformance() throws CorruptIndexException, IOException { + IndexSearcher searcher = new IndexSearcher(reader); + for (Term t : sampleTerms) { + TermQuery query = new TermQuery(t); + TopDocs topDocs = searcher.search(query, 10); + assertTrue(topDocs.totalHits > 0); + } + searcher.close(); + } + + private List sample(IndexReader reader, int size) throws IOException { + List sample = new ArrayList(); + Random random = new Random(); + FieldsEnum fieldsEnum = MultiFields.getFields(reader).iterator(); + String field; + while((field = fieldsEnum.next()) != null) { + TermsEnum terms = fieldsEnum.terms(); + while (terms.next() != null) { + if (sample.size() >= size) { + int pos = random.nextInt(size); + sample.set(pos, new Term(field, terms.term())); + } else { + sample.add(new Term(field, terms.term())); + } + } + } + Collections.shuffle(sample); + return sample; + } + + private Term findTermThatWouldBeAtIndex(SegmentTermEnum termEnum, int index) throws IOException { + int termPosition = index * termIndexInterval * indexDivisor; + for (int i = 0; i < termPosition; i++) { + if (!termEnum.next()) { + fail("Should not have run out of terms."); + } + } + return termEnum.term(); + } + + private int populate(Directory directory) throws CorruptIndexException, LockObtainFailedException, IOException { + IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random, MockTokenizer.KEYWORD, false)); + config.setCodec(new PreFlexRWCodec()); + // turn off compound file, this test will open some index files directly. + LogMergePolicy mp = newLogMergePolicy(); + mp.setUseCompoundFile(false); + config.setMergePolicy(mp); + + RandomIndexWriter writer = new RandomIndexWriter(random, directory, config); + for (int i = 0; i < NUMBER_OF_DOCUMENTS; i++) { + Document document = new Document(); + for (int f = 0; f < NUMBER_OF_FIELDS; f++) { + document.add(newField("field" + f, getText(), StringField.TYPE_UNSTORED)); + } + writer.addDocument(document); + } + writer.optimize(); + writer.close(); + return config.getTermIndexInterval(); + } + + private String getText() { + return Long.toString(random.nextLong(),Character.MAX_RADIX); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/perfield/TestPerFieldPostingsFormat.java lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/perfield/TestPerFieldPostingsFormat.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/perfield/TestPerFieldPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/perfield/TestPerFieldPostingsFormat.java 2011-11-01 21:10:35.848609906 -0400 @@ -0,0 +1,272 @@ +package org.apache.lucene.index.codecs.perfield; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; +import java.util.List; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.LogDocMergePolicy; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; +import org.apache.lucene.index.codecs.simpletext.SimpleTextPostingsFormat; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; +import org.junit.Test; + +/** + * + * + */ +//TODO: would be better in this test to pull termsenums and instanceof or something? +// this way we can verify PFPF is doing the right thing. +// for now we do termqueries. +public class TestPerFieldPostingsFormat extends LuceneTestCase { + + private IndexWriter newWriter(Directory dir, IndexWriterConfig conf) + throws IOException { + LogDocMergePolicy logByteSizeMergePolicy = new LogDocMergePolicy(); + logByteSizeMergePolicy.setUseCompoundFile(false); // make sure we use plain + // files + conf.setMergePolicy(logByteSizeMergePolicy); + + final IndexWriter writer = new IndexWriter(dir, conf); + writer.setInfoStream(VERBOSE ? System.out : null); + return writer; + } + + private void addDocs(IndexWriter writer, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); + writer.addDocument(doc); + } + } + + private void addDocs2(IndexWriter writer, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(newField("content", "bbb", TextField.TYPE_UNSTORED)); + writer.addDocument(doc); + } + } + + private void addDocs3(IndexWriter writer, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(newField("content", "ccc", TextField.TYPE_UNSTORED)); + doc.add(newField("id", "" + i, StringField.TYPE_STORED)); + writer.addDocument(doc); + } + } + + /* + * Test that heterogeneous index segments are merge successfully + */ + @Test + public void testMergeUnusedPerFieldCodec() throws IOException { + Directory dir = newDirectory(); + IndexWriterConfig iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodec(new MockCodec()); + IndexWriter writer = newWriter(dir, iwconf); + addDocs(writer, 10); + writer.commit(); + addDocs3(writer, 10); + writer.commit(); + addDocs2(writer, 10); + writer.commit(); + assertEquals(30, writer.maxDoc()); + _TestUtil.checkIndex(dir); + writer.optimize(); + assertEquals(30, writer.maxDoc()); + writer.close(); + dir.close(); + } + + /* + * Test that heterogeneous index segments are merged sucessfully + */ + // TODO: not sure this test is that great, we should probably peek inside PerFieldPostingsFormat or something?! + @Test + public void testChangeCodecAndMerge() throws IOException { + Directory dir = newDirectory(); + if (VERBOSE) { + System.out.println("TEST: make new index"); + } + IndexWriterConfig iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, + new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodec(new MockCodec()); + iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); + //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10); + IndexWriter writer = newWriter(dir, iwconf); + + addDocs(writer, 10); + writer.commit(); + assertQuery(new Term("content", "aaa"), dir, 10); + if (VERBOSE) { + System.out.println("TEST: addDocs3"); + } + addDocs3(writer, 10); + writer.commit(); + writer.close(); + + assertQuery(new Term("content", "ccc"), dir, 10); + assertQuery(new Term("content", "aaa"), dir, 10); + Lucene40Codec codec = (Lucene40Codec)iwconf.getCodec(); + + iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) + .setOpenMode(OpenMode.APPEND).setCodec(codec); + //((LogMergePolicy) iwconf.getMergePolicy()).setUseCompoundFile(false); + //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10); + iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); + + iwconf.setCodec(new MockCodec2()); // uses standard for field content + writer = newWriter(dir, iwconf); + // swap in new codec for currently written segments + if (VERBOSE) { + System.out.println("TEST: add docs w/ Standard codec for content field"); + } + addDocs2(writer, 10); + writer.commit(); + codec = (Lucene40Codec)iwconf.getCodec(); + PostingsFormat origContentCodec = PostingsFormat.forName("MockSep"); + PostingsFormat newContentCodec = PostingsFormat.forName("Lucene40"); + assertEquals(30, writer.maxDoc()); + assertQuery(new Term("content", "bbb"), dir, 10); + assertQuery(new Term("content", "ccc"), dir, 10); //// + assertQuery(new Term("content", "aaa"), dir, 10); + + if (VERBOSE) { + System.out.println("TEST: add more docs w/ new codec"); + } + addDocs2(writer, 10); + writer.commit(); + assertQuery(new Term("content", "ccc"), dir, 10); + assertQuery(new Term("content", "bbb"), dir, 20); + assertQuery(new Term("content", "aaa"), dir, 10); + assertEquals(40, writer.maxDoc()); + + if (VERBOSE) { + System.out.println("TEST: now optimize"); + } + writer.optimize(); + assertEquals(40, writer.maxDoc()); + writer.close(); + assertQuery(new Term("content", "ccc"), dir, 10); + assertQuery(new Term("content", "bbb"), dir, 20); + assertQuery(new Term("content", "aaa"), dir, 10); + + dir.close(); + } + + public void assertQuery(Term t, Directory dir, int num) + throws CorruptIndexException, IOException { + if (VERBOSE) { + System.out.println("\nTEST: assertQuery " + t); + } + IndexReader reader = IndexReader.open(dir, null, true, 1); + IndexSearcher searcher = newSearcher(reader); + TopDocs search = searcher.search(new TermQuery(t), num + 10); + assertEquals(num, search.totalHits); + searcher.close(); + reader.close(); + + } + + public static class MockCodec extends Lucene40Codec { + final PostingsFormat lucene40 = new Lucene40PostingsFormat(); + final PostingsFormat simpleText = new SimpleTextPostingsFormat(); + final PostingsFormat mockSep = new MockSepPostingsFormat(); + + @Override + public PostingsFormat getPostingsFormatForField(String field) { + if (field.equals("id")) { + return simpleText; + } else if (field.equals("content")) { + return mockSep; + } else { + return lucene40; + } + } + } + + public static class MockCodec2 extends Lucene40Codec { + final PostingsFormat lucene40 = new Lucene40PostingsFormat(); + final PostingsFormat simpleText = new SimpleTextPostingsFormat(); + + @Override + public PostingsFormat getPostingsFormatForField(String field) { + if (field.equals("id")) { + return simpleText; + } else { + return lucene40; + } + } + } + + /* + * Test per field codec support - adding fields with random codecs + */ + @Test + public void testStressPerFieldCodec() throws IOException { + Directory dir = newDirectory(random); + final int docsPerRound = 97; + int numRounds = atLeast(1); + for (int i = 0; i < numRounds; i++) { + int num = _TestUtil.nextInt(random, 30, 60); + IndexWriterConfig config = newIndexWriterConfig(random, + TEST_VERSION_CURRENT, new MockAnalyzer(random)); + config.setOpenMode(OpenMode.CREATE_OR_APPEND); + IndexWriter writer = newWriter(dir, config); + for (int j = 0; j < docsPerRound; j++) { + final Document doc = new Document(); + for (int k = 0; k < num; k++) { + FieldType customType = new FieldType(TextField.TYPE_UNSTORED); + customType.setTokenized(random.nextBoolean()); + customType.setOmitNorms(random.nextBoolean()); + Field field = newField("" + k, _TestUtil + .randomRealisticUnicodeString(random, 128), customType); + doc.add(field); + } + writer.addDocument(doc); + } + if (random.nextBoolean()) { + writer.optimize(); + } + writer.commit(); + assertEquals((i + 1) * docsPerRound, writer.maxDoc()); + writer.close(); + } + dir.close(); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java 2011-08-29 12:22:46.641459551 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,348 +0,0 @@ -package org.apache.lucene.index.codecs.preflex; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.store.*; -import org.apache.lucene.document.*; -import org.apache.lucene.analysis.*; -import org.apache.lucene.index.*; -import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; -import org.apache.lucene.util.*; - -import java.util.*; -import java.io.IOException; - -import org.junit.Test; - -public class TestSurrogates extends LuceneTestCase { - - private static String makeDifficultRandomUnicodeString(Random r) { - final int end = r.nextInt(20); - if (end == 0) { - // allow 0 length - return ""; - } - final char[] buffer = new char[end]; - for (int i = 0; i < end; i++) { - int t = r.nextInt(5); - - if (0 == t && i < end - 1) { - // hi - buffer[i++] = (char) (0xd800 + r.nextInt(2)); - // lo - buffer[i] = (char) (0xdc00 + r.nextInt(2)); - } else if (t <= 3) { - buffer[i] = (char) ('a' + r.nextInt(2)); - } else if (4 == t) { - buffer[i] = (char) (0xe000 + r.nextInt(2)); - } - } - - return new String(buffer, 0, end); - } - - private String toHexString(Term t) { - return t.field() + ":" + UnicodeUtil.toHexString(t.text()); - } - - private String getRandomString(Random r) { - String s; - if (r.nextInt(5) == 1) { - if (r.nextInt(3) == 1) { - s = makeDifficultRandomUnicodeString(r); - } else { - s = _TestUtil.randomUnicodeString(r); - } - } else { - s = _TestUtil.randomRealisticUnicodeString(r); - } - return s; - } - - private static class SortTermAsUTF16Comparator implements Comparator { - private static final Comparator legacyComparator = - BytesRef.getUTF8SortedAsUTF16Comparator(); - - public int compare(Term term1, Term term2) { - if (term1.field().equals(term2.field())) { - return legacyComparator.compare(term1.bytes(), term2.bytes()); - } else { - return term1.field().compareTo(term2.field()); - } - } - } - - private static final SortTermAsUTF16Comparator termAsUTF16Comparator = new SortTermAsUTF16Comparator(); - - // single straight enum - private void doTestStraightEnum(List fieldTerms, IndexReader reader, int uniqueTermCount) throws IOException { - - if (VERBOSE) { - System.out.println("\nTEST: top now enum reader=" + reader); - } - FieldsEnum fieldsEnum = MultiFields.getFields(reader).iterator(); - - { - // Test straight enum: - String field; - int termCount = 0; - while((field = fieldsEnum.next()) != null) { - TermsEnum termsEnum = fieldsEnum.terms(); - BytesRef text; - BytesRef lastText = null; - while((text = termsEnum.next()) != null) { - Term exp = fieldTerms.get(termCount); - if (VERBOSE) { - System.out.println(" got term=" + field + ":" + UnicodeUtil.toHexString(text.utf8ToString())); - System.out.println(" exp=" + exp.field() + ":" + UnicodeUtil.toHexString(exp.text().toString())); - System.out.println(); - } - if (lastText == null) { - lastText = new BytesRef(text); - } else { - assertTrue(lastText.compareTo(text) < 0); - lastText.copy(text); - } - assertEquals(exp.field(), field); - assertEquals(exp.bytes(), text); - termCount++; - } - if (VERBOSE) { - System.out.println(" no more terms for field=" + field); - } - } - assertEquals(uniqueTermCount, termCount); - } - } - - // randomly seeks to term that we know exists, then next's - // from there - private void doTestSeekExists(Random r, List fieldTerms, IndexReader reader) throws IOException { - - final Map tes = new HashMap(); - - // Test random seek to existing term, then enum: - if (VERBOSE) { - System.out.println("\nTEST: top now seek"); - } - - int num = atLeast(100); - for (int iter = 0; iter < num; iter++) { - - // pick random field+term - int spot = r.nextInt(fieldTerms.size()); - Term term = fieldTerms.get(spot); - String field = term.field(); - - if (VERBOSE) { - System.out.println("TEST: exist seek field=" + field + " term=" + UnicodeUtil.toHexString(term.text())); - } - - // seek to it - TermsEnum te = tes.get(field); - if (te == null) { - te = MultiFields.getTerms(reader, field).iterator(); - tes.put(field, te); - } - - if (VERBOSE) { - System.out.println(" done get enum"); - } - - // seek should find the term - assertEquals(TermsEnum.SeekStatus.FOUND, - te.seekCeil(term.bytes())); - - // now .next() this many times: - int ct = _TestUtil.nextInt(r, 5, 100); - for(int i=0;i= fieldTerms.size()) { - break; - } - term = fieldTerms.get(1+spot+i); - if (!term.field().equals(field)) { - assertNull(te.next()); - break; - } else { - BytesRef t = te.next(); - - if (VERBOSE) { - System.out.println(" got term=" + (t == null ? null : UnicodeUtil.toHexString(t.utf8ToString()))); - System.out.println(" exp=" + UnicodeUtil.toHexString(term.text().toString())); - } - - assertEquals(term.bytes(), t); - } - } - } - } - - private void doTestSeekDoesNotExist(Random r, int numField, List fieldTerms, Term[] fieldTermsArray, IndexReader reader) throws IOException { - - final Map tes = new HashMap(); - - if (VERBOSE) { - System.out.println("TEST: top random seeks"); - } - - { - int num = atLeast(100); - for (int iter = 0; iter < num; iter++) { - - // seek to random spot - String field = ("f" + r.nextInt(numField)).intern(); - Term tx = new Term(field, getRandomString(r)); - - int spot = Arrays.binarySearch(fieldTermsArray, tx); - - if (spot < 0) { - if (VERBOSE) { - System.out.println("TEST: non-exist seek to " + field + ":" + UnicodeUtil.toHexString(tx.text())); - } - - // term does not exist: - TermsEnum te = tes.get(field); - if (te == null) { - te = MultiFields.getTerms(reader, field).iterator(); - tes.put(field, te); - } - - if (VERBOSE) { - System.out.println(" got enum"); - } - - spot = -spot - 1; - - if (spot == fieldTerms.size() || !fieldTerms.get(spot).field().equals(field)) { - assertEquals(TermsEnum.SeekStatus.END, te.seekCeil(tx.bytes())); - } else { - assertEquals(TermsEnum.SeekStatus.NOT_FOUND, te.seekCeil(tx.bytes())); - - if (VERBOSE) { - System.out.println(" got term=" + UnicodeUtil.toHexString(te.term().utf8ToString())); - System.out.println(" exp term=" + UnicodeUtil.toHexString(fieldTerms.get(spot).text())); - } - - assertEquals(fieldTerms.get(spot).bytes(), - te.term()); - - // now .next() this many times: - int ct = _TestUtil.nextInt(r, 5, 100); - for(int i=0;i= fieldTerms.size()) { - break; - } - Term term = fieldTerms.get(1+spot+i); - if (!term.field().equals(field)) { - assertNull(te.next()); - break; - } else { - BytesRef t = te.next(); - - if (VERBOSE) { - System.out.println(" got term=" + (t == null ? null : UnicodeUtil.toHexString(t.utf8ToString()))); - System.out.println(" exp=" + UnicodeUtil.toHexString(term.text().toString())); - } - - assertEquals(term.bytes(), t); - } - } - - } - } - } - } - } - - - @Test - public void testSurrogatesOrder() throws Exception { - Directory dir = newDirectory(); - RandomIndexWriter w = new RandomIndexWriter(random, - dir, - newIndexWriterConfig( TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec(new PreFlexRWCodec()))); - - final int numField = _TestUtil.nextInt(random, 2, 5); - - int uniqueTermCount = 0; - - int tc = 0; - - List fieldTerms = new ArrayList(); - - for(int f=0;f uniqueTerms = new HashSet(); - - for(int i=0;i sampleTerms; - - @Override - public void setUp() throws Exception { - super.setUp(); - indexDivisor = _TestUtil.nextInt(random, 1, 10); - directory = newDirectory(); - termIndexInterval = populate(directory); - - IndexReader r0 = IndexReader.open(directory); - SegmentReader r = (SegmentReader) r0.getSequentialSubReaders()[0]; - String segment = r.getSegmentName(); - r.close(); - - FieldInfos fieldInfos = new FieldInfos(directory, IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELD_INFOS_EXTENSION)); - String segmentFileName = IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION); - long tiiFileLength = directory.fileLength(segmentFileName); - IndexInput input = directory.openInput(segmentFileName, newIOContext(random)); - termEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_EXTENSION), newIOContext(random)), fieldInfos, false); - int totalIndexInterval = termEnum.indexInterval * indexDivisor; - - SegmentTermEnum indexEnum = new SegmentTermEnum(input, fieldInfos, true); - index = new TermInfosReaderIndex(indexEnum, indexDivisor, tiiFileLength, totalIndexInterval); - indexEnum.close(); - input.close(); - - reader = IndexReader.open(directory); - sampleTerms = sample(reader,1000); - - } - - @Override - public void tearDown() throws Exception { - termEnum.close(); - reader.close(); - directory.close(); - super.tearDown(); - } - - public void testSeekEnum() throws CorruptIndexException, IOException { - int indexPosition = 3; - SegmentTermEnum clone = (SegmentTermEnum) termEnum.clone(); - Term term = findTermThatWouldBeAtIndex(clone, indexPosition); - SegmentTermEnum enumerator = clone; - index.seekEnum(enumerator, indexPosition); - assertEquals(term, enumerator.term()); - clone.close(); - } - - public void testCompareTo() throws IOException { - Term term = new Term("field" + random.nextInt(NUMBER_OF_FIELDS) ,getText()); - for (int i = 0; i < index.length(); i++) { - Term t = index.getTerm(i); - int compareTo = term.compareTo(t); - assertEquals(compareTo, index.compareTo(term, i)); - } - } - - public void testRandomSearchPerformance() throws CorruptIndexException, IOException { - IndexSearcher searcher = new IndexSearcher(reader); - for (Term t : sampleTerms) { - TermQuery query = new TermQuery(t); - TopDocs topDocs = searcher.search(query, 10); - assertTrue(topDocs.totalHits > 0); - } - searcher.close(); - } - - private List sample(IndexReader reader, int size) throws IOException { - List sample = new ArrayList(); - Random random = new Random(); - FieldsEnum fieldsEnum = MultiFields.getFields(reader).iterator(); - String field; - while((field = fieldsEnum.next()) != null) { - TermsEnum terms = fieldsEnum.terms(); - while (terms.next() != null) { - if (sample.size() >= size) { - int pos = random.nextInt(size); - sample.set(pos, new Term(field, terms.term())); - } else { - sample.add(new Term(field, terms.term())); - } - } - } - Collections.shuffle(sample); - return sample; - } - - private Term findTermThatWouldBeAtIndex(SegmentTermEnum termEnum, int index) throws IOException { - int termPosition = index * termIndexInterval * indexDivisor; - for (int i = 0; i < termPosition; i++) { - if (!termEnum.next()) { - fail("Should not have run out of terms."); - } - } - return termEnum.term(); - } - - private int populate(Directory directory) throws CorruptIndexException, LockObtainFailedException, IOException { - IndexWriterConfig config = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random, MockTokenizer.KEYWORD, false)); - CoreCodecProvider cp = new CoreCodecProvider(); - cp.unregister(cp.lookup("PreFlex")); - cp.register(new PreFlexRWCodec()); - cp.setDefaultFieldCodec("PreFlex"); - config.setCodecProvider(cp); - // turn off compound file, this test will open some index files directly. - LogMergePolicy mp = newLogMergePolicy(); - mp.setUseCompoundFile(false); - config.setMergePolicy(mp); - - RandomIndexWriter writer = new RandomIndexWriter(random, directory, config); - for (int i = 0; i < NUMBER_OF_DOCUMENTS; i++) { - Document document = new Document(); - for (int f = 0; f < NUMBER_OF_FIELDS; f++) { - document.add(newField("field" + f, getText(), StringField.TYPE_UNSTORED)); - } - writer.addDocument(document); - } - writer.optimize(); - writer.close(); - return config.getTermIndexInterval(); - } - - private String getText() { - return Long.toString(random.nextLong(),Character.MAX_RADIX); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java 2011-08-29 12:25:48.751459551 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java 2011-11-02 12:10:52.968582077 -0400 @@ -35,7 +35,8 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsBaseFormat; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; @@ -51,13 +52,13 @@ public class Test10KPulsings extends LuceneTestCase { public void test10kPulsed() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); + Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1)); File f = _TestUtil.getTempDir("10kpulsed"); MockDirectoryWrapper dir = newFSDirectory(f); dir.setCheckIndexOnClose(false); // we do this ourselves explicitly RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document document = new Document(); FieldType ft = new FieldType(TextField.TYPE_STORED); @@ -101,13 +102,14 @@ */ public void test10kNotPulsed() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); + int freqCutoff = _TestUtil.nextInt(random, 1, 10); + Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(freqCutoff)); File f = _TestUtil.getTempDir("10knotpulsed"); MockDirectoryWrapper dir = newFSDirectory(f); dir.setCheckIndexOnClose(false); // we do this ourselves explicitly RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document document = new Document(); FieldType ft = new FieldType(TextField.TYPE_STORED); @@ -123,10 +125,7 @@ NumberFormat df = new DecimalFormat("00000", new DecimalFormatSymbols(Locale.ENGLISH)); - Codec codec = cp.lookup(cp.getFieldCodec("field")); - assertTrue(codec instanceof PulsingCodec); - PulsingCodec pulsing = (PulsingCodec) codec; - final int freq = pulsing.getFreqCutoff() + 1; + final int freq = freqCutoff + 1; for (int i = 0; i < 10050; i++) { StringBuilder sb = new StringBuilder(); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java 2011-10-15 21:56:47.625365171 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java 2011-11-02 12:11:01.080582075 -0400 @@ -17,10 +17,8 @@ * limitations under the License. */ -import java.io.IOException; import java.util.IdentityHashMap; import java.util.Map; -import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -30,27 +28,10 @@ import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.BlockTreeTermsReader; -import org.apache.lucene.index.codecs.BlockTreeTermsWriter; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; -import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; +import org.apache.lucene.index.codecs.nestedpulsing.NestedPulsingPostingsFormat; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; @@ -63,10 +44,10 @@ // TODO: this is a basic test. this thing is complicated, add more public void testSophisticatedReuse() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); + Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1)); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document doc = new Document(); doc.add(new Field("foo", "a b b c c c d e f g g h i i j j k", TextField.TYPE_UNSTORED)); iw.addDocument(doc); @@ -101,11 +82,11 @@ /** tests reuse with Pulsing1(Pulsing2(Standard)) */ public void testNestedPulsing() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new NestedPulsing()); + Codec cp = _TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat()); MockDirectoryWrapper dir = newDirectory(); dir.setCheckIndexOnClose(false); // will do this ourselves, custom codec RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document doc = new Document(); doc.add(new Field("foo", "a b b c c c d e f g g g h i i j j k l l m m m", TextField.TYPE_UNSTORED)); // note: the reuse is imperfect, here we would have 4 enums (lost reuse when we get an enum for 'm') @@ -138,79 +119,7 @@ ir.close(); CheckIndex ci = new CheckIndex(dir); - ci.checkIndex(null, cp); + ci.checkIndex(null); dir.close(); } - - static class NestedPulsing extends Codec { - public NestedPulsing() { - super("NestedPulsing"); - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase docsWriter = new StandardPostingsWriter(state); - - PostingsWriterBase pulsingWriterInner = new PulsingPostingsWriter(2, docsWriter); - PostingsWriterBase pulsingWriter = new PulsingPostingsWriter(1, pulsingWriterInner); - - // Terms dict - boolean success = false; - try { - FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, - BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); - success = true; - return ret; - } finally { - if (!success) { - pulsingWriter.close(); - } - } - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); - PostingsReaderBase pulsingReaderInner = new PulsingPostingsReader(docsReader); - PostingsReaderBase pulsingReader = new PulsingPostingsReader(pulsingReaderInner); - boolean success = false; - try { - FieldsProducer ret = new BlockTreeTermsReader( - state.dir, state.fieldInfos, state.segmentInfo.name, - pulsingReader, - state.context, - state.codecId, - state.termsIndexDivisor); - success = true; - return ret; - } finally { - if (!success) { - pulsingReader.close(); - } - } - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); - } - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, id, files); - BlockTreeTermsReader.files(dir, segmentInfo, id, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); - } - - @Override - public void getExtensions(Set extensions) { - StandardCodec.getStandardExtensions(extensions); - DefaultDocValuesConsumer.getExtensions(extensions); - } - } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/Test2BPostings.java lucene-2621/lucene/src/test/org/apache/lucene/index/Test2BPostings.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/Test2BPostings.java 2011-09-28 16:10:03.928773855 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/Test2BPostings.java 2011-10-26 19:00:28.996880998 -0400 @@ -27,23 +27,20 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; /** * Test indexes ~82M docs with 26 terms each, so you get > Integer.MAX_VALUE terms/docs pairs * @lucene.experimental */ +@UseNoMemoryExpensiveCodec public class Test2BPostings extends LuceneTestCase { @Nightly public void test() throws Exception { - - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory")); - assumeFalse("This test is super-slow and very disk-space-consuming with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText")); - MockDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BPostings")); dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER); dir.setCheckIndexOnClose(false); // don't double-checkindex diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java lucene-2621/lucene/src/test/org/apache/lucene/index/Test2BTerms.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/Test2BTerms.java 2011-09-28 16:10:03.928773855 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/Test2BTerms.java 2011-11-01 21:10:35.864609908 -0400 @@ -24,7 +24,8 @@ import org.apache.lucene.analysis.tokenattributes.*; import org.apache.lucene.document.*; import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; + import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -142,7 +143,7 @@ @Ignore("Takes ~4 hours to run on a fast machine!! And requires that you don't use PreFlex codec.") public void test2BTerms() throws IOException { - if ("PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())) { + if ("Lucene3x".equals(Codec.getDefault().getName())) { throw new RuntimeException("thist test cannot run with PreFlex codec"); } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java 2011-09-28 16:10:03.928773855 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java 2011-11-02 21:33:17.400564695 -0400 @@ -17,9 +17,10 @@ * limitations under the License. */ -import java.io.IOException; import java.io.FileNotFoundException; +import java.io.IOException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; @@ -29,11 +30,22 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.mocksep.MockSepCodec; -import org.apache.lucene.index.codecs.pulsing.PulsingCodec; -import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; -import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesFormat; +import org.apache.lucene.index.codecs.DefaultFieldsFormat; +import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; +import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldsFormat; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.SegmentInfosFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsBaseFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.index.codecs.pulsing.Pulsing40PostingsFormat; +import org.apache.lucene.index.codecs.pulsing.PulsingPostingsFormat; +import org.apache.lucene.index.codecs.simpletext.SimpleTextPostingsFormat; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.store.AlreadyClosedException; @@ -975,30 +987,29 @@ } } - public void testSimpleCaseCustomCodecProvider() throws IOException { + public void testSimpleCaseCustomCodec() throws IOException { // main directory Directory dir = newDirectory(); // two auxiliary directories Directory aux = newDirectory(); Directory aux2 = newDirectory(); - CodecProvider provider = new MockCodecProvider(); + Codec codec = new CustomPerFieldCodec(); IndexWriter writer = null; writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodecProvider( - provider)); + new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodec(codec)); // add 100 documents addDocs3(writer, 100); assertEquals(100, writer.maxDoc()); writer.commit(); writer.close(); - _TestUtil.checkIndex(dir, provider); + _TestUtil.checkIndex(dir); writer = newWriter( aux, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.CREATE). - setCodecProvider(provider). + setCodec(codec). setMaxBufferedDocs(10). setMergePolicy(newLogMergePolicy(false)) ); @@ -1012,7 +1023,7 @@ aux2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.CREATE). - setCodecProvider(provider) + setCodec(codec) ); // add 40 documents in compound files addDocs2(writer, 50); @@ -1025,7 +1036,7 @@ dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.APPEND). - setCodecProvider(provider) + setCodec(codec) ); assertEquals(100, writer.maxDoc()); writer.addIndexes(aux, aux2); @@ -1037,19 +1048,24 @@ aux2.close(); } - public static class MockCodecProvider extends CodecProvider { - public MockCodecProvider() { - StandardCodec standardCodec = new StandardCodec(); - SimpleTextCodec simpleTextCodec = new SimpleTextCodec(); - MockSepCodec mockSepCodec = new MockSepCodec(); - register(standardCodec); - register(mockSepCodec); - register(simpleTextCodec); - setFieldCodec("id", simpleTextCodec.name); - setFieldCodec("content", mockSepCodec.name); + private static final class CustomPerFieldCodec extends Lucene40Codec { + private final PostingsFormat simpleTextFormat = PostingsFormat.forName("SimpleText"); + private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene40"); + private final PostingsFormat mockSepFormat = PostingsFormat.forName("MockSep"); + + @Override + public PostingsFormat getPostingsFormatForField(String field) { + if (field.equals("id")) { + return simpleTextFormat; + } else if (field.equals("content")) { + return mockSepFormat; + } else { + return defaultFormat; + } } } + // LUCENE-2790: tests that the non CFS files were deleted by addIndexes public void testNonCFSLeftovers() throws Exception { Directory[] dirs = new Directory[2]; @@ -1066,16 +1082,19 @@ IndexReader[] readers = new IndexReader[] { IndexReader.open(dirs[0]), IndexReader.open(dirs[1]) }; - Directory dir = new RAMDirectory(); + Directory dir = new MockDirectoryWrapper(random, new RAMDirectory()); IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(true); lmp.setNoCFSRatio(1.0); // Force creation of CFS IndexWriter w3 = new IndexWriter(dir, conf); + w3.setInfoStream(VERBOSE ? System.out : null); w3.addIndexes(readers); w3.close(); - // we should now see segments_X, segments.gen,_Y.cfs,_Y.cfe, _Z.fnx - assertEquals("Only one compound segment should exist", 5, dir.listAll().length); + // we should now see segments_X, + // segments.gen,_Y.cfs,_Y.cfe, _Z.fnx + assertEquals("Only one compound segment should exist, but got: " + Arrays.toString(dir.listAll()), 5, dir.listAll().length); + dir.close(); } // LUCENE-3126: tests that if a non-CFS segment is copied, it is converted to @@ -1136,18 +1155,45 @@ src.close(); target.close(); } + + private static class UnRegisteredCodec extends Codec { + public UnRegisteredCodec() { + super("NotRegistered"); + } + + @Override + public PostingsFormat postingsFormat() { + return PostingsFormat.forName("Lucene40"); + } + + @Override + public DocValuesFormat docValuesFormat() { + return new DefaultDocValuesFormat(); + } + + @Override + public FieldsFormat fieldsFormat() { + return new DefaultFieldsFormat(); + } + + @Override + public SegmentInfosFormat segmentInfosFormat() { + return new DefaultSegmentInfosFormat(); + } + } /* * simple test that ensures we getting expected exceptions */ public void testAddIndexMissingCodec() throws IOException { - Directory toAdd = newDirectory(); + MockDirectoryWrapper toAdd = newDirectory(); + // Disable checkIndex, else we get an exception because + // of the unregistered codec: + toAdd.setCheckIndexOnClose(false); { IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); - CodecProvider provider = new CodecProvider(); - provider.register(new StandardCodec()); - conf.setCodecProvider(provider); + conf.setCodec(new UnRegisteredCodec()); IndexWriter w = new IndexWriter(toAdd, conf); Document doc = new Document(); FieldType customType = new FieldType(); @@ -1156,13 +1202,12 @@ w.addDocument(doc); w.close(); } + { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); - CodecProvider provider = new CodecProvider(); - provider.register(new PulsingCodec(1 + random.nextInt(20))); - conf.setCodecProvider(provider); + conf.setCodec(_TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1 + random.nextInt(20)))); IndexWriter w = new IndexWriter(dir, conf); try { w.addIndexes(toAdd); @@ -1177,27 +1222,11 @@ dir.close(); } - { - Directory dir = newDirectory(); - IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)); - CodecProvider provider = new CodecProvider(); - provider.register(new PulsingCodec(1 + random.nextInt(20))); - conf.setCodecProvider(provider); - IndexWriter w = new IndexWriter(dir, conf); + try { IndexReader indexReader = IndexReader.open(toAdd); - try { - w.addIndexes(indexReader); - fail("no such codec"); - } catch (IllegalArgumentException ex) { - // expected - } - indexReader.close(); - w.close(); - IndexReader open = IndexReader.open(dir); - assertEquals(0, open.numDocs()); - open.close(); - dir.close(); + fail("no such codec"); + } catch (IllegalArgumentException ex) { + // expected } toAdd.close(); } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java 2011-09-28 16:10:03.928773855 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java 2011-11-01 21:10:35.860609907 -0400 @@ -568,7 +568,7 @@ "_0_1.s" + contentFieldIndex, "segments_2", "segments.gen", - "1.fnx"}; + "_1.fnx"}; String[] actual = dir.listAll(); Arrays.sort(expected); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java 2011-09-28 16:10:03.928773855 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java 2011-11-01 21:10:35.856609905 -0400 @@ -23,7 +23,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; @@ -37,7 +37,7 @@ public class TestBinaryTerms extends LuceneTestCase { public void testBinary() throws IOException { assumeFalse("PreFlex codec cannot work with binary terms!", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + Codec.getDefault().getName().equals("Lucene3x")); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestCodecs.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestCodecs.java 2011-09-15 13:19:27.426254549 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestCodecs.java 2011-11-02 12:11:13.076582069 -0400 @@ -26,14 +26,15 @@ import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.PostingsConsumer; import org.apache.lucene.index.codecs.TermStats; import org.apache.lucene.index.codecs.TermsConsumer; -import org.apache.lucene.index.codecs.mocksep.MockSepCodec; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xCodec; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; @@ -64,6 +65,7 @@ // goes to 1 before next one known to exist // - skipTo(term) // - skipTo(doc) + public class TestCodecs extends LuceneTestCase { private static String[] fieldNames = new String[] {"one", "two", "three", "four"}; @@ -255,9 +257,10 @@ final Directory dir = newDirectory(); FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone(); this.write(fieldInfos, dir, fields, true); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, clonedFieldInfos.buildSegmentCodecs(false), clonedFieldInfos); + Codec codec = Codec.getDefault(); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, codec, clonedFieldInfos); - final FieldsProducer reader = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); + final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); final FieldsEnum fieldsEnum = reader.iterator(); assertNotNull(fieldsEnum.next()); @@ -307,12 +310,13 @@ FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone(); this.write(fieldInfos, dir, fields, false); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, clonedFieldInfos.buildSegmentCodecs(false), clonedFieldInfos); + Codec codec = Codec.getDefault(); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, codec, clonedFieldInfos); if (VERBOSE) { System.out.println("TEST: now read postings"); } - final FieldsProducer terms = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); + final FieldsProducer terms = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); final Verify[] threads = new Verify[NUM_TEST_THREADS-1]; for(int i=0;i filesToDelete = merger.createCompoundFile(merged + ".cfs", info, newIOContext(random)); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java 2011-09-30 20:57:07.238773589 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java 2011-11-02 12:11:18.132582065 -0400 @@ -34,21 +34,17 @@ import org.apache.lucene.index.codecs.BlockTermsReader; import org.apache.lucene.index.codecs.BlockTermsWriter; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CoreCodecProvider; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.PostingsWriterBase; import org.apache.lucene.index.codecs.TermsIndexReaderBase; import org.apache.lucene.index.codecs.TermsIndexWriterBase; -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; -import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; import org.apache.lucene.search.FieldCache; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; @@ -106,130 +102,6 @@ dir.close(); } - private static class StandardCodecWithOrds extends Codec { - - public StandardCodecWithOrds() { - super("StandardOrds"); - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase docs = new StandardPostingsWriter(state); - - // TODO: should we make the terms index more easily - // pluggable? Ie so that this codec would record which - // index impl was used, and switch on loading? - // Or... you must make a new Codec for this? - TermsIndexWriterBase indexWriter; - boolean success = false; - try { - indexWriter = new FixedGapTermsIndexWriter(state); - success = true; - } finally { - if (!success) { - docs.close(); - } - } - - success = false; - try { - FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs); - success = true; - return ret; - } finally { - if (!success) { - try { - docs.close(); - } finally { - indexWriter.close(); - } - } - } - } - - public final static int TERMS_CACHE_SIZE = 1024; - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); - TermsIndexReaderBase indexReader; - - boolean success = false; - try { - indexReader = new FixedGapTermsIndexReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId, state.context); - success = true; - } finally { - if (!success) { - postings.close(); - } - } - - success = false; - try { - FieldsProducer ret = new BlockTermsReader(indexReader, - state.dir, - state.fieldInfos, - state.segmentInfo.name, - postings, - state.context, - TERMS_CACHE_SIZE, - state.codecId); - success = true; - return ret; - } finally { - if (!success) { - try { - postings.close(); - } finally { - indexReader.close(); - } - } - } - } - - /** Extension of freq postings file */ - static final String FREQ_EXTENSION = "frq"; - - /** Extension of prox postings file */ - static final String PROX_EXTENSION = "prx"; - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, id, files); - BlockTermsReader.files(dir, segmentInfo, id, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, id, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); - } - - @Override - public void getExtensions(Set extensions) { - getStandardExtensions(extensions); - DefaultDocValuesConsumer.getExtensions(extensions); - } - - public static void getStandardExtensions(Set extensions) { - extensions.add(FREQ_EXTENSION); - extensions.add(PROX_EXTENSION); - BlockTermsReader.getExtensions(extensions); - FixedGapTermsIndexReader.getIndexExtensions(extensions); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); - } - } - public void testRandom() throws Exception { MockDirectoryWrapper dir = newDirectory(); @@ -252,13 +124,8 @@ // Sometimes swap in codec that impls ord(): if (random.nextInt(10) == 7) { // Make sure terms index has ords: - CoreCodecProvider cp = new CoreCodecProvider(); - cp.register(new StandardCodecWithOrds()); - cp.setDefaultFieldCodec("StandardOrds"); - - // So checkIndex on close works - dir.setCodecProvider(cp); - conf.setCodecProvider(cp); + Codec codec = _TestUtil.alwaysPostingsFormat(PostingsFormat.forName("Lucene40WithOrds")); + conf.setCodec(codec); } final RandomIndexWriter w = new RandomIndexWriter(random, dir, conf); @@ -354,14 +221,8 @@ // Sometimes swap in codec that impls ord(): if (random.nextInt(10) == 7) { - // Make sure terms index has ords: - CoreCodecProvider cp = new CoreCodecProvider(); - cp.register(new StandardCodecWithOrds()); - cp.setDefaultFieldCodec("StandardOrds"); - - // So checkIndex on close works - dir.setCodecProvider(cp); - conf.setCodecProvider(cp); + Codec codec = _TestUtil.alwaysPostingsFormat(PostingsFormat.forName("Lucene40WithOrds")); + conf.setCodec(codec); } final RandomIndexWriter w = new RandomIndexWriter(random, dir, conf); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestFlex.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestFlex.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestFlex.java 2011-08-29 12:22:46.651459551 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestFlex.java 2011-11-02 11:31:08.652583306 -0400 @@ -20,6 +20,7 @@ import org.apache.lucene.store.*; import org.apache.lucene.analysis.*; import org.apache.lucene.document.*; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.util.*; public class TestFlex extends LuceneTestCase { @@ -64,7 +65,7 @@ public void testTermOrd() throws Exception { Directory d = newDirectory(); IndexWriter w = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); Document doc = new Document(); doc.add(newField("f", "a b c", TextField.TYPE_UNSTORED)); w.addDocument(doc); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java 2011-10-15 21:56:47.629365171 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java 2011-11-02 12:07:27.768582185 -0400 @@ -22,7 +22,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TopDocs; @@ -36,6 +35,9 @@ // Make sure we don't clone IndexInputs too frequently // during merging: public void test() throws Exception { + // NOTE: if we see a fail on this test with "NestedPulsing" its because its + // reuse isnt perfect (but reasonable). see TestPulsingReuse.testNestedPulsing + // for more details final MockDirectoryWrapper dir = newDirectory(); final TieredMergePolicy tmp = new TieredMergePolicy(); tmp.setMaxMergeAtOnce(2); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java 2011-09-28 16:10:03.928773855 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java 2011-11-01 21:10:35.856609905 -0400 @@ -58,25 +58,25 @@ } writer.commit(); Collection files = writer.getIndexFileNames(); - files.remove("1.fnx"); + files.remove("_1.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); writer.commit(); files = writer.getIndexFileNames(); - files.remove("2.fnx"); + files.remove("_2.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } { @@ -89,12 +89,12 @@ writer.addDocument(d); writer.close(); Collection files = writer.getIndexFileNames(); - files.remove("2.fnx"); + files.remove("_2.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( @@ -102,7 +102,7 @@ writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); dir.close(); } @@ -121,29 +121,29 @@ d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } IndexReader reader = IndexReader.open(dir, false); reader.deleteDocument(0); reader.commit(); reader.close(); // make sure this reader can not modify the field map - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); dir.close(); } @@ -162,7 +162,7 @@ d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); @@ -170,9 +170,9 @@ writer.commit(); writer.commit(); writer.commit(); - assertFNXFiles(dir, "1.fnx", "2.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx"); writer.close(); - assertFNXFiles(dir, "1.fnx", "2.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx"); } { @@ -184,14 +184,14 @@ d.add(new BinaryField("f3", new byte[] { 1, 2, 3, 4, 5 })); writer.addDocument(d); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); dir.close(); } } @@ -208,14 +208,14 @@ d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); writer.close(); - assertFNXFiles(dir, "1.fnx", "2.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx"); // open first commit List listCommits = IndexReader.listCommits(dir); assertEquals(2, listCommits.size()); @@ -229,18 +229,18 @@ writer.addDocument(d); writer.commit(); // now we have 3 files since f3 is not present in the first commit - assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx", "_3.fnx"); writer.close(); - assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx", "_3.fnx"); writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.commit(); listCommits = IndexReader.listCommits(dir); assertEquals(1, listCommits.size()); - assertFNXFiles(dir, "3.fnx"); + assertFNXFiles(dir, "_3.fnx"); writer.close(); - assertFNXFiles(dir, "3.fnx"); + assertFNXFiles(dir, "_3.fnx"); dir.close(); } } @@ -494,7 +494,7 @@ assertEquals(1, segmentInfos.getGlobalFieldMapVersion()); assertEquals(1, segmentInfos.getLastGlobalFieldMapVersion()); files = writer.getIndexFileNames(); - assertTrue(files.remove("1.fnx")); + assertTrue(files.remove("_1.fnx")); for (String string : files) { assertFalse(string.endsWith(".fnx")); } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexReader.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexReader.java 2011-10-25 12:50:53.500936931 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexReader.java 2011-11-02 11:31:08.660583305 -0400 @@ -39,7 +39,7 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader.FieldOption; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldCache; @@ -88,6 +88,7 @@ writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode( OpenMode.APPEND).setMaxBufferedDocs(2)); + writer.setInfoStream(VERBOSE ? System.out : null); for(int i=0;i<7;i++) addDocumentWithFields(writer); writer.close(); @@ -947,7 +948,7 @@ writer.close(); SegmentInfos sis = new SegmentInfos(); - sis.read(d, CodecProvider.getDefault()); + sis.read(d); IndexReader r = IndexReader.open(d, false); IndexCommit c = r.getIndexCommit(); @@ -1231,7 +1232,7 @@ // LUCENE-1609: don't load terms index public void testNoTermsIndex() throws Throwable { Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); Document doc = new Document(); doc.add(newField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", TextField.TYPE_UNSTORED)); doc.add(newField("number", "0 1 2 3 4 5 6 7 8 9", TextField.TYPE_UNSTORED)); @@ -1251,7 +1252,7 @@ writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setCodecProvider(_TestUtil.alwaysCodec("Standard")). + setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())). setMergePolicy(newLogMergePolicy(10)) ); writer.addDocument(doc); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java 2011-10-07 11:55:34.458772788 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java 2011-10-29 12:34:15.556759400 -0400 @@ -28,7 +28,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -169,8 +168,10 @@ // them, the merged result can easily be larger than the // sum because the merged FST may use array encoding for // some arcs (which uses more space): - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory")); - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory")); + + final String idFormat = _TestUtil.getPostingsFormat("id"); + final String contentFormat = _TestUtil.getPostingsFormat("content"); + assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory")); MockDirectoryWrapper dir = newDirectory(); Analyzer analyzer; if (random.nextBoolean()) { diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java 2011-09-15 13:19:27.426254549 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java 2011-11-02 21:33:26.808564690 -0400 @@ -26,6 +26,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.util.LuceneTestCase; @@ -71,6 +72,7 @@ assertEquals(ThreadAffinityDocumentsWriterThreadPool.class, conf.getIndexerThreadPool().getClass()); assertEquals(FlushByRamOrCountsPolicy.class, conf.getFlushPolicy().getClass()); assertEquals(IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB, conf.getRAMPerThreadHardLimitMB()); + assertEquals(Codec.getDefault(), conf.getCodec()); // Sanity check - validate that all getters are covered. Set getters = new HashSet(); getters.add("getAnalyzer"); @@ -88,7 +90,6 @@ getters.add("getMaxBufferedDocs"); getters.add("getIndexingChain"); getters.add("getMergedSegmentWarmer"); - getters.add("getCodecProvider"); getters.add("getMergePolicy"); getters.add("getMaxThreadStates"); getters.add("getReaderPooling"); @@ -96,6 +97,7 @@ getters.add("getReaderTermsIndexDivisor"); getters.add("getFlushPolicy"); getters.add("getRAMPerThreadHardLimitMB"); + getters.add("getCodec"); for (Method m : IndexWriterConfig.class.getDeclaredMethods()) { if (m.getDeclaringClass() == IndexWriterConfig.class && m.getName().startsWith("get")) { diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java 2011-10-25 12:50:53.500936931 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java 2011-11-01 21:10:35.864609908 -0400 @@ -31,7 +31,6 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -897,8 +896,9 @@ } public void testIndexingThenDeleting() throws Exception { - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory")); - assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText")); + final String fieldFormat = _TestUtil.getPostingsFormat("field"); + assumeFalse("This test cannot run with Memory codec", fieldFormat.equals("Memory")); + assumeFalse("This test cannot run with SimpleText codec", fieldFormat.equals("SimpleText")); final Random r = random; Directory dir = newDirectory(); // note this test explicitly disables payloads diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java 2011-09-28 16:10:03.928773855 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java 2011-11-01 21:10:35.856609905 -0400 @@ -23,6 +23,7 @@ import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Random; @@ -927,10 +928,10 @@ } catch (RuntimeException re) { // Expected } - assertTrue(dir.fileExists("1.fnx")); + assertTrue(dir.fileExists("_1.fnx")); assertTrue(failure.failOnCommit && failure.failOnDeleteFile); w.rollback(); - assertFalse(dir.fileExists("1.fnx")); + assertFalse(dir.fileExists("_1.fnx")); assertEquals(0, dir.listAll().length); dir.close(); } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java 2011-11-02 10:34:14.668585064 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java 2011-11-03 13:24:53.412535280 -0400 @@ -52,12 +52,17 @@ import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.store.NativeFSLockFactory; import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; @@ -1937,4 +1942,22 @@ assert(version3 > version2); d.close(); } + + public void testWhetherDeleteAllDeletesWriteLock() throws Exception { + Directory d = newFSDirectory(_TestUtil.getTempDir("TestIndexWriter.testWhetherDeleteAllDeletesWriteLock")); + // Must use SimpleFSLockFactory... NativeFSLockFactory + // somehow "knows" a lock is held against write.lock + // even if you remove that file: + d.setLockFactory(new SimpleFSLockFactory()); + RandomIndexWriter w1 = new RandomIndexWriter(random, d); + w1.deleteAll(); + try { + new RandomIndexWriter(random, d); + fail("should not be able to create another writer"); + } catch (LockObtainFailedException lofe) { + // expected + } + w1.close(); + d.close(); + } } diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java 2011-08-29 12:22:46.651459551 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java 2011-11-01 21:10:35.856609905 -0400 @@ -24,7 +24,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -154,8 +154,10 @@ // them, the merged result can easily be larger than the // sum because the merged FST may use array encoding for // some arcs (which uses more space): - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory")); - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory")); + + final String idFormat = _TestUtil.getPostingsFormat("id"); + final String contentFormat = _TestUtil.getPostingsFormat("content"); + assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory")); int START_COUNT = 57; int NUM_DIR = TEST_NIGHTLY ? 50 : 5; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java 2011-10-23 17:07:48.669018012 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java 2011-11-01 21:10:35.856609905 -0400 @@ -30,6 +30,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -988,15 +989,16 @@ public void testNoTermsIndex() throws Exception { // Some Codecs don't honor the ReaderTermsIndexDivisor, so skip the test if // they're picked. - HashSet illegalCodecs = new HashSet(); - illegalCodecs.add("PreFlex"); - illegalCodecs.add("SimpleText"); - illegalCodecs.add("Memory"); + assumeFalse("PreFlex codec does not support ReaderTermsIndexDivisor!", + "Lucene3x".equals(Codec.getDefault().getName())); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setReaderTermsIndexDivisor(-1); + // Don't proceed if picked Codec is in the list of illegal ones. - if (illegalCodecs.contains(conf.getCodecProvider().getFieldCodec("f"))) return; + final String format = _TestUtil.getPostingsFormat("f"); + assumeFalse("Format: " + format + " does not support ReaderTermsIndexDivisor!", + (format.equals("SimpleText") || format.equals("Memory"))); Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, conf); @@ -1006,7 +1008,7 @@ IndexReader r = IndexReader.open(w, true).getSequentialSubReaders()[0]; try { r.termDocsEnum(null, "f", new BytesRef("val")); - fail("should have failed to seek since terms index was not loaded. Codec used " + conf.getCodecProvider().getFieldCodec("f")); + fail("should have failed to seek since terms index was not loaded."); } catch (IllegalStateException e) { // expected - we didn't load the term index } finally { diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java 2011-09-27 14:52:46.368773982 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java 2011-10-29 12:39:36.500759237 -0400 @@ -23,7 +23,6 @@ import org.apache.lucene.analysis.*; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.ScoreDoc; @@ -34,6 +33,7 @@ import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util._TestUtil; /** * Tests lazy skipping on the proximity file. @@ -131,8 +131,10 @@ } public void testLazySkipping() throws IOException { - assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("SimpleText")); - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("Memory")); + final String fieldFormat = _TestUtil.getPostingsFormat(this.field); + assumeFalse("This test cannot run with Memory codec", fieldFormat.equals("Memory")); + assumeFalse("This test cannot run with SimpleText codec", fieldFormat.equals("SimpleText")); + // test whether only the minimum amount of seeks() // are performed performTest(5); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestLongPostings.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestLongPostings.java 2011-10-07 11:55:34.468772788 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestLongPostings.java 2011-11-01 21:10:35.860609907 -0400 @@ -29,7 +29,6 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java 2011-09-27 14:52:46.368773982 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java 2011-11-02 11:31:08.660583305 -0400 @@ -25,6 +25,7 @@ import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -68,7 +69,7 @@ public void testSimpleSkip() throws IOException { Directory dir = new CountingRAMDirectory(new RAMDirectory()); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new PayloadAnalyzer()).setCodecProvider(_TestUtil.alwaysCodec("Standard")).setMergePolicy(newLogMergePolicy())); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new PayloadAnalyzer()).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())).setMergePolicy(newLogMergePolicy())); Term term = new Term("test", "a"); for (int i = 0; i < 5000; i++) { Document d1 = new Document(); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java 2011-08-29 12:22:46.671459551 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,318 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; -import java.util.List; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.CheckIndex.Status.SegmentInfoStatus; -import org.apache.lucene.index.CheckIndex.Status; -import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec; -import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec; -import org.apache.lucene.index.codecs.mocksep.MockSepCodec; -import org.apache.lucene.index.codecs.pulsing.PulsingCodec; -import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util._TestUtil; -import org.junit.Test; - -/** - * - * - */ -public class TestPerFieldCodecSupport extends LuceneTestCase { - - private IndexWriter newWriter(Directory dir, IndexWriterConfig conf) - throws IOException { - LogDocMergePolicy logByteSizeMergePolicy = new LogDocMergePolicy(); - logByteSizeMergePolicy.setUseCompoundFile(false); // make sure we use plain - // files - conf.setMergePolicy(logByteSizeMergePolicy); - - final IndexWriter writer = new IndexWriter(dir, conf); - writer.setInfoStream(VERBOSE ? System.out : null); - return writer; - } - - private void addDocs(IndexWriter writer, int numDocs) throws IOException { - for (int i = 0; i < numDocs; i++) { - Document doc = new Document(); - doc.add(newField("content", "aaa", TextField.TYPE_UNSTORED)); - writer.addDocument(doc); - } - } - - private void addDocs2(IndexWriter writer, int numDocs) throws IOException { - for (int i = 0; i < numDocs; i++) { - Document doc = new Document(); - doc.add(newField("content", "bbb", TextField.TYPE_UNSTORED)); - writer.addDocument(doc); - } - } - - private void addDocs3(IndexWriter writer, int numDocs) throws IOException { - for (int i = 0; i < numDocs; i++) { - Document doc = new Document(); - doc.add(newField("content", "ccc", TextField.TYPE_UNSTORED)); - doc.add(newField("id", "" + i, StringField.TYPE_STORED)); - writer.addDocument(doc); - } - } - - /* - * Test is hetrogenous index segements are merge sucessfully - */ - @Test - public void testMergeUnusedPerFieldCodec() throws IOException { - Directory dir = newDirectory(); - CodecProvider provider = new MockCodecProvider(); - IndexWriterConfig iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodecProvider( - provider); - IndexWriter writer = newWriter(dir, iwconf); - addDocs(writer, 10); - writer.commit(); - addDocs3(writer, 10); - writer.commit(); - addDocs2(writer, 10); - writer.commit(); - assertEquals(30, writer.maxDoc()); - _TestUtil.checkIndex(dir, provider); - writer.optimize(); - assertEquals(30, writer.maxDoc()); - writer.close(); - dir.close(); - } - - /* - * Test that heterogeneous index segments are merged sucessfully - */ - @Test - public void testChangeCodecAndMerge() throws IOException { - Directory dir = newDirectory(); - CodecProvider provider = new MockCodecProvider(); - if (VERBOSE) { - System.out.println("TEST: make new index"); - } - IndexWriterConfig iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodecProvider(provider); - iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); - //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10); - IndexWriter writer = newWriter(dir, iwconf); - - addDocs(writer, 10); - writer.commit(); - assertQuery(new Term("content", "aaa"), dir, 10, provider); - if (VERBOSE) { - System.out.println("TEST: addDocs3"); - } - addDocs3(writer, 10); - writer.commit(); - writer.close(); - - assertQuery(new Term("content", "ccc"), dir, 10, provider); - assertQuery(new Term("content", "aaa"), dir, 10, provider); - assertCodecPerField(_TestUtil.checkIndex(dir, provider), "content", - provider.lookup("MockSep")); - - iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)) - .setOpenMode(OpenMode.APPEND).setCodecProvider(provider); - //((LogMergePolicy) iwconf.getMergePolicy()).setUseCompoundFile(false); - //((LogMergePolicy) iwconf.getMergePolicy()).setMergeFactor(10); - iwconf.setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH); - - provider = new MockCodecProvider2(); // uses standard for field content - iwconf.setCodecProvider(provider); - writer = newWriter(dir, iwconf); - // swap in new codec for currently written segments - if (VERBOSE) { - System.out.println("TEST: add docs w/ Standard codec for content field"); - } - addDocs2(writer, 10); - writer.commit(); - Codec origContentCodec = provider.lookup("MockSep"); - Codec newContentCodec = provider.lookup("Standard"); - assertHybridCodecPerField(_TestUtil.checkIndex(dir, provider), "content", - origContentCodec, origContentCodec, newContentCodec); - assertEquals(30, writer.maxDoc()); - assertQuery(new Term("content", "bbb"), dir, 10, provider); - assertQuery(new Term("content", "ccc"), dir, 10, provider); //// - assertQuery(new Term("content", "aaa"), dir, 10, provider); - - if (VERBOSE) { - System.out.println("TEST: add more docs w/ new codec"); - } - addDocs2(writer, 10); - writer.commit(); - assertQuery(new Term("content", "ccc"), dir, 10, provider); - assertQuery(new Term("content", "bbb"), dir, 20, provider); - assertQuery(new Term("content", "aaa"), dir, 10, provider); - assertEquals(40, writer.maxDoc()); - - if (VERBOSE) { - System.out.println("TEST: now optimize"); - } - writer.optimize(); - assertEquals(40, writer.maxDoc()); - writer.close(); - assertCodecPerFieldOptimized(_TestUtil.checkIndex(dir, provider), - "content", newContentCodec); - assertQuery(new Term("content", "ccc"), dir, 10, provider); - assertQuery(new Term("content", "bbb"), dir, 20, provider); - assertQuery(new Term("content", "aaa"), dir, 10, provider); - - dir.close(); - } - - public void assertCodecPerFieldOptimized(Status checkIndex, String field, - Codec codec) { - assertEquals(1, checkIndex.segmentInfos.size()); - final CodecProvider provider = checkIndex.segmentInfos.get(0).codec.provider; - assertEquals(codec, provider.lookup(provider.getFieldCodec(field))); - - } - - public void assertCodecPerField(Status checkIndex, String field, Codec codec) { - for (SegmentInfoStatus info : checkIndex.segmentInfos) { - final CodecProvider provider = info.codec.provider; - assertEquals(codec, provider.lookup(provider.getFieldCodec(field))); - } - } - - public void assertHybridCodecPerField(Status checkIndex, String field, - Codec... codec) throws IOException { - List segmentInfos = checkIndex.segmentInfos; - assertEquals(segmentInfos.size(), codec.length); - for (int i = 0; i < codec.length; i++) { - SegmentCodecs codecInfo = segmentInfos.get(i).codec; - FieldInfos fieldInfos = new FieldInfos(checkIndex.dir, IndexFileNames - .segmentFileName(segmentInfos.get(i).name, "", - IndexFileNames.FIELD_INFOS_EXTENSION)); - FieldInfo fieldInfo = fieldInfos.fieldInfo(field); - assertEquals("faild for segment index: " + i, codec[i], - codecInfo.codecs[fieldInfo.getCodecId()]); - } - } - - public void assertQuery(Term t, Directory dir, int num, CodecProvider codecs) - throws CorruptIndexException, IOException { - if (VERBOSE) { - System.out.println("\nTEST: assertQuery " + t); - } - IndexReader reader = IndexReader.open(dir, null, true, - IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, codecs); - IndexSearcher searcher = newSearcher(reader); - TopDocs search = searcher.search(new TermQuery(t), num + 10); - assertEquals(num, search.totalHits); - searcher.close(); - reader.close(); - - } - - public static class MockCodecProvider extends CodecProvider { - - public MockCodecProvider() { - StandardCodec standardCodec = new StandardCodec(); - setDefaultFieldCodec(standardCodec.name); - SimpleTextCodec simpleTextCodec = new SimpleTextCodec(); - MockSepCodec mockSepCodec = new MockSepCodec(); - register(standardCodec); - register(mockSepCodec); - register(simpleTextCodec); - setFieldCodec("id", simpleTextCodec.name); - setFieldCodec("content", mockSepCodec.name); - } - } - - public static class MockCodecProvider2 extends CodecProvider { - - public MockCodecProvider2() { - StandardCodec standardCodec = new StandardCodec(); - setDefaultFieldCodec(standardCodec.name); - SimpleTextCodec simpleTextCodec = new SimpleTextCodec(); - MockSepCodec mockSepCodec = new MockSepCodec(); - register(standardCodec); - register(mockSepCodec); - register(simpleTextCodec); - setFieldCodec("id", simpleTextCodec.name); - setFieldCodec("content", standardCodec.name); - } - } - - /* - * Test per field codec support - adding fields with random codecs - */ - @Test - public void testStressPerFieldCodec() throws IOException { - Directory dir = newDirectory(random); - final int docsPerRound = 97; - int numRounds = atLeast(1); - for (int i = 0; i < numRounds; i++) { - CodecProvider provider = new CodecProvider(); - Codec[] codecs = new Codec[] { new StandardCodec(), - new SimpleTextCodec(), new MockSepCodec(), - new PulsingCodec(1 + random.nextInt(20)), - new MockVariableIntBlockCodec(1 + random.nextInt(10)), - new MockFixedIntBlockCodec(1 + random.nextInt(10)) }; - for (Codec codec : codecs) { - provider.register(codec); - } - int num = _TestUtil.nextInt(random, 30, 60); - for (int j = 0; j < num; j++) { - provider.setFieldCodec("" + j, codecs[random.nextInt(codecs.length)].name); - } - IndexWriterConfig config = newIndexWriterConfig(random, - TEST_VERSION_CURRENT, new MockAnalyzer(random)); - config.setOpenMode(OpenMode.CREATE_OR_APPEND); - config.setCodecProvider(provider); - IndexWriter writer = newWriter(dir, config); - for (int j = 0; j < docsPerRound; j++) { - final Document doc = new Document(); - for (int k = 0; k < num; k++) { - FieldType customType = new FieldType(TextField.TYPE_UNSTORED); - customType.setTokenized(random.nextBoolean()); - customType.setOmitNorms(random.nextBoolean()); - Field field = newField("" + k, _TestUtil - .randomRealisticUnicodeString(random, 128), customType); - doc.add(field); - } - writer.addDocument(doc); - } - if (random.nextBoolean()) { - writer.optimize(); - } - writer.commit(); - assertEquals((i + 1) * docsPerRound, writer.maxDoc()); - writer.close(); - } - dir.close(); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java 2011-09-27 14:52:46.368773982 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestReaderClosed.java 2011-11-01 21:10:35.864609908 -0400 @@ -22,7 +22,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.store.AlreadyClosedException; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java 2011-10-25 12:50:53.500936931 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java 2011-11-02 11:31:08.664583307 -0400 @@ -19,7 +19,8 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.*; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.memory.MemoryPostingsFormat; import org.apache.lucene.store.*; import org.apache.lucene.util.*; import org.junit.Test; @@ -35,13 +36,12 @@ dir.setCheckIndexOnClose(false); // we use a custom codec provider final LineFileDocs docs = new LineFileDocs(random); - CodecProvider provider = CodecProvider.getDefault(); //provider.register(new MemoryCodec()); - if ( (!"PreFlex".equals(provider.getDefaultFieldCodec())) && random.nextBoolean()) { - provider.setFieldCodec("docid", "Memory"); + if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random.nextBoolean()) { + Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat())); } - final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(provider)); + final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); w.setInfoStream(VERBOSE ? System.out : null); final int SIZE = atLeast(TEST_NIGHTLY ? 100 : 20); int id = 0; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java 2011-10-25 12:50:53.500936931 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java 2011-11-02 12:17:24.812581876 -0400 @@ -23,6 +23,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.util.BytesRef; import java.io.IOException; @@ -74,7 +75,7 @@ } public void testMerge() throws IOException { - SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, null, new FieldInfos(), newIOContext(random)); + SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, null, new FieldInfos(), Codec.getDefault(), newIOContext(random)); merger.add(reader1); merger.add(reader2); int docsMerged = merger.merge(); @@ -82,7 +83,7 @@ final FieldInfos fieldInfos = merger.fieldInfos(); //Should be able to open a new SegmentReader against the new directory SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, - merger.getSegmentCodecs(), fieldInfos), + merger.getCodec(), fieldInfos), true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); assertTrue(mergedReader != null); assertTrue(mergedReader.numDocs() == 2); @@ -145,7 +146,7 @@ w.close(); // Assert that SM fails if .del exists - SegmentMerger sm = new SegmentMerger(dir, 1, "a", null, null, null, newIOContext(random)); + SegmentMerger sm = new SegmentMerger(dir, 1, "a", null, null, null, Codec.getDefault(), newIOContext(random)); boolean doFail = false; try { sm.createCompoundFile("b1", w.segmentInfos.info(0), newIOContext(random)); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java 2011-08-29 12:22:46.651459551 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java 2011-11-02 11:31:08.668583308 -0400 @@ -26,6 +26,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.store.Directory; @@ -74,7 +75,7 @@ public void testPrevTermAtEnd() throws IOException { - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); addDoc(writer, "aaa bbb"); writer.close(); SegmentReader reader = getOnlySegmentReader(IndexReader.open(dir, false)); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java 2011-08-29 12:22:46.671459551 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java 2011-11-01 21:10:35.864609908 -0400 @@ -29,7 +29,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.CheckHits; import org.apache.lucene.search.IndexSearcher; @@ -57,7 +57,7 @@ super.setUp(); // we generate aweful regexps: good for testing. // but for preflex codec, the test can be very slow, so use less iterations. - numIterations = CodecProvider.getDefault().getFieldCodec("field").equals("PreFlex") ? 10 * RANDOM_MULTIPLIER : atLeast(50); + numIterations = Codec.getDefault().getName().equals("Lucene3x") ? 10 * RANDOM_MULTIPLIER : atLeast(50); dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java lucene-2621/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java 2011-10-15 21:56:47.629365171 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java 2011-11-02 21:34:13.360564666 -0400 @@ -358,11 +358,7 @@ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); /* - CoreCodecProvider cp = new CoreCodecProvider(); - cp.unregister(cp.lookup("Standard")); - cp.register(new StandardCodec(minTermsInBlock, maxTermsInBlock)); - cp.setDefaultFieldCodec("Standard"); - iwc.setCodecProvider(cp); + iwc.setCodec(new StandardCodec(minTermsInBlock, maxTermsInBlock)); */ final RandomIndexWriter w = new RandomIndexWriter(random, d, iwc); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java lucene-2621/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java 2011-10-07 11:55:34.438772788 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java 2011-11-01 21:10:35.808609905 -0400 @@ -39,7 +39,7 @@ import org.apache.lucene.index.MultiPerDocValues; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.search.*; @@ -65,7 +65,7 @@ @Before public void setUp() throws Exception { super.setUp(); - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); + assumeFalse("cannot work with preflex codec", Codec.getDefault().getName().equals("Lucene3x")); } /* @@ -140,7 +140,7 @@ indexValues(w_1, valuesPerIndex, first, values, false, 7); w_1.commit(); assertEquals(valuesPerIndex, w_1.maxDoc()); - _TestUtil.checkIndex(d_1, w_1.getConfig().getCodecProvider()); + _TestUtil.checkIndex(d_1); // index second index Directory d_2 = newDirectory(); @@ -148,7 +148,7 @@ indexValues(w_2, valuesPerIndex, second, values, false, 7); w_2.commit(); assertEquals(valuesPerIndex, w_2.maxDoc()); - _TestUtil.checkIndex(d_2, w_2.getConfig().getCodecProvider()); + _TestUtil.checkIndex(d_2); Directory target = newDirectory(); IndexWriter w = new IndexWriter(target, writerConfig(random.nextBoolean())); @@ -162,7 +162,7 @@ w.optimize(true); w.commit(); - _TestUtil.checkIndex(target, w.getConfig().getCodecProvider()); + _TestUtil.checkIndex(target); assertEquals(valuesPerIndex * 2, w.maxDoc()); // check values @@ -546,7 +546,6 @@ } public void testMultiValuedIndexDocValuesField() throws Exception { - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); Directory d = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random, d); Document doc = new Document(); @@ -575,7 +574,6 @@ } public void testDifferentTypedDocValuesField() throws Exception { - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); Directory d = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random, d); Document doc = new Document(); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java lucene-2621/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java 2011-10-11 19:18:19.578772267 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java 2011-11-01 21:10:35.808609905 -0400 @@ -15,7 +15,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -42,8 +42,7 @@ @Before public void setUp() throws Exception { super.setUp(); - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault() - .getDefaultFieldCodec().equals("PreFlex")); + assumeFalse("cannot work with preflex codec", Codec.getDefault().getName().equals("Lucene3x")); } private static EnumSet INTEGERS = EnumSet.of(ValueType.VAR_INTS, diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java lucene-2621/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java 2011-11-03 08:30:44.116544373 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java 2011-11-01 21:10:35.792609908 -0400 @@ -29,7 +29,7 @@ import org.apache.lucene.index.OrdTermState; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.Explanation; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -566,7 +566,7 @@ /** Test whether all similarities return document 3 before documents 7 and 8. */ public void testHeartRanking() throws IOException { assumeFalse("PreFlex codec does not support the stats necessary for this test!", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + "Lucene3x".equals(Codec.getDefault().getName())); Query q = new TermQuery(new Term(FIELD_BODY, "heart")); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java lucene-2621/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java 2011-11-03 08:30:40.080544375 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java 2011-11-01 21:10:35.792609908 -0400 @@ -28,7 +28,7 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.search.similarities.Similarity; @@ -49,7 +49,7 @@ public void testSimple() throws Exception { assumeFalse("PreFlex codec cannot work with IndexDocValues!", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + "Lucene3x".equals(Codec.getDefault().getName())); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java lucene-2621/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java 2011-09-16 09:41:18.126281604 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/search/TestPrefixRandom.java 2011-11-01 21:10:35.792609908 -0400 @@ -30,7 +30,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.Directory; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; @@ -60,8 +60,8 @@ // we generate aweful prefixes: good for testing. // but for preflex codec, the test can be very slow, so use less iterations. - final String codec = CodecProvider.getDefault().getFieldCodec("field"); - int num = codec.equals("PreFlex") ? 200 * RANDOM_MULTIPLIER : atLeast(1000); + final String codec = Codec.getDefault().getName(); + int num = codec.equals("Lucene3x") ? 200 * RANDOM_MULTIPLIER : atLeast(1000); for (int i = 0; i < num; i++) { field.setValue(_TestUtil.randomUnicodeString(random, 10)); writer.addDocument(doc); diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java lucene-2621/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java 2011-10-24 12:30:31.652982074 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/search/TestRegexpRandom2.java 2011-11-01 21:10:35.792609908 -0400 @@ -33,7 +33,7 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.Directory; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; @@ -143,7 +143,7 @@ public void testRegexps() throws Exception { // we generate aweful regexps: good for testing. // but for preflex codec, the test can be very slow, so use less iterations. - int num = CodecProvider.getDefault().getFieldCodec(fieldName).equals("PreFlex") ? 100 * RANDOM_MULTIPLIER : atLeast(1000); + int num = Codec.getDefault().getName().equals("Lucene3x") ? 100 * RANDOM_MULTIPLIER : atLeast(1000); for (int i = 0; i < num; i++) { String reg = AutomatonTestUtil.randomRegexp(random); if (VERBOSE) { diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/TestSort.java lucene-2621/lucene/src/test/org/apache/lucene/search/TestSort.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/search/TestSort.java 2011-10-25 12:50:53.484936934 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/search/TestSort.java 2011-11-01 21:10:35.792609908 -0400 @@ -40,7 +40,7 @@ import org.apache.lucene.index.MultiReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.FieldValueHitQueue.Entry; @@ -70,7 +70,7 @@ public class TestSort extends LuceneTestCase { // true if our codec supports docvalues: true unless codec is preflex (3.x) - boolean supportsDocValues = CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex") == false; + boolean supportsDocValues = Codec.getDefault().getName().equals("Lucene3x") == false; private static int NUM_STRINGS; private IndexSearcher full; private IndexSearcher searchX; diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test/org/apache/lucene/TestExternalCodecs.java lucene-2621/lucene/src/test/org/apache/lucene/TestExternalCodecs.java --- lucene-clean-trunk/lucene/src/test/org/apache/lucene/TestExternalCodecs.java 2011-09-15 13:19:27.436254549 -0400 +++ lucene-2621/lucene/src/test/org/apache/lucene/TestExternalCodecs.java 2011-11-02 12:11:42.016582054 -0400 @@ -17,512 +17,47 @@ * limitations under the License. */ -import org.apache.lucene.util.*; -import org.apache.lucene.util.Bits; -import org.apache.lucene.index.*; -import org.apache.lucene.document.*; -import org.apache.lucene.search.*; +import java.io.*; +import java.util.*; + import org.apache.lucene.analysis.*; +import org.apache.lucene.document.*; +import org.apache.lucene.index.*; import org.apache.lucene.index.codecs.*; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.search.*; import org.apache.lucene.store.*; -import java.util.*; -import java.io.*; +import org.apache.lucene.util.*; +import org.apache.lucene.util.Bits; /* Intentionally outside of oal.index to verify fully external codecs work fine */ public class TestExternalCodecs extends LuceneTestCase { - // For fun, test that we can override how terms are - // sorted, and basic things still work -- this comparator - // sorts in reversed unicode code point order: - private static final Comparator reverseUnicodeComparator = new Comparator() { - public int compare(BytesRef t1, BytesRef t2) { - byte[] b1 = t1.bytes; - byte[] b2 = t2.bytes; - int b1Stop; - int b1Upto = t1.offset; - int b2Upto = t2.offset; - if (t1.length < t2.length) { - b1Stop = t1.offset + t1.length; - } else { - b1Stop = t1.offset + t2.length; - } - while(b1Upto < b1Stop) { - final int bb1 = b1[b1Upto++] & 0xff; - final int bb2 = b2[b2Upto++] & 0xff; - if (bb1 != bb2) { - //System.out.println("cmp 1=" + t1 + " 2=" + t2 + " return " + (bb2-bb1)); - return bb2 - bb1; - } - } - - // One is prefix of another, or they are equal - return t2.length-t1.length; - } - - @Override - public boolean equals(Object other) { - return this == other; - } - }; - - // TODO - // - good improvement would be to write through to disk, - // and then load into ram from disk - public static class RAMOnlyCodec extends Codec { + private static final class CustomPerFieldCodec extends Lucene40Codec { - public RAMOnlyCodec() { - super("RamOnly"); - } - // Postings state: - static class RAMPostings extends FieldsProducer { - final Map fieldToTerms = new TreeMap(); - - @Override - public Terms terms(String field) { - return fieldToTerms.get(field); - } - - @Override - public FieldsEnum iterator() { - return new RAMFieldsEnum(this); - } - - @Override - public void close() { - } - } - - static class RAMField extends Terms { - final String field; - final SortedMap termToDocs = new TreeMap(); - long sumTotalTermFreq; - long sumDocFreq; - int docCount; - - RAMField(String field) { - this.field = field; - } - - @Override - public long getUniqueTermCount() { - return termToDocs.size(); - } - - @Override - public long getSumTotalTermFreq() { - return sumTotalTermFreq; - } - - @Override - public long getSumDocFreq() throws IOException { - return sumDocFreq; - } - - @Override - public int getDocCount() throws IOException { - return docCount; - } - - @Override - public TermsEnum iterator() { - return new RAMTermsEnum(RAMOnlyCodec.RAMField.this); - } - - @Override - public Comparator getComparator() { - return reverseUnicodeComparator; - } - } - - static class RAMTerm { - final String term; - long totalTermFreq; - final List docs = new ArrayList(); - public RAMTerm(String term) { - this.term = term; - } - } - - static class RAMDoc { - final int docID; - final int[] positions; - byte[][] payloads; - - public RAMDoc(int docID, int freq) { - this.docID = docID; - positions = new int[freq]; - } - } - - // Classes for writing to the postings state - private static class RAMFieldsConsumer extends FieldsConsumer { - - private final RAMPostings postings; - private final RAMTermsConsumer termsConsumer = new RAMTermsConsumer(); - - public RAMFieldsConsumer(RAMPostings postings) { - this.postings = postings; - } - - @Override - public TermsConsumer addField(FieldInfo field) { - RAMField ramField = new RAMField(field.name); - postings.fieldToTerms.put(field.name, ramField); - termsConsumer.reset(ramField); - return termsConsumer; - } - - @Override - public void close() { - // TODO: finalize stuff - } - } - - private static class RAMTermsConsumer extends TermsConsumer { - private RAMField field; - private final RAMPostingsWriterImpl postingsWriter = new RAMPostingsWriterImpl(); - RAMTerm current; - - void reset(RAMField field) { - this.field = field; - } - - @Override - public PostingsConsumer startTerm(BytesRef text) { - final String term = text.utf8ToString(); - current = new RAMTerm(term); - postingsWriter.reset(current); - return postingsWriter; - } - - - @Override - public Comparator getComparator() { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } - - @Override - public void finishTerm(BytesRef text, TermStats stats) { - assert stats.docFreq > 0; - assert stats.docFreq == current.docs.size(); - current.totalTermFreq = stats.totalTermFreq; - field.termToDocs.put(current.term, current); - } - - @Override - public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) { - field.sumTotalTermFreq = sumTotalTermFreq; - field.sumDocFreq = sumDocFreq; - field.docCount = docCount; - } - } - - public static class RAMPostingsWriterImpl extends PostingsConsumer { - private RAMTerm term; - private RAMDoc current; - private int posUpto = 0; - - public void reset(RAMTerm term) { - this.term = term; - } - - @Override - public void startDoc(int docID, int freq) { - current = new RAMDoc(docID, freq); - term.docs.add(current); - posUpto = 0; - } - - @Override - public void addPosition(int position, BytesRef payload) { - current.positions[posUpto] = position; - if (payload != null && payload.length > 0) { - if (current.payloads == null) { - current.payloads = new byte[current.positions.length][]; - } - byte[] bytes = current.payloads[posUpto] = new byte[payload.length]; - System.arraycopy(payload.bytes, payload.offset, bytes, 0, payload.length); - } - posUpto++; - } - - @Override - public void finishDoc() { - assert posUpto == current.positions.length; - } - } - - // Classes for reading from the postings state - static class RAMFieldsEnum extends FieldsEnum { - private final RAMPostings postings; - private final Iterator it; - private String current; - - public RAMFieldsEnum(RAMPostings postings) { - this.postings = postings; - this.it = postings.fieldToTerms.keySet().iterator(); - } - - @Override - public String next() { - if (it.hasNext()) { - current = it.next(); - } else { - current = null; - } - return current; - } - - @Override - public TermsEnum terms() { - return new RAMTermsEnum(postings.fieldToTerms.get(current)); - } - } - - static class RAMTermsEnum extends TermsEnum { - Iterator it; - String current; - private final RAMField ramField; - - public RAMTermsEnum(RAMField field) { - this.ramField = field; - } - - @Override - public Comparator getComparator() { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } - - @Override - public BytesRef next() { - if (it == null) { - if (current == null) { - it = ramField.termToDocs.keySet().iterator(); - } else { - it = ramField.termToDocs.tailMap(current).keySet().iterator(); - } - } - if (it.hasNext()) { - current = it.next(); - return new BytesRef(current); - } else { - return null; - } - } - - @Override - public SeekStatus seekCeil(BytesRef term, boolean useCache) { - current = term.utf8ToString(); - it = null; - if (ramField.termToDocs.containsKey(current)) { - return SeekStatus.FOUND; - } else { - if (current.compareTo(ramField.termToDocs.lastKey()) > 0) { - return SeekStatus.END; - } else { - return SeekStatus.NOT_FOUND; - } - } - } - - @Override - public void seekExact(long ord) { - throw new UnsupportedOperationException(); - } - - @Override - public long ord() { - throw new UnsupportedOperationException(); - } - - @Override - public BytesRef term() { - // TODO: reuse BytesRef - return new BytesRef(current); - } - - @Override - public int docFreq() { - return ramField.termToDocs.get(current).docs.size(); - } - - @Override - public long totalTermFreq() { - return ramField.termToDocs.get(current).totalTermFreq; - } - - @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) { - return new RAMDocsEnum(ramField.termToDocs.get(current), liveDocs); - } - - @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) { - return new RAMDocsAndPositionsEnum(ramField.termToDocs.get(current), liveDocs); - } - } - - private static class RAMDocsEnum extends DocsEnum { - private final RAMTerm ramTerm; - private final Bits liveDocs; - private RAMDoc current; - int upto = -1; - int posUpto = 0; - - public RAMDocsEnum(RAMTerm ramTerm, Bits liveDocs) { - this.ramTerm = ramTerm; - this.liveDocs = liveDocs; - } - - @Override - public int advance(int targetDocID) { - do { - nextDoc(); - } while (upto < ramTerm.docs.size() && current.docID < targetDocID); - return NO_MORE_DOCS; - } - - // TODO: override bulk read, for better perf - @Override - public int nextDoc() { - while(true) { - upto++; - if (upto < ramTerm.docs.size()) { - current = ramTerm.docs.get(upto); - if (liveDocs == null || liveDocs.get(current.docID)) { - posUpto = 0; - return current.docID; - } - } else { - return NO_MORE_DOCS; - } - } - } - - @Override - public int freq() { - return current.positions.length; - } - - @Override - public int docID() { - return current.docID; - } - } - - private static class RAMDocsAndPositionsEnum extends DocsAndPositionsEnum { - private final RAMTerm ramTerm; - private final Bits liveDocs; - private RAMDoc current; - int upto = -1; - int posUpto = 0; - - public RAMDocsAndPositionsEnum(RAMTerm ramTerm, Bits liveDocs) { - this.ramTerm = ramTerm; - this.liveDocs = liveDocs; - } - - @Override - public int advance(int targetDocID) { - do { - nextDoc(); - } while (upto < ramTerm.docs.size() && current.docID < targetDocID); - return NO_MORE_DOCS; - } - - // TODO: override bulk read, for better perf - @Override - public int nextDoc() { - while(true) { - upto++; - if (upto < ramTerm.docs.size()) { - current = ramTerm.docs.get(upto); - if (liveDocs == null || liveDocs.get(current.docID)) { - posUpto = 0; - return current.docID; - } - } else { - return NO_MORE_DOCS; - } - } - } - - @Override - public int freq() { - return current.positions.length; - } - - @Override - public int docID() { - return current.docID; - } - - @Override - public int nextPosition() { - return current.positions[posUpto++]; - } - - @Override - public boolean hasPayload() { - return current.payloads != null && current.payloads[posUpto-1] != null; - } - - @Override - public BytesRef getPayload() { - return new BytesRef(current.payloads[posUpto-1]); - } - } - - // Holds all indexes created - private final Map state = new HashMap(); - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState writeState) { - RAMPostings postings = new RAMPostings(); - RAMFieldsConsumer consumer = new RAMFieldsConsumer(postings); - synchronized(state) { - state.put(writeState.segmentName, postings); - } - return consumer; - } + private final PostingsFormat ramFormat = PostingsFormat.forName("RAMOnly"); + private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene40"); + private final PostingsFormat pulsingFormat = PostingsFormat.forName("Pulsing40"); @Override - public FieldsProducer fieldsProducer(SegmentReadState readState) - throws IOException { - - synchronized(state) { - return state.get(readState.segmentInfo.name); + public PostingsFormat getPostingsFormatForField(String field) { + if (field.equals("field2") || field.equals("id")) { + return pulsingFormat; + } else if (field.equals("field1")) { + return defaultFormat; + } else { + return ramFormat; } } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return null; - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return null; - } - - @Override - public void getExtensions(Set extensions) { - } - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) { - } } // tests storing "id" and "field2" fields as pulsing codec, // whose term sort is backwards unicode code point, and // storing "field1" as a custom entirely-in-RAM codec public void testPerFieldCodec() throws Exception { - CodecProvider provider = new CoreCodecProvider(); - provider.register(new RAMOnlyCodec()); - provider.setDefaultFieldCodec("RamOnly"); final int NUM_DOCS = atLeast(173); MockDirectoryWrapper dir = newDirectory(); @@ -530,7 +65,7 @@ IndexWriter w = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setCodecProvider(provider). + setCodec(new CustomPerFieldCodec()). setMergePolicy(newLogMergePolicy(3)) ); w.setInfoStream(VERBOSE ? System.out : null); @@ -539,11 +74,9 @@ doc.add(newField("field1", "this field uses the standard codec as the test", TextField.TYPE_UNSTORED)); // uses pulsing codec: Field field2 = newField("field2", "this field uses the pulsing codec as the test", TextField.TYPE_UNSTORED); - provider.setFieldCodec(field2.name(), "Pulsing"); doc.add(field2); Field idField = newField("id", "", StringField.TYPE_UNSTORED); - provider.setFieldCodec(idField.name(), "Pulsing"); doc.add(idField); for(int i=0;i codecs = Codec.availableCodecs(); + assertTrue(codecs.contains("Lucene40")); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/BaseTokenStreamTestCase.java 2011-11-01 21:10:36.000609905 -0400 @@ -0,0 +1,334 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.StringReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; + +import org.apache.lucene.analysis.tokenattributes.*; +import org.apache.lucene.util.Attribute; +import org.apache.lucene.util.AttributeImpl; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +/** + * Base class for all Lucene unit tests that use TokenStreams. + *

+ * When writing unit tests for analysis components, its highly recommended + * to use the helper methods here (especially in conjunction with {@link MockAnalyzer} or + * {@link MockTokenizer}), as they contain many assertions and checks to + * catch bugs. + * + * @see MockAnalyzer + * @see MockTokenizer + */ +public abstract class BaseTokenStreamTestCase extends LuceneTestCase { + // some helpers to test Analyzers and TokenStreams: + + public static interface CheckClearAttributesAttribute extends Attribute { + boolean getAndResetClearCalled(); + } + + public static final class CheckClearAttributesAttributeImpl extends AttributeImpl implements CheckClearAttributesAttribute { + private boolean clearCalled = false; + + public boolean getAndResetClearCalled() { + try { + return clearCalled; + } finally { + clearCalled = false; + } + } + + @Override + public void clear() { + clearCalled = true; + } + + @Override + public boolean equals(Object other) { + return ( + other instanceof CheckClearAttributesAttributeImpl && + ((CheckClearAttributesAttributeImpl) other).clearCalled == this.clearCalled + ); + } + + @Override + public int hashCode() { + return 76137213 ^ Boolean.valueOf(clearCalled).hashCode(); + } + + @Override + public void copyTo(AttributeImpl target) { + ((CheckClearAttributesAttributeImpl) target).clear(); + } + } + + public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], Integer finalOffset) throws IOException { + assertNotNull(output); + CheckClearAttributesAttribute checkClearAtt = ts.addAttribute(CheckClearAttributesAttribute.class); + + assertTrue("has no CharTermAttribute", ts.hasAttribute(CharTermAttribute.class)); + CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class); + + OffsetAttribute offsetAtt = null; + if (startOffsets != null || endOffsets != null || finalOffset != null) { + assertTrue("has no OffsetAttribute", ts.hasAttribute(OffsetAttribute.class)); + offsetAtt = ts.getAttribute(OffsetAttribute.class); + } + + TypeAttribute typeAtt = null; + if (types != null) { + assertTrue("has no TypeAttribute", ts.hasAttribute(TypeAttribute.class)); + typeAtt = ts.getAttribute(TypeAttribute.class); + } + + PositionIncrementAttribute posIncrAtt = null; + if (posIncrements != null) { + assertTrue("has no PositionIncrementAttribute", ts.hasAttribute(PositionIncrementAttribute.class)); + posIncrAtt = ts.getAttribute(PositionIncrementAttribute.class); + } + + ts.reset(); + for (int i = 0; i < output.length; i++) { + // extra safety to enforce, that the state is not preserved and also assign bogus values + ts.clearAttributes(); + termAtt.setEmpty().append("bogusTerm"); + if (offsetAtt != null) offsetAtt.setOffset(14584724,24683243); + if (typeAtt != null) typeAtt.setType("bogusType"); + if (posIncrAtt != null) posIncrAtt.setPositionIncrement(45987657); + + checkClearAtt.getAndResetClearCalled(); // reset it, because we called clearAttribute() before + assertTrue("token "+i+" does not exist", ts.incrementToken()); + assertTrue("clearAttributes() was not called correctly in TokenStream chain", checkClearAtt.getAndResetClearCalled()); + + assertEquals("term "+i, output[i], termAtt.toString()); + if (startOffsets != null) + assertEquals("startOffset "+i, startOffsets[i], offsetAtt.startOffset()); + if (endOffsets != null) + assertEquals("endOffset "+i, endOffsets[i], offsetAtt.endOffset()); + if (types != null) + assertEquals("type "+i, types[i], typeAtt.type()); + if (posIncrements != null) + assertEquals("posIncrement "+i, posIncrements[i], posIncrAtt.getPositionIncrement()); + + // we can enforce some basic things about a few attributes even if the caller doesn't check: + if (offsetAtt != null) { + assertTrue("startOffset must be >= 0", offsetAtt.startOffset() >= 0); + assertTrue("endOffset must be >= 0", offsetAtt.endOffset() >= 0); + assertTrue("endOffset must be >= startOffset", offsetAtt.endOffset() >= offsetAtt.startOffset()); + } + if (posIncrAtt != null) { + assertTrue("posIncrement must be >= 0", posIncrAtt.getPositionIncrement() >= 0); + } + } + assertFalse("end of stream", ts.incrementToken()); + ts.end(); + if (finalOffset != null) + assertEquals("finalOffset ", finalOffset.intValue(), offsetAtt.endOffset()); + if (offsetAtt != null) { + assertTrue("finalOffset must be >= 0", offsetAtt.endOffset() >= 0); + } + ts.close(); + } + + public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException { + assertTokenStreamContents(ts, output, startOffsets, endOffsets, types, posIncrements, null); + } + + public static void assertTokenStreamContents(TokenStream ts, String[] output) throws IOException { + assertTokenStreamContents(ts, output, null, null, null, null, null); + } + + public static void assertTokenStreamContents(TokenStream ts, String[] output, String[] types) throws IOException { + assertTokenStreamContents(ts, output, null, null, types, null, null); + } + + public static void assertTokenStreamContents(TokenStream ts, String[] output, int[] posIncrements) throws IOException { + assertTokenStreamContents(ts, output, null, null, null, posIncrements, null); + } + + public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[]) throws IOException { + assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, null, null); + } + + public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], Integer finalOffset) throws IOException { + assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, null, finalOffset); + } + + public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements) throws IOException { + assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, posIncrements, null); + } + + public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements, Integer finalOffset) throws IOException { + assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, posIncrements, finalOffset); + } + + public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException { + assertTokenStreamContents(a.tokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements, input.length()); + } + + public static void assertAnalyzesTo(Analyzer a, String input, String[] output) throws IOException { + assertAnalyzesTo(a, input, output, null, null, null, null); + } + + public static void assertAnalyzesTo(Analyzer a, String input, String[] output, String[] types) throws IOException { + assertAnalyzesTo(a, input, output, null, null, types, null); + } + + public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int[] posIncrements) throws IOException { + assertAnalyzesTo(a, input, output, null, null, null, posIncrements); + } + + public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[]) throws IOException { + assertAnalyzesTo(a, input, output, startOffsets, endOffsets, null, null); + } + + public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements) throws IOException { + assertAnalyzesTo(a, input, output, startOffsets, endOffsets, null, posIncrements); + } + + + public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException { + assertTokenStreamContents(a.tokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements, input.length()); + } + + public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output) throws IOException { + assertAnalyzesToReuse(a, input, output, null, null, null, null); + } + + public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, String[] types) throws IOException { + assertAnalyzesToReuse(a, input, output, null, null, types, null); + } + + public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int[] posIncrements) throws IOException { + assertAnalyzesToReuse(a, input, output, null, null, null, posIncrements); + } + + public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[]) throws IOException { + assertAnalyzesToReuse(a, input, output, startOffsets, endOffsets, null, null); + } + + public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements) throws IOException { + assertAnalyzesToReuse(a, input, output, startOffsets, endOffsets, null, posIncrements); + } + + // simple utility method for testing stemmers + + public static void checkOneTerm(Analyzer a, final String input, final String expected) throws IOException { + assertAnalyzesTo(a, input, new String[]{expected}); + } + + public static void checkOneTermReuse(Analyzer a, final String input, final String expected) throws IOException { + assertAnalyzesToReuse(a, input, new String[]{expected}); + } + + // simple utility method for blasting tokenstreams with data to make sure they don't do anything crazy + + public static void checkRandomData(Random random, Analyzer a, int iterations) throws IOException { + checkRandomData(random, a, iterations, 20); + } + + public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength) throws IOException { + for (int i = 0; i < iterations; i++) { + String text; + switch(_TestUtil.nextInt(random, 0, 3)) { + case 0: + text = _TestUtil.randomSimpleString(random); + break; + case 1: + text = _TestUtil.randomRealisticUnicodeString(random, maxWordLength); + break; + default: + text = _TestUtil.randomUnicodeString(random, maxWordLength); + } + + if (VERBOSE) { + System.out.println("NOTE: BaseTokenStreamTestCase: get first token stream now text=" + text); + } + + TokenStream ts = a.tokenStream("dummy", new StringReader(text)); + assertTrue("has no CharTermAttribute", ts.hasAttribute(CharTermAttribute.class)); + CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class); + OffsetAttribute offsetAtt = ts.hasAttribute(OffsetAttribute.class) ? ts.getAttribute(OffsetAttribute.class) : null; + PositionIncrementAttribute posIncAtt = ts.hasAttribute(PositionIncrementAttribute.class) ? ts.getAttribute(PositionIncrementAttribute.class) : null; + TypeAttribute typeAtt = ts.hasAttribute(TypeAttribute.class) ? ts.getAttribute(TypeAttribute.class) : null; + List tokens = new ArrayList(); + List types = new ArrayList(); + List positions = new ArrayList(); + List startOffsets = new ArrayList(); + List endOffsets = new ArrayList(); + ts.reset(); + while (ts.incrementToken()) { + tokens.add(termAtt.toString()); + if (typeAtt != null) types.add(typeAtt.type()); + if (posIncAtt != null) positions.add(posIncAtt.getPositionIncrement()); + if (offsetAtt != null) { + startOffsets.add(offsetAtt.startOffset()); + endOffsets.add(offsetAtt.endOffset()); + } + } + ts.end(); + ts.close(); + // verify reusing is "reproducable" and also get the normal tokenstream sanity checks + if (!tokens.isEmpty()) { + if (VERBOSE) { + System.out.println("NOTE: BaseTokenStreamTestCase: re-run analysis"); + } + if (typeAtt != null && posIncAtt != null && offsetAtt != null) { + // offset + pos + type + assertAnalyzesToReuse(a, text, + tokens.toArray(new String[tokens.size()]), + toIntArray(startOffsets), + toIntArray(endOffsets), + types.toArray(new String[types.size()]), + toIntArray(positions)); + } else if (posIncAtt != null && offsetAtt != null) { + // offset + pos + assertAnalyzesToReuse(a, text, + tokens.toArray(new String[tokens.size()]), + toIntArray(startOffsets), + toIntArray(endOffsets), + toIntArray(positions)); + } else if (offsetAtt != null) { + // offset + assertAnalyzesToReuse(a, text, + tokens.toArray(new String[tokens.size()]), + toIntArray(startOffsets), + toIntArray(endOffsets)); + } else { + // terms only + assertAnalyzesToReuse(a, text, + tokens.toArray(new String[tokens.size()])); + } + } + } + } + + static int[] toIntArray(List list) { + int ret[] = new int[list.size()]; + int offset = 0; + for (Integer i : list) { + ret[offset++] = i; + } + return ret; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/CollationTestBase.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/CollationTestBase.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/CollationTestBase.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/CollationTestBase.java 2011-11-01 21:10:36.000609905 -0400 @@ -0,0 +1,308 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; +import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.Term; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.TermRangeFilter; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TermRangeQuery; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IndexableBinaryStringTools; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +import java.io.StringReader; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +public abstract class CollationTestBase extends LuceneTestCase { + + protected String firstRangeBeginningOriginal = "\u062F"; + protected String firstRangeEndOriginal = "\u0698"; + + protected String secondRangeBeginningOriginal = "\u0633"; + protected String secondRangeEndOriginal = "\u0638"; + + /** + * Convenience method to perform the same function as CollationKeyFilter. + * + * @param keyBits the result from + * collator.getCollationKey(original).toByteArray() + * @return The encoded collation key for the original String + * @deprecated only for testing deprecated filters + */ + @Deprecated + protected String encodeCollationKey(byte[] keyBits) { + // Ensure that the backing char[] array is large enough to hold the encoded + // Binary String + int encodedLength = IndexableBinaryStringTools.getEncodedLength(keyBits, 0, keyBits.length); + char[] encodedBegArray = new char[encodedLength]; + IndexableBinaryStringTools.encode(keyBits, 0, keyBits.length, encodedBegArray, 0, encodedLength); + return new String(encodedBegArray); + } + + public void testFarsiRangeFilterCollating(Analyzer analyzer, BytesRef firstBeg, + BytesRef firstEnd, BytesRef secondBeg, + BytesRef secondEnd) throws Exception { + RAMDirectory ramDir = new RAMDirectory(); + IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig( + TEST_VERSION_CURRENT, analyzer)); + Document doc = new Document(); + doc.add(new Field("content", "\u0633\u0627\u0628", TextField.TYPE_STORED)); + doc.add(new Field("body", "body", StringField.TYPE_STORED)); + writer.addDocument(doc); + writer.close(); + IndexSearcher searcher = new IndexSearcher(ramDir, true); + Query query = new TermQuery(new Term("body","body")); + + // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi + // orders the U+0698 character before the U+0633 character, so the single + // index Term below should NOT be returned by a TermRangeFilter with a Farsi + // Collator (or an Arabic one for the case when Farsi searcher not + // supported). + ScoreDoc[] result = searcher.search + (query, new TermRangeFilter("content", firstBeg, firstEnd, true, true), 1).scoreDocs; + assertEquals("The index Term should not be included.", 0, result.length); + + result = searcher.search + (query, new TermRangeFilter("content", secondBeg, secondEnd, true, true), 1).scoreDocs; + assertEquals("The index Term should be included.", 1, result.length); + + searcher.close(); + } + + public void testFarsiRangeQueryCollating(Analyzer analyzer, BytesRef firstBeg, + BytesRef firstEnd, BytesRef secondBeg, + BytesRef secondEnd) throws Exception { + RAMDirectory ramDir = new RAMDirectory(); + IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig( + TEST_VERSION_CURRENT, analyzer)); + Document doc = new Document(); + + // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi + // orders the U+0698 character before the U+0633 character, so the single + // index Term below should NOT be returned by a TermRangeQuery with a Farsi + // Collator (or an Arabic one for the case when Farsi is not supported). + doc.add(new Field("content", "\u0633\u0627\u0628", TextField.TYPE_STORED)); + writer.addDocument(doc); + writer.close(); + IndexSearcher searcher = new IndexSearcher(ramDir, true); + + Query query = new TermRangeQuery("content", firstBeg, firstEnd, true, true); + ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; + assertEquals("The index Term should not be included.", 0, hits.length); + + query = new TermRangeQuery("content", secondBeg, secondEnd, true, true); + hits = searcher.search(query, null, 1000).scoreDocs; + assertEquals("The index Term should be included.", 1, hits.length); + searcher.close(); + } + + public void testFarsiTermRangeQuery(Analyzer analyzer, BytesRef firstBeg, + BytesRef firstEnd, BytesRef secondBeg, BytesRef secondEnd) throws Exception { + + RAMDirectory farsiIndex = new RAMDirectory(); + IndexWriter writer = new IndexWriter(farsiIndex, new IndexWriterConfig( + TEST_VERSION_CURRENT, analyzer)); + Document doc = new Document(); + doc.add(new Field("content", "\u0633\u0627\u0628", TextField.TYPE_STORED)); + doc.add(new Field("body", "body", StringField.TYPE_STORED)); + writer.addDocument(doc); + writer.close(); + + IndexReader reader = IndexReader.open(farsiIndex, true); + IndexSearcher search = newSearcher(reader); + + // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi + // orders the U+0698 character before the U+0633 character, so the single + // index Term below should NOT be returned by a TermRangeQuery + // with a Farsi Collator (or an Arabic one for the case when Farsi is + // not supported). + Query csrq + = new TermRangeQuery("content", firstBeg, firstEnd, true, true); + ScoreDoc[] result = search.search(csrq, null, 1000).scoreDocs; + assertEquals("The index Term should not be included.", 0, result.length); + + csrq = new TermRangeQuery + ("content", secondBeg, secondEnd, true, true); + result = search.search(csrq, null, 1000).scoreDocs; + assertEquals("The index Term should be included.", 1, result.length); + search.close(); + } + + // Test using various international locales with accented characters (which + // sort differently depending on locale) + // + // Copied (and slightly modified) from + // org.apache.lucene.search.TestSort.testInternationalSort() + // + // TODO: this test is really fragile. there are already 3 different cases, + // depending upon unicode version. + public void testCollationKeySort(Analyzer usAnalyzer, + Analyzer franceAnalyzer, + Analyzer swedenAnalyzer, + Analyzer denmarkAnalyzer, + String usResult, + String frResult, + String svResult, + String dkResult) throws Exception { + RAMDirectory indexStore = new RAMDirectory(); + IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig( + TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false))); + + // document data: + // the tracer field is used to determine which document was hit + String[][] sortData = new String[][] { + // tracer contents US France Sweden (sv_SE) Denmark (da_DK) + { "A", "x", "p\u00EAche", "p\u00EAche", "p\u00EAche", "p\u00EAche" }, + { "B", "y", "HAT", "HAT", "HAT", "HAT" }, + { "C", "x", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9" }, + { "D", "y", "HUT", "HUT", "HUT", "HUT" }, + { "E", "x", "peach", "peach", "peach", "peach" }, + { "F", "y", "H\u00C5T", "H\u00C5T", "H\u00C5T", "H\u00C5T" }, + { "G", "x", "sin", "sin", "sin", "sin" }, + { "H", "y", "H\u00D8T", "H\u00D8T", "H\u00D8T", "H\u00D8T" }, + { "I", "x", "s\u00EDn", "s\u00EDn", "s\u00EDn", "s\u00EDn" }, + { "J", "y", "HOT", "HOT", "HOT", "HOT" }, + }; + + FieldType customType = new FieldType(); + customType.setStored(true); + + for (int i = 0 ; i < sortData.length ; ++i) { + Document doc = new Document(); + doc.add(new Field("tracer", sortData[i][0], customType)); + doc.add(new TextField("contents", sortData[i][1])); + if (sortData[i][2] != null) + doc.add(new TextField("US", usAnalyzer.tokenStream("US", new StringReader(sortData[i][2])))); + if (sortData[i][3] != null) + doc.add(new TextField("France", franceAnalyzer.tokenStream("France", new StringReader(sortData[i][3])))); + if (sortData[i][4] != null) + doc.add(new TextField("Sweden", swedenAnalyzer.tokenStream("Sweden", new StringReader(sortData[i][4])))); + if (sortData[i][5] != null) + doc.add(new TextField("Denmark", denmarkAnalyzer.tokenStream("Denmark", new StringReader(sortData[i][5])))); + writer.addDocument(doc); + } + writer.optimize(); + writer.close(); + IndexSearcher searcher = new IndexSearcher(indexStore, true); + + Sort sort = new Sort(); + Query queryX = new TermQuery(new Term ("contents", "x")); + Query queryY = new TermQuery(new Term ("contents", "y")); + + sort.setSort(new SortField("US", SortField.Type.STRING)); + assertMatches(searcher, queryY, sort, usResult); + + sort.setSort(new SortField("France", SortField.Type.STRING)); + assertMatches(searcher, queryX, sort, frResult); + + sort.setSort(new SortField("Sweden", SortField.Type.STRING)); + assertMatches(searcher, queryY, sort, svResult); + + sort.setSort(new SortField("Denmark", SortField.Type.STRING)); + assertMatches(searcher, queryY, sort, dkResult); + } + + // Make sure the documents returned by the search match the expected list + // Copied from TestSort.java + private void assertMatches(IndexSearcher searcher, Query query, Sort sort, + String expectedResult) throws IOException { + ScoreDoc[] result = searcher.search(query, null, 1000, sort).scoreDocs; + StringBuilder buff = new StringBuilder(10); + int n = result.length; + for (int i = 0 ; i < n ; ++i) { + Document doc = searcher.doc(result[i].doc); + IndexableField[] v = doc.getFields("tracer"); + for (int j = 0 ; j < v.length ; ++j) { + buff.append(v[j].stringValue()); + } + } + assertEquals(expectedResult, buff.toString()); + } + + public void assertThreadSafe(final Analyzer analyzer) throws Exception { + int numTestPoints = 100; + int numThreads = _TestUtil.nextInt(random, 3, 5); + final HashMap map = new HashMap(); + + // create a map up front. + // then with multiple threads, generate sort keys for all the keys in the map + // and ensure they are the same as the ones we produced in serial fashion. + + for (int i = 0; i < numTestPoints; i++) { + String term = _TestUtil.randomSimpleString(random); + TokenStream ts = analyzer.tokenStream("fake", new StringReader(term)); + TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); + BytesRef bytes = termAtt.getBytesRef(); + ts.reset(); + assertTrue(ts.incrementToken()); + termAtt.fillBytesRef(); + // ensure we make a copy of the actual bytes too + map.put(term, new BytesRef(bytes)); + } + + Thread threads[] = new Thread[numThreads]; + for (int i = 0; i < numThreads; i++) { + threads[i] = new Thread() { + @Override + public void run() { + try { + for (Map.Entry mapping : map.entrySet()) { + String term = mapping.getKey(); + BytesRef expected = mapping.getValue(); + TokenStream ts = analyzer.tokenStream("fake", new StringReader(term)); + TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); + BytesRef bytes = termAtt.getBytesRef(); + ts.reset(); + assertTrue(ts.incrementToken()); + termAtt.fillBytesRef(); + assertEquals(expected, bytes); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }; + } + for (int i = 0; i < numThreads; i++) { + threads[i].start(); + } + for (int i = 0; i < numThreads; i++) { + threads[i].join(); + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockAnalyzer.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockAnalyzer.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockAnalyzer.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockAnalyzer.java 2011-11-01 21:10:36.000609905 -0400 @@ -0,0 +1,140 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Reader; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; + +/** + * Analyzer for testing + *

+ * This analyzer is a replacement for Whitespace/Simple/KeywordAnalyzers + * for unit tests. If you are testing a custom component such as a queryparser + * or analyzer-wrapper that consumes analysis streams, its a great idea to test + * it with this analyzer instead. MockAnalyzer has the following behavior: + *

    + *
  • By default, the assertions in {@link MockTokenizer} are turned on for extra + * checks that the consumer is consuming properly. These checks can be disabled + * with {@link #setEnableChecks(boolean)}. + *
  • Payload data is randomly injected into the stream for more thorough testing + * of payloads. + *
+ * @see MockTokenizer + */ +public final class MockAnalyzer extends Analyzer { + private final CharacterRunAutomaton runAutomaton; + private final boolean lowerCase; + private final CharacterRunAutomaton filter; + private final boolean enablePositionIncrements; + private int positionIncrementGap; + private final Random random; + private Map previousMappings = new HashMap(); + private boolean enableChecks = true; + + /** + * Creates a new MockAnalyzer. + * + * @param random Random for payloads behavior + * @param runAutomaton DFA describing how tokenization should happen (e.g. [a-zA-Z]+) + * @param lowerCase true if the tokenizer should lowercase terms + * @param filter DFA describing how terms should be filtered (set of stopwords, etc) + * @param enablePositionIncrements true if position increments should reflect filtered terms. + */ + public MockAnalyzer(Random random, CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter, boolean enablePositionIncrements) { + super(new PerFieldReuseStrategy()); + this.random = random; + this.runAutomaton = runAutomaton; + this.lowerCase = lowerCase; + this.filter = filter; + this.enablePositionIncrements = enablePositionIncrements; + } + + /** + * Calls {@link #MockAnalyzer(Random, CharacterRunAutomaton, boolean, CharacterRunAutomaton, boolean) + * MockAnalyzer(random, runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false}). + */ + public MockAnalyzer(Random random, CharacterRunAutomaton runAutomaton, boolean lowerCase) { + this(random, runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false); + } + + /** + * Create a Whitespace-lowercasing analyzer with no stopwords removal. + *

+ * Calls {@link #MockAnalyzer(Random, CharacterRunAutomaton, boolean, CharacterRunAutomaton, boolean) + * MockAnalyzer(random, MockTokenizer.WHITESPACE, true, MockTokenFilter.EMPTY_STOPSET, false}). + */ + public MockAnalyzer(Random random) { + this(random, MockTokenizer.WHITESPACE, true); + } + + @Override + public TokenStreamComponents createComponents(String fieldName, Reader reader) { + MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase); + tokenizer.setEnableChecks(enableChecks); + TokenFilter filt = new MockTokenFilter(tokenizer, filter, enablePositionIncrements); + return new TokenStreamComponents(tokenizer, maybePayload(filt, fieldName)); + } + + private synchronized TokenFilter maybePayload(TokenFilter stream, String fieldName) { + Integer val = previousMappings.get(fieldName); + if (val == null) { + val = -1; // no payloads + if (LuceneTestCase.rarely(random)) { + switch(random.nextInt(3)) { + case 0: val = -1; // no payloads + break; + case 1: val = Integer.MAX_VALUE; // variable length payload + break; + case 2: val = random.nextInt(12); // fixed length payload + break; + } + } + previousMappings.put(fieldName, val); // save it so we are consistent for this field + } + + if (val == -1) + return stream; + else if (val == Integer.MAX_VALUE) + return new MockVariableLengthPayloadFilter(random, stream); + else + return new MockFixedLengthPayloadFilter(random, stream, val); + } + + public void setPositionIncrementGap(int positionIncrementGap){ + this.positionIncrementGap = positionIncrementGap; + } + + @Override + public int getPositionIncrementGap(String fieldName){ + return positionIncrementGap; + } + + /** + * Toggle consumer workflow checking: if your test consumes tokenstreams normally you + * should leave this enabled. + */ + public void setEnableChecks(boolean enableChecks) { + this.enableChecks = enableChecks; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java 2011-11-01 21:10:36.000609905 -0400 @@ -0,0 +1,49 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Random; + +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.index.Payload; + +public final class MockFixedLengthPayloadFilter extends TokenFilter { + private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class); + private final Random random; + private final byte[] bytes; + private final Payload payload; + + public MockFixedLengthPayloadFilter(Random random, TokenStream in, int length) { + super(in); + this.random = random; + this.bytes = new byte[length]; + this.payload = new Payload(bytes); + } + + @Override + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + random.nextBytes(bytes); + payloadAtt.setPayload(payload); + return true; + } else { + return false; + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockPayloadAnalyzer.java 2011-11-01 21:10:36.000609905 -0400 @@ -0,0 +1,93 @@ +package org.apache.lucene.analysis; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.index.Payload; + +import java.io.IOException; +import java.io.Reader; + + +/** + * + * + **/ +public final class MockPayloadAnalyzer extends Analyzer { + + @Override + public TokenStreamComponents createComponents(String fieldName, Reader reader) { + Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true); + return new TokenStreamComponents(result, new MockPayloadFilter(result, fieldName)); + } +} + +/** + * + * + **/ +final class MockPayloadFilter extends TokenFilter { + String fieldName; + + int pos; + + int i; + + final PositionIncrementAttribute posIncrAttr; + final PayloadAttribute payloadAttr; + final CharTermAttribute termAttr; + + public MockPayloadFilter(TokenStream input, String fieldName) { + super(input); + this.fieldName = fieldName; + pos = 0; + i = 0; + posIncrAttr = input.addAttribute(PositionIncrementAttribute.class); + payloadAttr = input.addAttribute(PayloadAttribute.class); + termAttr = input.addAttribute(CharTermAttribute.class); + } + + @Override + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + payloadAttr.setPayload(new Payload(("pos: " + pos).getBytes())); + int posIncr; + if (i % 2 == 1) { + posIncr = 1; + } else { + posIncr = 0; + } + posIncrAttr.setPositionIncrement(posIncr); + pos += posIncr; + i++; + return true; + } else { + return false; + } + } + + @Override + public void reset() throws IOException { + super.reset(); + i = 0; + pos = 0; + } +} + diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenFilter.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenFilter.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenFilter.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenFilter.java 2011-11-01 21:10:36.000609905 -0400 @@ -0,0 +1,108 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import static org.apache.lucene.util.automaton.BasicAutomata.makeEmpty; +import static org.apache.lucene.util.automaton.BasicAutomata.makeString; + +import java.io.IOException; +import java.util.Arrays; + +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.util.automaton.BasicOperations; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; + +/** + * A tokenfilter for testing that removes terms accepted by a DFA. + *

    + *
  • Union a list of singletons to act like a stopfilter. + *
  • Use the complement to act like a keepwordfilter + *
  • Use a regex like .{12,} to act like a lengthfilter + *
+ */ +public final class MockTokenFilter extends TokenFilter { + /** Empty set of stopwords */ + public static final CharacterRunAutomaton EMPTY_STOPSET = + new CharacterRunAutomaton(makeEmpty()); + + /** Set of common english stopwords */ + public static final CharacterRunAutomaton ENGLISH_STOPSET = + new CharacterRunAutomaton(BasicOperations.union(Arrays.asList( + makeString("a"), makeString("an"), makeString("and"), makeString("are"), + makeString("as"), makeString("at"), makeString("be"), makeString("but"), + makeString("by"), makeString("for"), makeString("if"), makeString("in"), + makeString("into"), makeString("is"), makeString("it"), makeString("no"), + makeString("not"), makeString("of"), makeString("on"), makeString("or"), + makeString("such"), makeString("that"), makeString("the"), makeString("their"), + makeString("then"), makeString("there"), makeString("these"), makeString("they"), + makeString("this"), makeString("to"), makeString("was"), makeString("will"), + makeString("with")))); + + private final CharacterRunAutomaton filter; + private boolean enablePositionIncrements = false; + + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); + + /** + * Create a new MockTokenFilter. + * + * @param input TokenStream to filter + * @param filter DFA representing the terms that should be removed. + * @param enablePositionIncrements true if the removal should accumulate position increments. + */ + public MockTokenFilter(TokenStream input, CharacterRunAutomaton filter, boolean enablePositionIncrements) { + super(input); + this.filter = filter; + this.enablePositionIncrements = enablePositionIncrements; + } + + @Override + public boolean incrementToken() throws IOException { + // return the first non-stop word found + int skippedPositions = 0; + while (input.incrementToken()) { + if (!filter.run(termAtt.buffer(), 0, termAtt.length())) { + if (enablePositionIncrements) { + posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions); + } + return true; + } + skippedPositions += posIncrAtt.getPositionIncrement(); + } + // reached EOS -- return false + return false; + } + + /** + * @see #setEnablePositionIncrements(boolean) + */ + public boolean getEnablePositionIncrements() { + return enablePositionIncrements; + } + + /** + * If true, this Filter will preserve + * positions of the incoming tokens (ie, accumulate and + * set position increments of the removed stop tokens). + */ + public void setEnablePositionIncrements(boolean enable) { + this.enablePositionIncrements = enable; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockTokenizer.java 2011-11-01 21:10:36.000609905 -0400 @@ -0,0 +1,204 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.Reader; + +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.util.AttributeSource.AttributeFactory; +import org.apache.lucene.util.automaton.CharacterRunAutomaton; +import org.apache.lucene.util.automaton.RegExp; + +/** + * Tokenizer for testing. + *

+ * This tokenizer is a replacement for {@link #WHITESPACE}, {@link #SIMPLE}, and {@link #KEYWORD} + * tokenizers. If you are writing a component such as a TokenFilter, its a great idea to test + * it wrapping this tokenizer instead for extra checks. This tokenizer has the following behavior: + *

    + *
  • An internal state-machine is used for checking consumer consistency. These checks can + * be disabled with {@link #setEnableChecks(boolean)}. + *
  • For convenience, optionally lowercases terms that it outputs. + *
+ */ +public class MockTokenizer extends Tokenizer { + /** Acts Similar to WhitespaceTokenizer */ + public static final CharacterRunAutomaton WHITESPACE = + new CharacterRunAutomaton(new RegExp("[^ \t\r\n]+").toAutomaton()); + /** Acts Similar to KeywordTokenizer. + * TODO: Keyword returns an "empty" token for an empty reader... + */ + public static final CharacterRunAutomaton KEYWORD = + new CharacterRunAutomaton(new RegExp(".*").toAutomaton()); + /** Acts like LetterTokenizer. */ + // the ugly regex below is Unicode 5.2 [:Letter:] + public static final CharacterRunAutomaton SIMPLE = + new CharacterRunAutomaton(new RegExp("[A-Za-zªµºÀ-ÖØ-öø-ˁˆ-ˑˠ-ˤˬˮͰ-ʹͶͷͺ-ͽΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԥԱ-Ֆՙա-ևא-תװ-ײء-يٮٯٱ-ۓەۥۦۮۯۺ-ۼۿܐܒ-ܯݍ-ޥޱߊ-ߪߴߵߺࠀ-ࠕࠚࠤࠨऄ-हऽॐक़-ॡॱॲॹ-ॿঅ-ঌএঐও-নপ-রলশ-হঽৎড়ঢ়য়-ৡৰৱਅ-ਊਏਐਓ-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹਖ਼-ੜਫ਼ੲ-ੴઅ-ઍએ-ઑઓ-નપ-રલળવ-હઽૐૠૡଅ-ଌଏଐଓ-ନପ-ରଲଳଵ-ହଽଡ଼ଢ଼ୟ-ୡୱஃஅ-ஊஎ-ஐஒ-கஙசஜஞடணதந-பம-ஹௐఅ-ఌఎ-ఐఒ-నప-ళవ-హఽౘౙౠౡಅ-ಌಎ-ಐಒ-ನಪ-ಳವ-ಹಽೞೠೡഅ-ഌഎ-ഐഒ-നപ-ഹഽൠൡൺ-ൿඅ-ඖක-නඳ-රලව-ෆก-ะาำเ-ๆກຂຄງຈຊຍດ-ທນ-ຟມ-ຣລວສຫອ-ະາຳຽເ-ໄໆໜໝༀཀ-ཇཉ-ཬྈ-ྋက-ဪဿၐ-ၕၚ-ၝၡၥၦၮ-ၰၵ-ႁႎႠ-Ⴥა-ჺჼᄀ-ቈቊ-ቍቐ-ቖቘቚ-ቝበ-ኈኊ-ኍነ-ኰኲ-ኵኸ-ኾዀዂ-ዅወ-ዖዘ-ጐጒ-ጕጘ-ፚᎀ-ᎏᎠ-Ᏼᐁ-ᙬᙯ-ᙿᚁ-ᚚᚠ-ᛪᜀ-ᜌᜎ-ᜑᜠ-ᜱᝀ-ᝑᝠ-ᝬᝮ-ᝰក-ឳៗៜᠠ-ᡷᢀ-ᢨᢪᢰ-ᣵᤀ-ᤜᥐ-ᥭᥰ-ᥴᦀ-ᦫᧁ-ᧇᨀ-ᨖᨠ-ᩔᪧᬅ-ᬳᭅ-ᭋᮃ-ᮠᮮᮯᰀ-ᰣᱍ-ᱏᱚ-ᱽᳩ-ᳬᳮ-ᳱᴀ-ᶿḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼⁱⁿₐ-ₔℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-Ⱞⰰ-ⱞⱠ-ⳤⳫ-ⳮⴀ-ⴥⴰ-ⵥⵯⶀ-ⶖⶠ-ⶦⶨ-ⶮⶰ-ⶶⶸ-ⶾⷀ-ⷆⷈ-ⷎⷐ-ⷖⷘ-ⷞⸯ々〆〱-〵〻〼ぁ-ゖゝ-ゟァ-ヺー-ヿㄅ-ㄭㄱ-ㆎㆠ-ㆷㇰ-ㇿ㐀-䶵一-鿋ꀀ-ꒌꓐ-ꓽꔀ-ꘌꘐ-ꘟꘪꘫꙀ-ꙟꙢ-ꙮꙿ-ꚗꚠ-ꛥꜗ-ꜟꜢ-ꞈꞋꞌꟻ-ꠁꠃ-ꠅꠇ-ꠊꠌ-ꠢꡀ-ꡳꢂ-ꢳꣲ-ꣷꣻꤊ-ꤥꤰ-ꥆꥠ-ꥼꦄ-ꦲꧏꨀ-ꨨꩀ-ꩂꩄ-ꩋꩠ-ꩶꩺꪀ-ꪯꪱꪵꪶꪹ-ꪽꫀꫂꫛ-ꫝꯀ-ꯢ가-힣ힰ-ퟆퟋ-ퟻ豈-鶴侮-舘並-龎ff-stﬓ-ﬗיִײַ-ﬨשׁ-זּטּ-לּמּנּסּףּפּצּ-ﮱﯓ-ﴽﵐ-ﶏﶒ-ﷇﷰ-ﷻﹰ-ﹴﹶ-ﻼＡ-Ｚａ-ｚｦ-ﾾￂ-ￇￊ-ￏￒ-ￗￚ-ￜ𐀀-𐀋𐀍-𐀦𐀨-𐀺𐀼𐀽𐀿-𐁍𐁐-𐁝𐂀-𐃺𐊀-𐊜𐊠-𐋐𐌀-𐌞𐌰-𐍀𐍂-𐍉𐎀-𐎝𐎠-𐏃𐏈-𐏏𐐀-𐒝𐠀-𐠅𐠈𐠊-𐠵𐠷𐠸𐠼𐠿-𐡕𐤀-𐤕𐤠-𐤹𐨀𐨐-𐨓𐨕-𐨗𐨙-𐨳𐩠-𐩼𐬀-𐬵𐭀-𐭕𐭠-𐭲𐰀-𐱈𑂃-𑂯𒀀-𒍮𓀀-𓐮𝐀-𝑔𝑖-𝒜𝒞𝒟𝒢𝒥𝒦𝒩-𝒬𝒮-𝒹𝒻𝒽-𝓃𝓅-𝔅𝔇-𝔊𝔍-𝔔𝔖-𝔜𝔞-𝔹𝔻-𝔾𝕀-𝕄𝕆𝕊-𝕐𝕒-𝚥𝚨-𝛀𝛂-𝛚𝛜-𝛺𝛼-𝜔𝜖-𝜴𝜶-𝝎𝝐-𝝮𝝰-𝞈𝞊-𝞨𝞪-𝟂𝟄-𝟋𠀀-𪛖𪜀-𫜴丽-𪘀]+").toAutomaton()); + + private final CharacterRunAutomaton runAutomaton; + private final boolean lowerCase; + private final int maxTokenLength; + public static final int DEFAULT_MAX_TOKEN_LENGTH = Integer.MAX_VALUE; + private int state; + + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); + int off = 0; + + // TODO: "register" with LuceneTestCase to ensure all streams are closed() ? + // currently, we can only check that the lifecycle is correct if someone is reusing, + // but not for "one-offs". + private static enum State { + SETREADER, // consumer set a reader input either via ctor or via reset(Reader) + RESET, // consumer has called reset() + INCREMENT, // consumer is consuming, has called incrementToken() == true + INCREMENT_FALSE, // consumer has called incrementToken() which returned false + END, // consumer has called end() to perform end of stream operations + CLOSE // consumer has called close() to release any resources + }; + + private State streamState = State.CLOSE; + private boolean enableChecks = true; + + public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) { + super(factory, input); + this.runAutomaton = runAutomaton; + this.lowerCase = lowerCase; + this.state = runAutomaton.getInitialState(); + this.streamState = State.SETREADER; + this.maxTokenLength = maxTokenLength; + } + + public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) { + this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, runAutomaton, lowerCase, maxTokenLength); + } + + public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) { + this(input, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH); + } + + @Override + public final boolean incrementToken() throws IOException { + assert !enableChecks || (streamState == State.RESET || streamState == State.INCREMENT) + : "incrementToken() called while in wrong state: " + streamState; + clearAttributes(); + for (;;) { + int startOffset = off; + int cp = readCodePoint(); + if (cp < 0) { + break; + } else if (isTokenChar(cp)) { + int endOffset; + do { + char chars[] = Character.toChars(normalize(cp)); + for (int i = 0; i < chars.length; i++) + termAtt.append(chars[i]); + endOffset = off; + if (termAtt.length() >= maxTokenLength) { + break; + } + cp = readCodePoint(); + } while (cp >= 0 && isTokenChar(cp)); + offsetAtt.setOffset(correctOffset(startOffset), correctOffset(endOffset)); + streamState = State.INCREMENT; + return true; + } + } + streamState = State.INCREMENT_FALSE; + return false; + } + + protected int readCodePoint() throws IOException { + int ch = input.read(); + if (ch < 0) { + return ch; + } else { + assert !Character.isLowSurrogate((char) ch); + off++; + if (Character.isHighSurrogate((char) ch)) { + int ch2 = input.read(); + if (ch2 >= 0) { + off++; + assert Character.isLowSurrogate((char) ch2); + return Character.toCodePoint((char) ch, (char) ch2); + } + } + return ch; + } + } + + protected boolean isTokenChar(int c) { + state = runAutomaton.step(state, c); + if (state < 0) { + state = runAutomaton.getInitialState(); + return false; + } else { + return true; + } + } + + protected int normalize(int c) { + return lowerCase ? Character.toLowerCase(c) : c; + } + + @Override + public void reset() throws IOException { + super.reset(); + state = runAutomaton.getInitialState(); + off = 0; + assert !enableChecks || streamState != State.RESET : "double reset()"; + streamState = State.RESET; + } + + @Override + public void close() throws IOException { + super.close(); + // in some exceptional cases (e.g. TestIndexWriterExceptions) a test can prematurely close() + // these tests should disable this check, by default we check the normal workflow. + // TODO: investigate the CachingTokenFilter "double-close"... for now we ignore this + assert !enableChecks || streamState == State.END || streamState == State.CLOSE : "close() called in wrong state: " + streamState; + streamState = State.CLOSE; + } + + @Override + public void reset(Reader input) throws IOException { + super.reset(input); + assert !enableChecks || streamState == State.CLOSE : "setReader() called in wrong state: " + streamState; + streamState = State.SETREADER; + } + + @Override + public void end() throws IOException { + int finalOffset = correctOffset(off); + offsetAtt.setOffset(finalOffset, finalOffset); + // some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns false. + // these tests should disable this check (in general you should consume the entire stream) + assert !enableChecks || streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!"; + streamState = State.END; + } + + /** + * Toggle consumer workflow checking: if your test consumes tokenstreams normally you + * should leave this enabled. + */ + public void setEnableChecks(boolean enableChecks) { + this.enableChecks = enableChecks; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java 2011-11-01 21:10:36.000609905 -0400 @@ -0,0 +1,51 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Random; + +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.index.Payload; + +public final class MockVariableLengthPayloadFilter extends TokenFilter { + private static final int MAXLENGTH = 129; + + private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class); + private final Random random; + private final byte[] bytes = new byte[MAXLENGTH]; + private final Payload payload; + + public MockVariableLengthPayloadFilter(Random random, TokenStream in) { + super(in); + this.random = random; + this.payload = new Payload(bytes); + } + + @Override + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + random.nextBytes(bytes); + payload.setData(bytes, 0, random.nextInt(MAXLENGTH)); + payloadAtt.setPayload(payload); + return true; + } else { + return false; + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/VocabularyAssert.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/VocabularyAssert.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/analysis/VocabularyAssert.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/analysis/VocabularyAssert.java 2011-11-01 21:10:36.000609905 -0400 @@ -0,0 +1,83 @@ +package org.apache.lucene.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.zip.ZipFile; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.junit.Assert; + +/** Utility class for doing vocabulary-based stemming tests */ +public class VocabularyAssert { + /** Run a vocabulary test against two data files. */ + public static void assertVocabulary(Analyzer a, InputStream voc, InputStream out) + throws IOException { + BufferedReader vocReader = new BufferedReader( + new InputStreamReader(voc, "UTF-8")); + BufferedReader outputReader = new BufferedReader( + new InputStreamReader(out, "UTF-8")); + String inputWord = null; + while ((inputWord = vocReader.readLine()) != null) { + String expectedWord = outputReader.readLine(); + Assert.assertNotNull(expectedWord); + BaseTokenStreamTestCase.checkOneTermReuse(a, inputWord, expectedWord); + } + } + + /** Run a vocabulary test against one file: tab separated. */ + public static void assertVocabulary(Analyzer a, InputStream vocOut) + throws IOException { + BufferedReader vocReader = new BufferedReader( + new InputStreamReader(vocOut, "UTF-8")); + String inputLine = null; + while ((inputLine = vocReader.readLine()) != null) { + if (inputLine.startsWith("#") || inputLine.trim().length() == 0) + continue; /* comment */ + String words[] = inputLine.split("\t"); + BaseTokenStreamTestCase.checkOneTermReuse(a, words[0], words[1]); + } + } + + /** Run a vocabulary test against two data files inside a zip file */ + public static void assertVocabulary(Analyzer a, File zipFile, String voc, String out) + throws IOException { + ZipFile zip = new ZipFile(zipFile); + InputStream v = zip.getInputStream(zip.getEntry(voc)); + InputStream o = zip.getInputStream(zip.getEntry(out)); + assertVocabulary(a, v, o); + v.close(); + o.close(); + zip.close(); + } + + /** Run a vocabulary test against a tab-separated data file inside a zip file */ + public static void assertVocabulary(Analyzer a, File zipFile, String vocOut) + throws IOException { + ZipFile zip = new ZipFile(zipFile); + InputStream vo = zip.getInputStream(zip.getEntry(vocOut)); + assertVocabulary(a, vo); + vo.close(); + zip.close(); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/lucene40ords/Lucene40WithOrds.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/lucene40ords/Lucene40WithOrds.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/lucene40ords/Lucene40WithOrds.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/lucene40ords/Lucene40WithOrds.java 2011-11-02 21:26:22.572564906 -0400 @@ -0,0 +1,145 @@ +package org.apache.lucene.index.codecs.lucene40ords; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.BlockTermsReader; +import org.apache.lucene.index.codecs.BlockTermsWriter; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; +import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.TermsIndexReaderBase; +import org.apache.lucene.index.codecs.TermsIndexWriterBase; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; + +// TODO: we could make separate base class that can wrapp +// any PostingsBaseFormat and make it ord-able... + +public class Lucene40WithOrds extends PostingsFormat { + + public Lucene40WithOrds() { + super("Lucene40WithOrds"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docs = new Lucene40PostingsWriter(state); + + // TODO: should we make the terms index more easily + // pluggable? Ie so that this codec would record which + // index impl was used, and switch on loading? + // Or... you must make a new Codec for this? + TermsIndexWriterBase indexWriter; + boolean success = false; + try { + indexWriter = new FixedGapTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + docs.close(); + } + } + + success = false; + try { + // Must use BlockTermsWriter (not BlockTree) because + // BlockTree doens't support ords (yet)... + FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, docs); + success = true; + return ret; + } finally { + if (!success) { + try { + docs.close(); + } finally { + indexWriter.close(); + } + } + } + } + + public final static int TERMS_CACHE_SIZE = 1024; + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase postings = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + TermsIndexReaderBase indexReader; + + boolean success = false; + try { + indexReader = new FixedGapTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.segmentSuffix, state.context); + success = true; + } finally { + if (!success) { + postings.close(); + } + } + + success = false; + try { + FieldsProducer ret = new BlockTermsReader(indexReader, + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postings, + state.context, + TERMS_CACHE_SIZE, + state.segmentSuffix); + success = true; + return ret; + } finally { + if (!success) { + try { + postings.close(); + } finally { + indexReader.close(); + } + } + } + } + + /** Extension of freq postings file */ + static final String FREQ_EXTENSION = "frq"; + + /** Extension of prox postings file */ + static final String PROX_EXTENSION = "prx"; + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockPostingsFormat.java 2011-11-02 21:26:22.560564909 -0400 @@ -0,0 +1,210 @@ +package org.apache.lucene.index.codecs.mockintblock; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.sep.IntStreamFactory; +import org.apache.lucene.index.codecs.sep.IntIndexInput; +import org.apache.lucene.index.codecs.sep.IntIndexOutput; +import org.apache.lucene.index.codecs.sep.SepPostingsReader; +import org.apache.lucene.index.codecs.sep.SepPostingsWriter; +import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput; +import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; +import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.BlockTermsReader; +import org.apache.lucene.index.codecs.BlockTermsWriter; +import org.apache.lucene.index.codecs.TermsIndexReaderBase; +import org.apache.lucene.index.codecs.TermsIndexWriterBase; +import org.apache.lucene.store.*; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; + +/** + * A silly test codec to verify core support for fixed + * sized int block encoders is working. The int encoder + * used here just writes each block as a series of vInt. + */ + +public class MockFixedIntBlockPostingsFormat extends PostingsFormat { + + private final int blockSize; + + public MockFixedIntBlockPostingsFormat() { + this(1); + } + + public MockFixedIntBlockPostingsFormat(int blockSize) { + super("MockFixedIntBlock"); + this.blockSize = blockSize; + } + + @Override + public String toString() { + return getName() + "(blockSize=" + blockSize + ")"; + } + + // only for testing + public IntStreamFactory getIntFactory() { + return new MockIntFactory(blockSize); + } + + public static class MockIntFactory extends IntStreamFactory { + private final int blockSize; + + public MockIntFactory(int blockSize) { + this.blockSize = blockSize; + } + + @Override + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + return new FixedIntBlockIndexInput(dir.openInput(fileName, context)) { + + @Override + protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException { + return new BlockReader() { + public void seek(long pos) {} + public void readBlock() throws IOException { + for(int i=0;i= 0; + out.writeVInt(buffer[i]); + } + } + }; + success = true; + return ret; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + } + } + } + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new MockIntFactory(blockSize)); + + boolean success = false; + TermsIndexWriterBase indexWriter; + try { + indexWriter = new FixedGapTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + postingsWriter.close(); + } + } + + success = false; + try { + FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, postingsWriter); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, + state.segmentInfo, + state.context, + new MockIntFactory(blockSize), state.segmentSuffix); + + TermsIndexReaderBase indexReader; + boolean success = false; + try { + indexReader = new FixedGapTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator(), state.segmentSuffix, + IOContext.DEFAULT); + success = true; + } finally { + if (!success) { + postingsReader.close(); + } + } + + success = false; + try { + FieldsProducer ret = new BlockTermsReader(indexReader, + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postingsReader, + state.context, + Lucene40PostingsFormat.TERMS_CACHE_SIZE, + state.segmentSuffix); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsReader.close(); + } finally { + indexReader.close(); + } + } + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + SepPostingsReader.files(segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockPostingsFormat.java 2011-11-02 21:26:22.560564909 -0400 @@ -0,0 +1,233 @@ +package org.apache.lucene.index.codecs.mockintblock; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.sep.IntStreamFactory; +import org.apache.lucene.index.codecs.sep.IntIndexInput; +import org.apache.lucene.index.codecs.sep.IntIndexOutput; +import org.apache.lucene.index.codecs.sep.SepPostingsReader; +import org.apache.lucene.index.codecs.sep.SepPostingsWriter; +import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput; +import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; +import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.BlockTermsReader; +import org.apache.lucene.index.codecs.BlockTermsWriter; +import org.apache.lucene.index.codecs.TermsIndexReaderBase; +import org.apache.lucene.index.codecs.TermsIndexWriterBase; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; + +/** + * A silly test codec to verify core support for variable + * sized int block encoders is working. The int encoder + * used here writes baseBlockSize ints at once, if the first + * int is <= 3, else 2*baseBlockSize. + */ + +public class MockVariableIntBlockPostingsFormat extends PostingsFormat { + private final int baseBlockSize; + + public MockVariableIntBlockPostingsFormat() { + this(1); + } + + public MockVariableIntBlockPostingsFormat(int baseBlockSize) { + super("MockVariableIntBlock"); + this.baseBlockSize = baseBlockSize; + } + + @Override + public String toString() { + return getName() + "(baseBlockSize="+ baseBlockSize + ")"; + } + + public static class MockIntFactory extends IntStreamFactory { + + private final int baseBlockSize; + + public MockIntFactory(int baseBlockSize) { + this.baseBlockSize = baseBlockSize; + } + + @Override + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + final IndexInput in = dir.openInput(fileName, context); + final int baseBlockSize = in.readInt(); + return new VariableIntBlockIndexInput(in) { + + @Override + protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException { + return new BlockReader() { + public void seek(long pos) {} + public int readBlock() throws IOException { + buffer[0] = in.readVInt(); + final int count = buffer[0] <= 3 ? baseBlockSize-1 : 2*baseBlockSize-1; + assert buffer.length >= count: "buffer.length=" + buffer.length + " count=" + count; + for(int i=0;i= 0; + buffer[pendingCount++] = value; + // silly variable block length int encoder: if + // first value <= 3, we write N vints at once; + // else, 2*N + final int flushAt = buffer[0] <= 3 ? baseBlockSize : 2*baseBlockSize; + + // intentionally be non-causal here: + if (pendingCount == flushAt+1) { + for(int i=0;i files) throws IOException { + SepPostingsReader.files(segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockrandom/MockRandomPostingsFormat.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockrandom/MockRandomPostingsFormat.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockrandom/MockRandomPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mockrandom/MockRandomPostingsFormat.java 2011-11-02 21:26:22.572564906 -0400 @@ -0,0 +1,435 @@ +package org.apache.lucene.index.codecs.mockrandom; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.Set; + +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.BlockTreeTermsReader; +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; +import org.apache.lucene.index.codecs.BlockTermsReader; +import org.apache.lucene.index.codecs.BlockTermsWriter; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; +import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.TermStats; +import org.apache.lucene.index.codecs.TermsIndexReaderBase; +import org.apache.lucene.index.codecs.TermsIndexWriterBase; +import org.apache.lucene.index.codecs.VariableGapTermsIndexReader; +import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; +import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSingleIntFactory; +import org.apache.lucene.index.codecs.pulsing.PulsingPostingsReader; +import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriter; +import org.apache.lucene.index.codecs.sep.IntIndexInput; +import org.apache.lucene.index.codecs.sep.IntIndexOutput; +import org.apache.lucene.index.codecs.sep.IntStreamFactory; +import org.apache.lucene.index.codecs.sep.SepPostingsReader; +import org.apache.lucene.index.codecs.sep.SepPostingsWriter; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +/** + * Randomly combines terms index impl w/ postings impls. + */ + +public class MockRandomPostingsFormat extends PostingsFormat { + private final Random seedRandom; + private final String SEED_EXT = "sd"; + + public MockRandomPostingsFormat() { + // just for reading, we are gonna setSeed from the .seed file... right? + this(new Random()); + } + + public MockRandomPostingsFormat(Random random) { + super("MockRandom"); + this.seedRandom = new Random(random.nextLong()); + } + + // Chooses random IntStreamFactory depending on file's extension + private static class MockIntStreamFactory extends IntStreamFactory { + private final int salt; + private final List delegates = new ArrayList(); + + public MockIntStreamFactory(Random random) { + salt = random.nextInt(); + delegates.add(new MockSingleIntFactory()); + final int blockSize = _TestUtil.nextInt(random, 1, 2000); + delegates.add(new MockFixedIntBlockPostingsFormat.MockIntFactory(blockSize)); + final int baseBlockSize = _TestUtil.nextInt(random, 1, 127); + delegates.add(new MockVariableIntBlockPostingsFormat.MockIntFactory(baseBlockSize)); + // TODO: others + } + + private static String getExtension(String fileName) { + final int idx = fileName.indexOf('.'); + assert idx != -1; + return fileName.substring(idx); + } + + @Override + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + // Must only use extension, because IW.addIndexes can + // rename segment! + final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size()); + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: read using int factory " + f + " from fileName=" + fileName); + } + return f.openInput(dir, fileName, context); + } + + @Override + public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context) throws IOException { + final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size()); + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: write using int factory " + f + " to fileName=" + fileName); + } + return f.createOutput(dir, fileName, context); + } + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + // we pull this before the seed intentionally: because its not consumed at runtime + // (the skipInterval is written into postings header) + int skipInterval = _TestUtil.nextInt(seedRandom, 2, 10); + + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: skipInterval=" + skipInterval); + } + + final long seed = seedRandom.nextLong(); + + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: writing to seg=" + state.segmentName + " formatID=" + state.segmentSuffix + " seed=" + seed); + } + + final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, SEED_EXT); + final IndexOutput out = state.directory.createOutput(seedFileName, state.context); + try { + out.writeLong(seed); + } finally { + out.close(); + } + + final Random random = new Random(seed); + + random.nextInt(); // consume a random for buffersize + + PostingsWriterBase postingsWriter; + if (random.nextBoolean()) { + postingsWriter = new SepPostingsWriter(state, new MockIntStreamFactory(random), skipInterval); + } else { + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: writing Standard postings"); + } + postingsWriter = new Lucene40PostingsWriter(state, skipInterval); + } + + if (random.nextBoolean()) { + final int totTFCutoff = _TestUtil.nextInt(random, 1, 20); + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: writing pulsing postings with totTFCutoff=" + totTFCutoff); + } + postingsWriter = new PulsingPostingsWriter(totTFCutoff, postingsWriter); + } + + final FieldsConsumer fields; + + if (random.nextBoolean()) { + // Use BlockTree terms dict + + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: writing BlockTree terms dict"); + } + + // TODO: would be nice to allow 1 but this is very + // slow to write + final int minTermsInBlock = _TestUtil.nextInt(random, 2, 100); + final int maxTermsInBlock = Math.max(2, (minTermsInBlock-1)*2 + random.nextInt(100)); + + boolean success = false; + try { + fields = new BlockTreeTermsWriter(state, postingsWriter, minTermsInBlock, maxTermsInBlock); + success = true; + } finally { + if (!success) { + postingsWriter.close(); + } + } + } else { + + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: writing Block terms dict"); + } + + boolean success = false; + + final TermsIndexWriterBase indexWriter; + try { + if (random.nextBoolean()) { + state.termIndexInterval = _TestUtil.nextInt(random, 1, 100); + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: fixed-gap terms index (tii=" + state.termIndexInterval + ")"); + } + indexWriter = new FixedGapTermsIndexWriter(state); + } else { + final VariableGapTermsIndexWriter.IndexTermSelector selector; + final int n2 = random.nextInt(3); + if (n2 == 0) { + final int tii = _TestUtil.nextInt(random, 1, 100); + selector = new VariableGapTermsIndexWriter.EveryNTermSelector(tii); + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: variable-gap terms index (tii=" + tii + ")"); + } + } else if (n2 == 1) { + final int docFreqThresh = _TestUtil.nextInt(random, 2, 100); + final int tii = _TestUtil.nextInt(random, 1, 100); + selector = new VariableGapTermsIndexWriter.EveryNOrDocFreqTermSelector(docFreqThresh, tii); + } else { + final long seed2 = random.nextLong(); + final int gap = _TestUtil.nextInt(random, 2, 40); + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: random-gap terms index (max gap=" + gap + ")"); + } + selector = new VariableGapTermsIndexWriter.IndexTermSelector() { + final Random rand = new Random(seed2); + + @Override + public boolean isIndexTerm(BytesRef term, TermStats stats) { + return rand.nextInt(gap) == gap/2; + } + + @Override + public void newField(FieldInfo fieldInfo) { + } + }; + } + indexWriter = new VariableGapTermsIndexWriter(state, selector); + } + success = true; + } finally { + if (!success) { + postingsWriter.close(); + } + } + + success = false; + try { + fields = new BlockTermsWriter(indexWriter, state, postingsWriter); + success = true; + } finally { + if (!success) { + try { + postingsWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + return fields; + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + + final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, SEED_EXT); + final IndexInput in = state.dir.openInput(seedFileName, state.context); + final long seed = in.readLong(); + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " formatID=" + state.segmentSuffix + " seed=" + seed); + } + in.close(); + + final Random random = new Random(seed); + + int readBufferSize = _TestUtil.nextInt(random, 1, 4096); + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: readBufferSize=" + readBufferSize); + } + + PostingsReaderBase postingsReader; + + if (random.nextBoolean()) { + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: reading Sep postings"); + } + postingsReader = new SepPostingsReader(state.dir, state.segmentInfo, + state.context, new MockIntStreamFactory(random), state.segmentSuffix); + } else { + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: reading Standard postings"); + } + postingsReader = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + } + + if (random.nextBoolean()) { + final int totTFCutoff = _TestUtil.nextInt(random, 1, 20); + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: reading pulsing postings with totTFCutoff=" + totTFCutoff); + } + postingsReader = new PulsingPostingsReader(postingsReader); + } + + final FieldsProducer fields; + + if (random.nextBoolean()) { + // Use BlockTree terms dict + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: reading BlockTree terms dict"); + } + + boolean success = false; + try { + fields = new BlockTreeTermsReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + postingsReader, + state.context, + state.segmentSuffix, + state.termsIndexDivisor); + success = true; + } finally { + if (!success) { + postingsReader.close(); + } + } + } else { + + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: reading Block terms dict"); + } + final TermsIndexReaderBase indexReader; + boolean success = false; + try { + final boolean doFixedGap = random.nextBoolean(); + + // randomness diverges from writer, here: + if (state.termsIndexDivisor != -1) { + state.termsIndexDivisor = _TestUtil.nextInt(random, 1, 10); + } + + if (doFixedGap) { + // if termsIndexDivisor is set to -1, we should not touch it. It means a + // test explicitly instructed not to load the terms index. + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: fixed-gap terms index (divisor=" + state.termsIndexDivisor + ")"); + } + indexReader = new FixedGapTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.segmentSuffix, state.context); + } else { + final int n2 = random.nextInt(3); + if (n2 == 1) { + random.nextInt(); + } else if (n2 == 2) { + random.nextLong(); + } + if (LuceneTestCase.VERBOSE) { + System.out.println("MockRandomCodec: variable-gap terms index (divisor=" + state.termsIndexDivisor + ")"); + } + indexReader = new VariableGapTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + state.segmentSuffix, state.context); + + } + + success = true; + } finally { + if (!success) { + postingsReader.close(); + } + } + + final int termsCacheSize = _TestUtil.nextInt(random, 1, 1024); + + success = false; + try { + fields = new BlockTermsReader(indexReader, + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postingsReader, + state.context, + termsCacheSize, + state.segmentSuffix); + success = true; + } finally { + if (!success) { + try { + postingsReader.close(); + } finally { + indexReader.close(); + } + } + } + } + + return fields; + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + final String seedFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SEED_EXT); + files.add(seedFileName); + SepPostingsReader.files(segmentInfo, segmentSuffix, files); + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + VariableGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + // hackish! + Iterator it = files.iterator(); + while(it.hasNext()) { + final String file = it.next(); + if (!dir.fileExists(file)) { + it.remove(); + } + } + //System.out.println("MockRandom.files return " + files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepDocValuesFormat.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepDocValuesFormat.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepDocValuesFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepDocValuesFormat.java 2011-11-02 21:26:22.564564909 -0400 @@ -0,0 +1,54 @@ +package org.apache.lucene.index.codecs.mocksep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.PerDocWriteState; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.PerDocConsumer; +import org.apache.lucene.index.codecs.PerDocValues; +import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; +import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; +import org.apache.lucene.store.Directory; + +/** + * Separate-file docvalues implementation + * @lucene.experimental + */ +// TODO: we could move this out of src/test ? +public class MockSepDocValuesFormat extends DocValuesFormat { + + @Override + public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { + return new SepDocValuesConsumer(state); + } + + @Override + public PerDocValues docsProducer(SegmentReadState state) throws IOException { + return new SepDocValuesProducer(state); + } + + @Override + public void files(Directory dir, SegmentInfo info, Set files) throws IOException { + SepDocValuesConsumer.files(dir, info, files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepPostingsFormat.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepPostingsFormat.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSepPostingsFormat.java 2011-11-02 21:26:22.564564909 -0400 @@ -0,0 +1,138 @@ +package org.apache.lucene.index.codecs.mocksep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; +import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.BlockTermsReader; +import org.apache.lucene.index.codecs.BlockTermsWriter; +import org.apache.lucene.index.codecs.TermsIndexReaderBase; +import org.apache.lucene.index.codecs.TermsIndexWriterBase; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.sep.SepPostingsWriter; +import org.apache.lucene.index.codecs.sep.SepPostingsReader; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; + +/** + * A silly codec that simply writes each file separately as + * single vInts. Don't use this (performance will be poor)! + * This is here just to test the core sep codec + * classes. + */ +public class MockSepPostingsFormat extends PostingsFormat { + + public MockSepPostingsFormat() { + super("MockSep"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + + PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new MockSingleIntFactory()); + + boolean success = false; + TermsIndexWriterBase indexWriter; + try { + indexWriter = new FixedGapTermsIndexWriter(state); + success = true; + } finally { + if (!success) { + postingsWriter.close(); + } + } + + success = false; + try { + FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, postingsWriter); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsWriter.close(); + } finally { + indexWriter.close(); + } + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + + PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, state.segmentInfo, + state.context, new MockSingleIntFactory(), state.segmentSuffix); + + TermsIndexReaderBase indexReader; + boolean success = false; + try { + indexReader = new FixedGapTermsIndexReader(state.dir, + state.fieldInfos, + state.segmentInfo.name, + state.termsIndexDivisor, + BytesRef.getUTF8SortedAsUnicodeComparator(), + state.segmentSuffix, state.context); + success = true; + } finally { + if (!success) { + postingsReader.close(); + } + } + + success = false; + try { + FieldsProducer ret = new BlockTermsReader(indexReader, + state.dir, + state.fieldInfos, + state.segmentInfo.name, + postingsReader, + state.context, + Lucene40PostingsFormat.TERMS_CACHE_SIZE, + state.segmentSuffix); + success = true; + return ret; + } finally { + if (!success) { + try { + postingsReader.close(); + } finally { + indexReader.close(); + } + } + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + SepPostingsReader.files(segmentInfo, segmentSuffix, files); + BlockTermsReader.files(dir, segmentInfo, segmentSuffix, files); + FixedGapTermsIndexReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java 2011-11-01 21:10:36.020609905 -0400 @@ -0,0 +1,38 @@ +package org.apache.lucene.index.codecs.mocksep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.index.codecs.sep.IntStreamFactory; +import org.apache.lucene.index.codecs.sep.IntIndexInput; +import org.apache.lucene.index.codecs.sep.IntIndexOutput; + +import java.io.IOException; + +/** @lucene.experimental */ +public class MockSingleIntFactory extends IntStreamFactory { + @Override + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + return new MockSingleIntIndexInput(dir, fileName, context); + } + @Override + public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context) throws IOException { + return new MockSingleIntIndexOutput(dir, fileName, context); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java 2011-11-01 21:10:36.020609905 -0400 @@ -0,0 +1,114 @@ +package org.apache.lucene.index.codecs.mocksep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.codecs.sep.IntIndexInput; +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.util.CodecUtil; + +/** Reads IndexInputs written with {@link + * MockSingleIntIndexOutput}. NOTE: this class is just for + * demonstration puprposes (it is a very slow way to read a + * block of ints). + * + * @lucene.experimental + */ +public class MockSingleIntIndexInput extends IntIndexInput { + private final IndexInput in; + + public MockSingleIntIndexInput(Directory dir, String fileName, IOContext context) + throws IOException { + in = dir.openInput(fileName, context); + CodecUtil.checkHeader(in, MockSingleIntIndexOutput.CODEC, + MockSingleIntIndexOutput.VERSION_START, + MockSingleIntIndexOutput.VERSION_START); + } + + @Override + public Reader reader() throws IOException { + return new Reader((IndexInput) in.clone()); + } + + @Override + public void close() throws IOException { + in.close(); + } + + public static class Reader extends IntIndexInput.Reader { + // clone: + private final IndexInput in; + + public Reader(IndexInput in) { + this.in = in; + } + + /** Reads next single int */ + @Override + public int next() throws IOException { + //System.out.println("msii.next() fp=" + in.getFilePointer() + " vs " + in.length()); + return in.readVInt(); + } + } + + class Index extends IntIndexInput.Index { + private long fp; + + @Override + public void read(DataInput indexIn, boolean absolute) + throws IOException { + if (absolute) { + fp = indexIn.readVLong(); + } else { + fp += indexIn.readVLong(); + } + } + + @Override + public void set(IntIndexInput.Index other) { + fp = ((Index) other).fp; + } + + @Override + public void seek(IntIndexInput.Reader other) throws IOException { + ((Reader) other).in.seek(fp); + } + + @Override + public String toString() { + return Long.toString(fp); + } + + @Override + public Object clone() { + Index other = new Index(); + other.fp = fp; + return other; + } + } + + @Override + public Index index() { + return new Index(); + } +} + diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java 2011-11-01 21:10:36.020609905 -0400 @@ -0,0 +1,104 @@ +package org.apache.lucene.index.codecs.mocksep; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.index.codecs.sep.IntIndexOutput; +import java.io.IOException; + +/** Writes ints directly to the file (not in blocks) as + * vInt. + * + * @lucene.experimental +*/ +public class MockSingleIntIndexOutput extends IntIndexOutput { + private final IndexOutput out; + final static String CODEC = "SINGLE_INTS"; + final static int VERSION_START = 0; + final static int VERSION_CURRENT = VERSION_START; + + public MockSingleIntIndexOutput(Directory dir, String fileName, IOContext context) throws IOException { + out = dir.createOutput(fileName, context); + boolean success = false; + try { + CodecUtil.writeHeader(out, CODEC, VERSION_CURRENT); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + } + } + } + + /** Write an int to the primary file */ + @Override + public void write(int v) throws IOException { + assert v >= 0; + out.writeVInt(v); + } + + @Override + public Index index() { + return new Index(); + } + + @Override + public void close() throws IOException { + out.close(); + } + + @Override + public String toString() { + return "MockSingleIntIndexOutput fp=" + out.getFilePointer(); + } + + private class Index extends IntIndexOutput.Index { + long fp; + long lastFP; + @Override + public void mark() { + fp = out.getFilePointer(); + } + @Override + public void copyFrom(IntIndexOutput.Index other, boolean copyLast) { + fp = ((Index) other).fp; + if (copyLast) { + lastFP = ((Index) other).fp; + } + } + @Override + public void write(IndexOutput indexOut, boolean absolute) + throws IOException { + if (absolute) { + indexOut.writeVLong(fp); + } else { + indexOut.writeVLong(fp - lastFP); + } + lastFP = fp; + } + + @Override + public String toString() { + return Long.toString(fp); + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/nestedpulsing/NestedPulsingPostingsFormat.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/nestedpulsing/NestedPulsingPostingsFormat.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/nestedpulsing/NestedPulsingPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/nestedpulsing/NestedPulsingPostingsFormat.java 2011-11-02 21:26:22.568564907 -0400 @@ -0,0 +1,99 @@ +package org.apache.lucene.index.codecs.nestedpulsing; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Set; + +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.BlockTreeTermsReader; +import org.apache.lucene.index.codecs.BlockTreeTermsWriter; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.PostingsReaderBase; +import org.apache.lucene.index.codecs.PostingsWriterBase; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsReader; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsWriter; +import org.apache.lucene.index.codecs.pulsing.PulsingPostingsReader; +import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriter; +import org.apache.lucene.store.Directory; + +/** + * Pulsing(1, Pulsing(2, Lucene40)) + * @lucene.experimental + */ +// TODO: if we create PulsingPostingsBaseFormat then we +// can simplify this? note: I don't like the *BaseFormat +// hierarchy, maybe we can clean that up... +public class NestedPulsingPostingsFormat extends PostingsFormat { + public NestedPulsingPostingsFormat() { + super("NestedPulsing"); + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + PostingsWriterBase docsWriter = new Lucene40PostingsWriter(state); + + PostingsWriterBase pulsingWriterInner = new PulsingPostingsWriter(2, docsWriter); + PostingsWriterBase pulsingWriter = new PulsingPostingsWriter(1, pulsingWriterInner); + + // Terms dict + boolean success = false; + try { + FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, + BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); + success = true; + return ret; + } finally { + if (!success) { + pulsingWriter.close(); + } + } + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + PostingsReaderBase docsReader = new Lucene40PostingsReader(state.dir, state.segmentInfo, state.context, state.segmentSuffix); + PostingsReaderBase pulsingReaderInner = new PulsingPostingsReader(docsReader); + PostingsReaderBase pulsingReader = new PulsingPostingsReader(pulsingReaderInner); + boolean success = false; + try { + FieldsProducer ret = new BlockTreeTermsReader( + state.dir, state.fieldInfos, state.segmentInfo.name, + pulsingReader, + state.context, + state.segmentSuffix, + state.termsIndexDivisor); + success = true; + return ret; + } finally { + if (!success) { + pulsingReader.close(); + } + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) throws IOException { + Lucene40PostingsReader.files(dir, segmentInfo, segmentSuffix, files); + BlockTreeTermsReader.files(dir, segmentInfo, segmentSuffix, files); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java 2011-11-01 21:10:36.068609905 -0400 @@ -0,0 +1,207 @@ +package org.apache.lucene.index.codecs.preflexrw; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Comparator; + +import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.PostingsConsumer; +import org.apache.lucene.index.codecs.TermStats; +import org.apache.lucene.index.codecs.TermsConsumer; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; +import org.apache.lucene.index.codecs.lucene3x.TermInfo; +import org.apache.lucene.index.codecs.lucene40.DefaultSkipListWriter; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.IOUtils; + +class PreFlexFieldsWriter extends FieldsConsumer { + + private final TermInfosWriter termsOut; + private final IndexOutput freqOut; + private final IndexOutput proxOut; + private final DefaultSkipListWriter skipListWriter; + private final int totalNumDocs; + + public PreFlexFieldsWriter(SegmentWriteState state) throws IOException { + termsOut = new TermInfosWriter(state.directory, + state.segmentName, + state.fieldInfos, + state.termIndexInterval); + + final String freqFile = IndexFileNames.segmentFileName(state.segmentName, "", Lucene3xPostingsFormat.FREQ_EXTENSION); + freqOut = state.directory.createOutput(freqFile, state.context); + totalNumDocs = state.numDocs; + + if (state.fieldInfos.hasProx()) { + final String proxFile = IndexFileNames.segmentFileName(state.segmentName, "", Lucene3xPostingsFormat.PROX_EXTENSION); + proxOut = state.directory.createOutput(proxFile, state.context); + } else { + proxOut = null; + } + + skipListWriter = new DefaultSkipListWriter(termsOut.skipInterval, + termsOut.maxSkipLevels, + totalNumDocs, + freqOut, + proxOut); + //System.out.println("\nw start seg=" + segment); + } + + @Override + public TermsConsumer addField(FieldInfo field) throws IOException { + assert field.number != -1; + //System.out.println("w field=" + field.name + " storePayload=" + field.storePayloads + " number=" + field.number); + return new PreFlexTermsWriter(field); + } + + @Override + public void close() throws IOException { + IOUtils.close(termsOut, freqOut, proxOut); + } + + private class PreFlexTermsWriter extends TermsConsumer { + private final FieldInfo fieldInfo; + private final boolean omitTF; + private final boolean storePayloads; + + private final TermInfo termInfo = new TermInfo(); + private final PostingsWriter postingsWriter = new PostingsWriter(); + + public PreFlexTermsWriter(FieldInfo fieldInfo) { + this.fieldInfo = fieldInfo; + omitTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY; + storePayloads = fieldInfo.storePayloads; + } + + private class PostingsWriter extends PostingsConsumer { + private int lastDocID; + private int lastPayloadLength = -1; + private int lastPosition; + private int df; + + public PostingsWriter reset() { + df = 0; + lastDocID = 0; + lastPayloadLength = -1; + return this; + } + + @Override + public void startDoc(int docID, int termDocFreq) throws IOException { + //System.out.println(" w doc=" + docID); + + final int delta = docID - lastDocID; + if (docID < 0 || (df > 0 && delta <= 0)) { + throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )"); + } + + if ((++df % termsOut.skipInterval) == 0) { + skipListWriter.setSkipData(lastDocID, storePayloads, lastPayloadLength); + skipListWriter.bufferSkip(df); + } + + lastDocID = docID; + + assert docID < totalNumDocs: "docID=" + docID + " totalNumDocs=" + totalNumDocs; + + if (omitTF) { + freqOut.writeVInt(delta); + } else { + final int code = delta << 1; + if (termDocFreq == 1) { + freqOut.writeVInt(code|1); + } else { + freqOut.writeVInt(code); + freqOut.writeVInt(termDocFreq); + } + } + lastPosition = 0; + } + + @Override + public void addPosition(int position, BytesRef payload) throws IOException { + assert proxOut != null; + + //System.out.println(" w pos=" + position + " payl=" + payload); + final int delta = position - lastPosition; + lastPosition = position; + + if (storePayloads) { + final int payloadLength = payload == null ? 0 : payload.length; + if (payloadLength != lastPayloadLength) { + //System.out.println(" write payload len=" + payloadLength); + lastPayloadLength = payloadLength; + proxOut.writeVInt((delta<<1)|1); + proxOut.writeVInt(payloadLength); + } else { + proxOut.writeVInt(delta << 1); + } + if (payloadLength > 0) { + proxOut.writeBytes(payload.bytes, payload.offset, payload.length); + } + } else { + proxOut.writeVInt(delta); + } + } + + @Override + public void finishDoc() throws IOException { + } + } + + @Override + public PostingsConsumer startTerm(BytesRef text) throws IOException { + //System.out.println(" w term=" + text.utf8ToString()); + skipListWriter.resetSkip(); + termInfo.freqPointer = freqOut.getFilePointer(); + if (proxOut != null) { + termInfo.proxPointer = proxOut.getFilePointer(); + } + return postingsWriter.reset(); + } + + @Override + public void finishTerm(BytesRef text, TermStats stats) throws IOException { + if (stats.docFreq > 0) { + long skipPointer = skipListWriter.writeSkip(freqOut); + termInfo.docFreq = stats.docFreq; + termInfo.skipOffset = (int) (skipPointer - termInfo.freqPointer); + //System.out.println(" w finish term=" + text.utf8ToString() + " fnum=" + fieldInfo.number); + termsOut.add(fieldInfo.number, + text, + termInfo); + } + } + + @Override + public void finish(long sumTotalTermCount, long sumDocFreq, int docCount) throws IOException { + } + + @Override + public Comparator getComparator() throws IOException { + return BytesRef.getUTF8SortedAsUTF16Comparator(); + } + } +} \ No newline at end of file diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java 2011-11-02 10:13:48.212585697 -0400 @@ -0,0 +1,39 @@ +package org.apache.lucene.index.codecs.preflexrw; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xCodec; +import org.apache.lucene.util.LuceneTestCase; + +/** + * Writes 3.x-like indexes (not perfect emulation yet) for testing only! + * @lucene.experimental + */ +public class PreFlexRWCodec extends Lucene3xCodec { + private final PostingsFormat postings = new PreFlexRWPostingsFormat(); + + @Override + public PostingsFormat postingsFormat() { + if (LuceneTestCase.PREFLEX_IMPERSONATION_IS_ACTIVE) { + return postings; + } else { + return super.postingsFormat(); + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWPostingsFormat.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWPostingsFormat.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/PreFlexRWPostingsFormat.java 2011-11-01 21:10:36.068609905 -0400 @@ -0,0 +1,76 @@ +package org.apache.lucene.index.codecs.preflexrw; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xFields; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.util.LuceneTestCase; + +/** Codec, only for testing, that can write and read the + * pre-flex index format. + * + * @lucene.experimental + */ +public class PreFlexRWPostingsFormat extends Lucene3xPostingsFormat { + + public PreFlexRWPostingsFormat() { + // NOTE: we impersonate the PreFlex codec so that it can + // read the segments we write! + } + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { + return new PreFlexFieldsWriter(state); + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + + // Whenever IW opens readers, eg for merging, we have to + // keep terms order in UTF16: + + return new Lucene3xFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor) { + @Override + protected boolean sortTermsByUnicode() { + // We carefully peek into stack track above us: if + // we are part of a "merge", we must sort by UTF16: + boolean unicodeSortOrder = true; + + StackTraceElement[] trace = new Exception().getStackTrace(); + for (int i = 0; i < trace.length; i++) { + //System.out.println(trace[i].getClassName()); + if ("merge".equals(trace[i].getMethodName())) { + unicodeSortOrder = false; + if (LuceneTestCase.VERBOSE) { + System.out.println("NOTE: PreFlexRW codec: forcing legacy UTF16 term sort order"); + } + break; + } + } + + return unicodeSortOrder; + } + }; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java 2011-11-01 21:10:36.068609905 -0400 @@ -0,0 +1,286 @@ +package org.apache.lucene.index.codecs.preflexrw; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +import java.io.Closeable; +import java.io.IOException; + +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; +import org.apache.lucene.index.codecs.lucene3x.TermInfo; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.IOUtils; +import org.apache.lucene.util.UnicodeUtil; + + +/** This stores a monotonically increasing set of pairs in a + Directory. A TermInfos can be written once, in order. */ + +final class TermInfosWriter implements Closeable { + /** The file format version, a negative number. */ + public static final int FORMAT = -3; + + // Changed strings to true utf8 with length-in-bytes not + // length-in-chars + public static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = -4; + + // NOTE: always change this if you switch to a new format! + public static final int FORMAT_CURRENT = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; + + private FieldInfos fieldInfos; + private IndexOutput output; + private TermInfo lastTi = new TermInfo(); + private long size; + + // TODO: the default values for these two parameters should be settable from + // IndexWriter. However, once that's done, folks will start setting them to + // ridiculous values and complaining that things don't work well, as with + // mergeFactor. So, let's wait until a number of folks find that alternate + // values work better. Note that both of these values are stored in the + // segment, so that it's safe to change these w/o rebuilding all indexes. + + /** Expert: The fraction of terms in the "dictionary" which should be stored + * in RAM. Smaller values use more memory, but make searching slightly + * faster, while larger values use less memory and make searching slightly + * slower. Searching is typically not dominated by dictionary lookup, so + * tweaking this is rarely useful.*/ + int indexInterval = 128; + + /** Expert: The fraction of {@link TermDocs} entries stored in skip tables, + * used to accelerate {@link TermDocs#skipTo(int)}. Larger values result in + * smaller indexes, greater acceleration, but fewer accelerable cases, while + * smaller values result in bigger indexes, less acceleration and more + * accelerable cases. More detailed experiments would be useful here. */ + int skipInterval = 16; + + /** Expert: The maximum number of skip levels. Smaller values result in + * slightly smaller indexes, but slower skipping in big posting lists. + */ + int maxSkipLevels = 10; + + private long lastIndexPointer; + private boolean isIndex; + private final BytesRef lastTerm = new BytesRef(); + private int lastFieldNumber = -1; + + private TermInfosWriter other; + + TermInfosWriter(Directory directory, String segment, FieldInfos fis, + int interval) + throws IOException { + initialize(directory, segment, fis, interval, false); + boolean success = false; + try { + other = new TermInfosWriter(directory, segment, fis, interval, true); + other.other = this; + success = true; + } finally { + if (!success) { + try { + IOUtils.closeWhileHandlingException(output); + } catch (IOException e) { + // cannot happen since we suppress exceptions + throw new RuntimeException(e); + } + + try { + directory.deleteFile(IndexFileNames.segmentFileName(segment, "", + (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION + : Lucene3xPostingsFormat.TERMS_EXTENSION))); + } catch (IOException ignored) { + } + } + } + } + + private TermInfosWriter(Directory directory, String segment, FieldInfos fis, + int interval, boolean isIndex) throws IOException { + initialize(directory, segment, fis, interval, isIndex); + } + + private void initialize(Directory directory, String segment, FieldInfos fis, + int interval, boolean isi) throws IOException { + indexInterval = interval; + fieldInfos = fis; + isIndex = isi; + output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", + (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION + : Lucene3xPostingsFormat.TERMS_EXTENSION)), IOContext.DEFAULT); + boolean success = false; + try { + output.writeInt(FORMAT_CURRENT); // write format + output.writeLong(0); // leave space for size + output.writeInt(indexInterval); // write indexInterval + output.writeInt(skipInterval); // write skipInterval + output.writeInt(maxSkipLevels); // write maxSkipLevels + assert initUTF16Results(); + success = true; + } finally { + if (!success) { + try { + IOUtils.closeWhileHandlingException(output); + } catch (IOException e) { + // cannot happen since we suppress exceptions + throw new RuntimeException(e); + } + + try { + directory.deleteFile(IndexFileNames.segmentFileName(segment, "", + (isIndex ? Lucene3xPostingsFormat.TERMS_INDEX_EXTENSION + : Lucene3xPostingsFormat.TERMS_EXTENSION))); + } catch (IOException ignored) { + } + } + } + } + + // Currently used only by assert statements + CharsRef utf16Result1; + CharsRef utf16Result2; + private final BytesRef scratchBytes = new BytesRef(); + + // Currently used only by assert statements + private boolean initUTF16Results() { + utf16Result1 = new CharsRef(10); + utf16Result2 = new CharsRef(10); + return true; + } + + // Currently used only by assert statement + private int compareToLastTerm(int fieldNumber, BytesRef term) { + + if (lastFieldNumber != fieldNumber) { + final int cmp = fieldInfos.fieldName(lastFieldNumber).compareTo(fieldInfos.fieldName(fieldNumber)); + // If there is a field named "" (empty string) then we + // will get 0 on this comparison, yet, it's "OK". But + // it's not OK if two different field numbers map to + // the same name. + if (cmp != 0 || lastFieldNumber != -1) + return cmp; + } + + scratchBytes.copy(term); + assert lastTerm.offset == 0; + UnicodeUtil.UTF8toUTF16(lastTerm.bytes, 0, lastTerm.length, utf16Result1); + + assert scratchBytes.offset == 0; + UnicodeUtil.UTF8toUTF16(scratchBytes.bytes, 0, scratchBytes.length, utf16Result2); + + final int len; + if (utf16Result1.length < utf16Result2.length) + len = utf16Result1.length; + else + len = utf16Result2.length; + + for(int i=0;i, TermInfo> pair to the set. + Term must be lexicographically greater than all previous Terms added. + TermInfo pointers must be positive and greater than all previous.*/ + public void add(int fieldNumber, BytesRef term, TermInfo ti) + throws IOException { + + assert compareToLastTerm(fieldNumber, term) < 0 || + (isIndex && term.length == 0 && lastTerm.length == 0) : + "Terms are out of order: field=" + fieldInfos.fieldName(fieldNumber) + " (number " + fieldNumber + ")" + + " lastField=" + fieldInfos.fieldName(lastFieldNumber) + " (number " + lastFieldNumber + ")" + + " text=" + term.utf8ToString() + " lastText=" + lastTerm.utf8ToString(); + + assert ti.freqPointer >= lastTi.freqPointer: "freqPointer out of order (" + ti.freqPointer + " < " + lastTi.freqPointer + ")"; + assert ti.proxPointer >= lastTi.proxPointer: "proxPointer out of order (" + ti.proxPointer + " < " + lastTi.proxPointer + ")"; + + if (!isIndex && size % indexInterval == 0) + other.add(lastFieldNumber, lastTerm, lastTi); // add an index term + + writeTerm(fieldNumber, term); // write term + + output.writeVInt(ti.docFreq); // write doc freq + output.writeVLong(ti.freqPointer - lastTi.freqPointer); // write pointers + output.writeVLong(ti.proxPointer - lastTi.proxPointer); + + if (ti.docFreq >= skipInterval) { + output.writeVInt(ti.skipOffset); + } + + if (isIndex) { + output.writeVLong(other.output.getFilePointer() - lastIndexPointer); + lastIndexPointer = other.output.getFilePointer(); // write pointer + } + + lastFieldNumber = fieldNumber; + lastTi.set(ti); + size++; + } + + private void writeTerm(int fieldNumber, BytesRef term) + throws IOException { + + //System.out.println(" tiw.write field=" + fieldNumber + " term=" + term.utf8ToString()); + + // TODO: UTF16toUTF8 could tell us this prefix + // Compute prefix in common with last term: + int start = 0; + final int limit = term.length < lastTerm.length ? term.length : lastTerm.length; + while(start < limit) { + if (term.bytes[start+term.offset] != lastTerm.bytes[start+lastTerm.offset]) + break; + start++; + } + + final int length = term.length - start; + output.writeVInt(start); // write shared prefix length + output.writeVInt(length); // write delta length + output.writeBytes(term.bytes, start+term.offset, length); // write delta bytes + output.writeVInt(fieldNumber); // write field num + lastTerm.copy(term); + } + + /** Called to complete TermInfos creation. */ + public void close() throws IOException { + try { + output.seek(4); // write size after format + output.writeLong(size); + } finally { + try { + output.close(); + } finally { + if (!isIndex) { + other.close(); + } + } + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java 2011-11-03 08:29:28.476544412 -0400 @@ -0,0 +1,575 @@ +package org.apache.lucene.index.codecs.ramonly; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.SortedMap; +import java.util.TreeMap; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.index.DocsAndPositionsEnum; +import org.apache.lucene.index.DocsEnum; +import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IndexFileNames; +import org.apache.lucene.index.SegmentInfo; +import org.apache.lucene.index.SegmentReadState; +import org.apache.lucene.index.SegmentWriteState; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.index.codecs.FieldsConsumer; +import org.apache.lucene.index.codecs.FieldsProducer; +import org.apache.lucene.index.codecs.PostingsConsumer; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.TermStats; +import org.apache.lucene.index.codecs.TermsConsumer; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.IndexInput; +import org.apache.lucene.store.IndexOutput; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CodecUtil; +import org.apache.lucene.util.IOUtils; + +/** Stores all postings data in RAM, but writes a small + * token (header + single int) to identify which "slot" the + * index is using in RAM HashMap. + * + * NOTE: this codec sorts terms by reverse-unicode-order! */ + +public class RAMOnlyPostingsFormat extends PostingsFormat { + + // For fun, test that we can override how terms are + // sorted, and basic things still work -- this comparator + // sorts in reversed unicode code point order: + private static final Comparator reverseUnicodeComparator = new Comparator() { + public int compare(BytesRef t1, BytesRef t2) { + byte[] b1 = t1.bytes; + byte[] b2 = t2.bytes; + int b1Stop; + int b1Upto = t1.offset; + int b2Upto = t2.offset; + if (t1.length < t2.length) { + b1Stop = t1.offset + t1.length; + } else { + b1Stop = t1.offset + t2.length; + } + while(b1Upto < b1Stop) { + final int bb1 = b1[b1Upto++] & 0xff; + final int bb2 = b2[b2Upto++] & 0xff; + if (bb1 != bb2) { + //System.out.println("cmp 1=" + t1 + " 2=" + t2 + " return " + (bb2-bb1)); + return bb2 - bb1; + } + } + + // One is prefix of another, or they are equal + return t2.length-t1.length; + } + + @Override + public boolean equals(Object other) { + return this == other; + } + }; + + public RAMOnlyPostingsFormat() { + super("RAMOnly"); + } + + // Postings state: + static class RAMPostings extends FieldsProducer { + final Map fieldToTerms = new TreeMap(); + + @Override + public Terms terms(String field) { + return fieldToTerms.get(field); + } + + @Override + public FieldsEnum iterator() { + return new RAMFieldsEnum(this); + } + + @Override + public void close() { + } + } + + static class RAMField extends Terms { + final String field; + final SortedMap termToDocs = new TreeMap(); + long sumTotalTermFreq; + long sumDocFreq; + int docCount; + + RAMField(String field) { + this.field = field; + } + + @Override + public long getUniqueTermCount() { + return termToDocs.size(); + } + + @Override + public long getSumTotalTermFreq() { + return sumTotalTermFreq; + } + + @Override + public long getSumDocFreq() throws IOException { + return sumDocFreq; + } + + @Override + public int getDocCount() throws IOException { + return docCount; + } + + @Override + public TermsEnum iterator() { + return new RAMTermsEnum(RAMOnlyPostingsFormat.RAMField.this); + } + + @Override + public Comparator getComparator() { + return reverseUnicodeComparator; + } + } + + static class RAMTerm { + final String term; + long totalTermFreq; + final List docs = new ArrayList(); + public RAMTerm(String term) { + this.term = term; + } + } + + static class RAMDoc { + final int docID; + final int[] positions; + byte[][] payloads; + + public RAMDoc(int docID, int freq) { + this.docID = docID; + positions = new int[freq]; + } + } + + // Classes for writing to the postings state + private static class RAMFieldsConsumer extends FieldsConsumer { + + private final RAMPostings postings; + private final RAMTermsConsumer termsConsumer = new RAMTermsConsumer(); + + public RAMFieldsConsumer(RAMPostings postings) { + this.postings = postings; + } + + @Override + public TermsConsumer addField(FieldInfo field) { + RAMField ramField = new RAMField(field.name); + postings.fieldToTerms.put(field.name, ramField); + termsConsumer.reset(ramField); + return termsConsumer; + } + + @Override + public void close() { + // TODO: finalize stuff + } + } + + private static class RAMTermsConsumer extends TermsConsumer { + private RAMField field; + private final RAMPostingsWriterImpl postingsWriter = new RAMPostingsWriterImpl(); + RAMTerm current; + + void reset(RAMField field) { + this.field = field; + } + + @Override + public PostingsConsumer startTerm(BytesRef text) { + final String term = text.utf8ToString(); + current = new RAMTerm(term); + postingsWriter.reset(current); + return postingsWriter; + } + + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public void finishTerm(BytesRef text, TermStats stats) { + assert stats.docFreq > 0; + assert stats.docFreq == current.docs.size(); + current.totalTermFreq = stats.totalTermFreq; + field.termToDocs.put(current.term, current); + } + + @Override + public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) { + field.sumTotalTermFreq = sumTotalTermFreq; + field.sumDocFreq = sumDocFreq; + field.docCount = docCount; + } + } + + public static class RAMPostingsWriterImpl extends PostingsConsumer { + private RAMTerm term; + private RAMDoc current; + private int posUpto = 0; + + public void reset(RAMTerm term) { + this.term = term; + } + + @Override + public void startDoc(int docID, int freq) { + current = new RAMDoc(docID, freq); + term.docs.add(current); + posUpto = 0; + } + + @Override + public void addPosition(int position, BytesRef payload) { + current.positions[posUpto] = position; + if (payload != null && payload.length > 0) { + if (current.payloads == null) { + current.payloads = new byte[current.positions.length][]; + } + byte[] bytes = current.payloads[posUpto] = new byte[payload.length]; + System.arraycopy(payload.bytes, payload.offset, bytes, 0, payload.length); + } + posUpto++; + } + + @Override + public void finishDoc() { + assert posUpto == current.positions.length; + } + } + + // Classes for reading from the postings state + static class RAMFieldsEnum extends FieldsEnum { + private final RAMPostings postings; + private final Iterator it; + private String current; + + public RAMFieldsEnum(RAMPostings postings) { + this.postings = postings; + this.it = postings.fieldToTerms.keySet().iterator(); + } + + @Override + public String next() { + if (it.hasNext()) { + current = it.next(); + } else { + current = null; + } + return current; + } + + @Override + public TermsEnum terms() { + return new RAMTermsEnum(postings.fieldToTerms.get(current)); + } + } + + static class RAMTermsEnum extends TermsEnum { + Iterator it; + String current; + private final RAMField ramField; + + public RAMTermsEnum(RAMField field) { + this.ramField = field; + } + + @Override + public Comparator getComparator() { + return BytesRef.getUTF8SortedAsUnicodeComparator(); + } + + @Override + public BytesRef next() { + if (it == null) { + if (current == null) { + it = ramField.termToDocs.keySet().iterator(); + } else { + it = ramField.termToDocs.tailMap(current).keySet().iterator(); + } + } + if (it.hasNext()) { + current = it.next(); + return new BytesRef(current); + } else { + return null; + } + } + + @Override + public SeekStatus seekCeil(BytesRef term, boolean useCache) { + current = term.utf8ToString(); + it = null; + if (ramField.termToDocs.containsKey(current)) { + return SeekStatus.FOUND; + } else { + if (current.compareTo(ramField.termToDocs.lastKey()) > 0) { + return SeekStatus.END; + } else { + return SeekStatus.NOT_FOUND; + } + } + } + + @Override + public void seekExact(long ord) { + throw new UnsupportedOperationException(); + } + + @Override + public long ord() { + throw new UnsupportedOperationException(); + } + + @Override + public BytesRef term() { + // TODO: reuse BytesRef + return new BytesRef(current); + } + + @Override + public int docFreq() { + return ramField.termToDocs.get(current).docs.size(); + } + + @Override + public long totalTermFreq() { + return ramField.termToDocs.get(current).totalTermFreq; + } + + @Override + public DocsEnum docs(Bits liveDocs, DocsEnum reuse) { + return new RAMDocsEnum(ramField.termToDocs.get(current), liveDocs); + } + + @Override + public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) { + return new RAMDocsAndPositionsEnum(ramField.termToDocs.get(current), liveDocs); + } + } + + private static class RAMDocsEnum extends DocsEnum { + private final RAMTerm ramTerm; + private final Bits liveDocs; + private RAMDoc current; + int upto = -1; + int posUpto = 0; + + public RAMDocsEnum(RAMTerm ramTerm, Bits liveDocs) { + this.ramTerm = ramTerm; + this.liveDocs = liveDocs; + } + + @Override + public int advance(int targetDocID) { + do { + nextDoc(); + } while (upto < ramTerm.docs.size() && current.docID < targetDocID); + return NO_MORE_DOCS; + } + + // TODO: override bulk read, for better perf + @Override + public int nextDoc() { + while(true) { + upto++; + if (upto < ramTerm.docs.size()) { + current = ramTerm.docs.get(upto); + if (liveDocs == null || liveDocs.get(current.docID)) { + posUpto = 0; + return current.docID; + } + } else { + return NO_MORE_DOCS; + } + } + } + + @Override + public int freq() { + return current.positions.length; + } + + @Override + public int docID() { + return current.docID; + } + } + + private static class RAMDocsAndPositionsEnum extends DocsAndPositionsEnum { + private final RAMTerm ramTerm; + private final Bits liveDocs; + private RAMDoc current; + int upto = -1; + int posUpto = 0; + + public RAMDocsAndPositionsEnum(RAMTerm ramTerm, Bits liveDocs) { + this.ramTerm = ramTerm; + this.liveDocs = liveDocs; + } + + @Override + public int advance(int targetDocID) { + do { + nextDoc(); + } while (upto < ramTerm.docs.size() && current.docID < targetDocID); + return NO_MORE_DOCS; + } + + // TODO: override bulk read, for better perf + @Override + public int nextDoc() { + while(true) { + upto++; + if (upto < ramTerm.docs.size()) { + current = ramTerm.docs.get(upto); + if (liveDocs == null || liveDocs.get(current.docID)) { + posUpto = 0; + return current.docID; + } + } else { + return NO_MORE_DOCS; + } + } + } + + @Override + public int freq() { + return current.positions.length; + } + + @Override + public int docID() { + return current.docID; + } + + @Override + public int nextPosition() { + return current.positions[posUpto++]; + } + + @Override + public boolean hasPayload() { + return current.payloads != null && current.payloads[posUpto-1] != null; + } + + @Override + public BytesRef getPayload() { + return new BytesRef(current.payloads[posUpto-1]); + } + } + + // Holds all indexes created, keyed by the ID assigned in fieldsConsumer + private final Map state = new HashMap(); + + private final AtomicInteger nextID = new AtomicInteger(); + + private final String RAM_ONLY_NAME = "RAMOnly"; + private final static int VERSION_START = 0; + private final static int VERSION_LATEST = VERSION_START; + + private static final String ID_EXTENSION = "id"; + + @Override + public FieldsConsumer fieldsConsumer(SegmentWriteState writeState) throws IOException { + final int id = nextID.getAndIncrement(); + + // TODO -- ok to do this up front instead of + // on close....? should be ok? + // Write our ID: + final String idFileName = IndexFileNames.segmentFileName(writeState.segmentName, writeState.segmentSuffix, ID_EXTENSION); + IndexOutput out = writeState.directory.createOutput(idFileName, writeState.context); + boolean success = false; + try { + CodecUtil.writeHeader(out, RAM_ONLY_NAME, VERSION_LATEST); + out.writeVInt(id); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(out); + } else { + IOUtils.close(out); + } + } + + final RAMPostings postings = new RAMPostings(); + final RAMFieldsConsumer consumer = new RAMFieldsConsumer(postings); + + synchronized(state) { + state.put(id, postings); + } + return consumer; + } + + @Override + public FieldsProducer fieldsProducer(SegmentReadState readState) + throws IOException { + + // Load our ID: + final String idFileName = IndexFileNames.segmentFileName(readState.segmentInfo.name, readState.segmentSuffix, ID_EXTENSION); + IndexInput in = readState.dir.openInput(idFileName, readState.context); + boolean success = false; + final int id; + try { + CodecUtil.checkHeader(in, RAM_ONLY_NAME, VERSION_START, VERSION_LATEST); + id = in.readVInt(); + success = true; + } finally { + if (!success) { + IOUtils.closeWhileHandlingException(in); + } else { + IOUtils.close(in); + } + } + + synchronized(state) { + return state.get(id); + } + } + + @Override + public void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Set files) { + final String idFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, ID_EXTENSION); + files.add(idFileName); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/DocHelper.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/DocHelper.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/DocHelper.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/DocHelper.java 2011-11-01 21:10:36.092609905 -0400 @@ -0,0 +1,319 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.analysis.MockTokenizer; +import org.apache.lucene.document.BinaryField; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.FieldInfo.IndexOptions; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.TextField; +import org.apache.lucene.search.similarities.SimilarityProvider; +import org.apache.lucene.store.Directory; + +import static org.apache.lucene.util.LuceneTestCase.TEST_VERSION_CURRENT; + +class DocHelper { + + public static final FieldType customType; + public static final String FIELD_1_TEXT = "field one text"; + public static final String TEXT_FIELD_1_KEY = "textField1"; + public static Field textField1; + static { + customType = new FieldType(TextField.TYPE_STORED); + textField1 = new Field(TEXT_FIELD_1_KEY, FIELD_1_TEXT, customType); + } + + public static final FieldType customType2; + public static final String FIELD_2_TEXT = "field field field two text"; + //Fields will be lexicographically sorted. So, the order is: field, text, two + public static final int [] FIELD_2_FREQS = {3, 1, 1}; + public static final String TEXT_FIELD_2_KEY = "textField2"; + public static Field textField2; + static { + customType2 = new FieldType(TextField.TYPE_STORED); + customType2.setStoreTermVectors(true); + customType2.setStoreTermVectorPositions(true); + customType2.setStoreTermVectorOffsets(true); + textField2 = new Field(TEXT_FIELD_2_KEY, FIELD_2_TEXT, customType2); + } + + public static final FieldType customType3; + public static final String FIELD_3_TEXT = "aaaNoNorms aaaNoNorms bbbNoNorms"; + public static final String TEXT_FIELD_3_KEY = "textField3"; + public static Field textField3; + + static { + customType3 = new FieldType(TextField.TYPE_STORED); + customType3.setOmitNorms(true); + textField3 = new Field(TEXT_FIELD_3_KEY, FIELD_3_TEXT, customType3); + } + + public static final String KEYWORD_TEXT = "Keyword"; + public static final String KEYWORD_FIELD_KEY = "keyField"; + public static Field keyField; + static { + keyField = new Field(KEYWORD_FIELD_KEY, KEYWORD_TEXT, StringField.TYPE_STORED); + } + + public static final FieldType customType5; + public static final String NO_NORMS_TEXT = "omitNormsText"; + public static final String NO_NORMS_KEY = "omitNorms"; + public static Field noNormsField; + static { + customType5 = new FieldType(TextField.TYPE_STORED); + customType5.setOmitNorms(true); + customType5.setTokenized(false); + noNormsField = new Field(NO_NORMS_KEY, NO_NORMS_TEXT, customType5); + } + + public static final FieldType customType6; + public static final String NO_TF_TEXT = "analyzed with no tf and positions"; + public static final String NO_TF_KEY = "omitTermFreqAndPositions"; + public static Field noTFField; + static { + customType6 = new FieldType(TextField.TYPE_STORED); + customType6.setIndexOptions(IndexOptions.DOCS_ONLY); + noTFField = new Field(NO_TF_KEY, NO_TF_TEXT, customType6); + } + + public static final FieldType customType7; + public static final String UNINDEXED_FIELD_TEXT = "unindexed field text"; + public static final String UNINDEXED_FIELD_KEY = "unIndField"; + public static Field unIndField; + static { + customType7 = new FieldType(); + customType7.setStored(true); + unIndField = new Field(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT, customType7); + } + + + public static final String UNSTORED_1_FIELD_TEXT = "unstored field text"; + public static final String UNSTORED_FIELD_1_KEY = "unStoredField1"; + public static Field unStoredField1 = new Field(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT, TextField.TYPE_UNSTORED); + + public static final FieldType customType8; + public static final String UNSTORED_2_FIELD_TEXT = "unstored field text"; + public static final String UNSTORED_FIELD_2_KEY = "unStoredField2"; + public static Field unStoredField2; + static { + customType8 = new FieldType(TextField.TYPE_UNSTORED); + customType8.setStoreTermVectors(true); + unStoredField2 = new Field(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT, customType8); + } + + public static final String LAZY_FIELD_BINARY_KEY = "lazyFieldBinary"; + public static byte [] LAZY_FIELD_BINARY_BYTES; + public static Field lazyFieldBinary; + + public static final String LAZY_FIELD_KEY = "lazyField"; + public static final String LAZY_FIELD_TEXT = "These are some field bytes"; + public static Field lazyField = new Field(LAZY_FIELD_KEY, LAZY_FIELD_TEXT, customType); + + public static final String LARGE_LAZY_FIELD_KEY = "largeLazyField"; + public static String LARGE_LAZY_FIELD_TEXT; + public static Field largeLazyField; + + //From Issue 509 + public static final String FIELD_UTF1_TEXT = "field one \u4e00text"; + public static final String TEXT_FIELD_UTF1_KEY = "textField1Utf8"; + public static Field textUtfField1 = new Field(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT, customType); + + public static final String FIELD_UTF2_TEXT = "field field field \u4e00two text"; + //Fields will be lexicographically sorted. So, the order is: field, text, two + public static final int [] FIELD_UTF2_FREQS = {3, 1, 1}; + public static final String TEXT_FIELD_UTF2_KEY = "textField2Utf8"; + public static Field textUtfField2 = new Field(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT, customType2); + + + + + public static Map nameValues = null; + + // ordered list of all the fields... + // could use LinkedHashMap for this purpose if Java1.4 is OK + public static Field[] fields = new Field[] { + textField1, + textField2, + textField3, + keyField, + noNormsField, + noTFField, + unIndField, + unStoredField1, + unStoredField2, + textUtfField1, + textUtfField2, + lazyField, + lazyFieldBinary,//placeholder for binary field, since this is null. It must be second to last. + largeLazyField//placeholder for large field, since this is null. It must always be last + }; + + public static Map all =new HashMap(); + public static Map indexed =new HashMap(); + public static Map stored =new HashMap(); + public static Map unstored=new HashMap(); + public static Map unindexed=new HashMap(); + public static Map termvector=new HashMap(); + public static Map notermvector=new HashMap(); + public static Map lazy= new HashMap(); + public static Map noNorms=new HashMap(); + public static Map noTf=new HashMap(); + + static { + //Initialize the large Lazy Field + StringBuilder buffer = new StringBuilder(); + for (int i = 0; i < 10000; i++) + { + buffer.append("Lazily loading lengths of language in lieu of laughing "); + } + + try { + LAZY_FIELD_BINARY_BYTES = "These are some binary field bytes".getBytes("UTF8"); + } catch (UnsupportedEncodingException e) { + } + lazyFieldBinary = new BinaryField(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES); + fields[fields.length - 2] = lazyFieldBinary; + LARGE_LAZY_FIELD_TEXT = buffer.toString(); + largeLazyField = new Field(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT, customType); + fields[fields.length - 1] = largeLazyField; + for (int i=0; i map, IndexableField field) { + map.put(field.name(), field); + } + + + static + { + nameValues = new HashMap(); + nameValues.put(TEXT_FIELD_1_KEY, FIELD_1_TEXT); + nameValues.put(TEXT_FIELD_2_KEY, FIELD_2_TEXT); + nameValues.put(TEXT_FIELD_3_KEY, FIELD_3_TEXT); + nameValues.put(KEYWORD_FIELD_KEY, KEYWORD_TEXT); + nameValues.put(NO_NORMS_KEY, NO_NORMS_TEXT); + nameValues.put(NO_TF_KEY, NO_TF_TEXT); + nameValues.put(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT); + nameValues.put(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT); + nameValues.put(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT); + nameValues.put(LAZY_FIELD_KEY, LAZY_FIELD_TEXT); + nameValues.put(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES); + nameValues.put(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT); + nameValues.put(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT); + nameValues.put(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT); + } + + /** + * Adds the fields above to a document + * @param doc The document to write + */ + public static void setupDoc(Document doc) { + for (int i=0; i= remainder ? remainder : bytesInBuffer; + System.arraycopy(buffer, bufferOffset, dest, destOffset, bytesToCopy); + destOffset += bytesToCopy; + start += bytesToCopy; + remainder -= bytesToCopy; + } + pointer += len; + } + + @Override + public void close() { + // ignore + } + + @Override + protected void seekInternal(long pos) { + pointer = (int) pos; + } + + @Override + public long length() { + return length; + } + +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/MockRandomMergePolicy.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/MockRandomMergePolicy.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/MockRandomMergePolicy.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/MockRandomMergePolicy.java 2011-11-01 21:10:36.092609905 -0400 @@ -0,0 +1,111 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.Map; + +import org.apache.lucene.util._TestUtil; + +public class MockRandomMergePolicy extends MergePolicy { + private final Random random; + + public MockRandomMergePolicy(Random random) { + // fork a private random, since we are called + // unpredictably from threads: + this.random = new Random(random.nextLong()); + } + + @Override + public MergeSpecification findMerges(SegmentInfos segmentInfos) { + MergeSpecification mergeSpec = null; + //System.out.println("MRMP: findMerges sis=" + segmentInfos); + + if (segmentInfos.size() > 1 && random.nextInt(5) == 3) { + + List segments = new ArrayList(segmentInfos.asList()); + Collections.shuffle(segments, random); + + // TODO: sometimes make more than 1 merge? + mergeSpec = new MergeSpecification(); + final int segsToMerge = _TestUtil.nextInt(random, 1, segmentInfos.size()); + mergeSpec.add(new OneMerge(segments.subList(0, segsToMerge))); + } + + return mergeSpec; + } + + @Override + public MergeSpecification findMergesForOptimize( + SegmentInfos segmentInfos, int maxSegmentCount, Map segmentsToOptimize) + throws CorruptIndexException, IOException { + + final List eligibleSegments = new ArrayList(); + for(SegmentInfo info : segmentInfos) { + if (segmentsToOptimize.containsKey(info)) { + eligibleSegments.add(info); + } + } + + //System.out.println("MRMP: findMergesForOptimize sis=" + segmentInfos + " eligible=" + eligibleSegments); + MergeSpecification mergeSpec = null; + if (eligibleSegments.size() > 1 || (eligibleSegments.size() == 1 && eligibleSegments.get(0).hasDeletions())) { + mergeSpec = new MergeSpecification(); + // Already shuffled having come out of a set but + // shuffle again for good measure: + Collections.shuffle(eligibleSegments, random); + int upto = 0; + while(upto < eligibleSegments.size()) { + int max = Math.min(10, eligibleSegments.size()-upto); + int inc = max <= 2 ? max : _TestUtil.nextInt(random, 2, max); + mergeSpec.add(new OneMerge(eligibleSegments.subList(upto, upto+inc))); + upto += inc; + } + } + + if (mergeSpec != null) { + for(OneMerge merge : mergeSpec.merges) { + for(SegmentInfo info : merge.segments) { + assert segmentsToOptimize.containsKey(info); + } + } + } + return mergeSpec; + } + + @Override + public MergeSpecification findMergesToExpungeDeletes( + SegmentInfos segmentInfos) + throws CorruptIndexException, IOException { + return findMerges(segmentInfos); + } + + @Override + public void close() { + } + + @Override + public boolean useCompoundFile(SegmentInfos infos, SegmentInfo mergedInfo) throws IOException { + // 80% of the time we create CFS: + return random.nextInt(5) != 1; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/RandomCodec.java 2011-11-02 21:32:53.528564707 -0400 @@ -0,0 +1,103 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Random; + +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.lucene40ords.Lucene40WithOrds; +import org.apache.lucene.index.codecs.memory.MemoryPostingsFormat; +import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mockrandom.MockRandomPostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; +import org.apache.lucene.index.codecs.nestedpulsing.NestedPulsingPostingsFormat; +import org.apache.lucene.index.codecs.pulsing.Pulsing40PostingsFormat; +import org.apache.lucene.index.codecs.simpletext.SimpleTextPostingsFormat; +import org.apache.lucene.util._TestUtil; + +/** + * Codec that assigns per-field random postings formats. + *

+ * The same field/format assignment will happen regardless of order, + * a hash is computed up front that determines the mapping. + * This means fields can be put into things like HashSets and added to + * documents in different orders and the test will still be deterministic + * and reproducable. + */ +public class RandomCodec extends Lucene40Codec { + /** shuffled list of postingsformats to use for new mappings */ + private List formats = new ArrayList(); + /** memorized field->postingsformat mappings */ + private Map previousMappings = new HashMap(); + private final int perFieldSeed; + + @Override + public PostingsFormat getPostingsFormatForField(String name) { + PostingsFormat codec = previousMappings.get(name); + if (codec == null) { + codec = formats.get(Math.abs(perFieldSeed ^ name.hashCode()) % formats.size()); + if (codec instanceof SimpleTextPostingsFormat && perFieldSeed % 5 != 0) { + // make simpletext rarer, choose again + codec = formats.get(Math.abs(perFieldSeed ^ name.toUpperCase(Locale.ENGLISH).hashCode()) % formats.size()); + } + previousMappings.put(name, codec); + // Safety: + assert previousMappings.size() < 10000; + } + return codec; + } + + public RandomCodec(Random random, boolean useNoMemoryExpensiveCodec) { + this.perFieldSeed = random.nextInt(); + // TODO: make it possible to specify min/max iterms per + // block via CL: + int minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); + int maxItemsPerBlock = 2*(Math.max(2, minItemsPerBlock-1)) + random.nextInt(100); + formats.add(new Lucene40PostingsFormat(minItemsPerBlock, maxItemsPerBlock)); + // TODO: make it possible to specify min/max iterms per + // block via CL: + minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); + maxItemsPerBlock = 2*(Math.max(1, minItemsPerBlock-1)) + random.nextInt(100); + formats.add(new Pulsing40PostingsFormat(1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock)); + formats.add(new MockSepPostingsFormat()); + formats.add(new MockFixedIntBlockPostingsFormat(_TestUtil.nextInt(random, 1, 2000))); + formats.add(new MockVariableIntBlockPostingsFormat( _TestUtil.nextInt(random, 1, 127))); + formats.add(new MockRandomPostingsFormat(random)); + formats.add(new NestedPulsingPostingsFormat()); + formats.add(new Lucene40WithOrds()); + if (!useNoMemoryExpensiveCodec) { + formats.add(new SimpleTextPostingsFormat()); + formats.add(new MemoryPostingsFormat()); + } + Collections.shuffle(formats, random); + } + + @Override + public String toString() { + return super.toString() + ": " + previousMappings.toString(); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/RandomIndexWriter.java 2011-11-01 21:10:36.092609905 -0400 @@ -0,0 +1,415 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Closeable; +import java.io.IOException; +import java.util.Iterator; +import java.util.Random; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.IndexDocValuesField; +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexWriter; // javadoc +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.values.ValueType; +import org.apache.lucene.search.Query; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.Version; +import org.apache.lucene.util._TestUtil; + +/** Silly class that randomizes the indexing experience. EG + * it may swap in a different merge policy/scheduler; may + * commit periodically; may or may not optimize in the end, + * may flush by doc count instead of RAM, etc. + */ + +public class RandomIndexWriter implements Closeable { + + public IndexWriter w; + private final Random r; + int docCount; + int flushAt; + private double flushAtFactor = 1.0; + private boolean getReaderCalled; + private final int fixedBytesLength; + private final long docValuesFieldPrefix; + private volatile boolean doDocValues; + private final Codec codec; // sugar + + // Randomly calls Thread.yield so we mixup thread scheduling + private static final class MockIndexWriter extends IndexWriter { + + private final Random r; + + public MockIndexWriter(Random r, Directory dir, IndexWriterConfig conf) throws IOException { + super(dir, conf); + // must make a private random since our methods are + // called from different threads; else test failures may + // not be reproducible from the original seed + this.r = new Random(r.nextInt()); + } + + @Override + boolean testPoint(String name) { + if (r.nextInt(4) == 2) + Thread.yield(); + return true; + } + } + + /** create a RandomIndexWriter with a random config: Uses TEST_VERSION_CURRENT and MockAnalyzer */ + public RandomIndexWriter(Random r, Directory dir) throws IOException { + this(r, dir, LuceneTestCase.newIndexWriterConfig(r, LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r))); + } + + /** create a RandomIndexWriter with a random config: Uses TEST_VERSION_CURRENT */ + public RandomIndexWriter(Random r, Directory dir, Analyzer a) throws IOException { + this(r, dir, LuceneTestCase.newIndexWriterConfig(r, LuceneTestCase.TEST_VERSION_CURRENT, a)); + } + + /** create a RandomIndexWriter with a random config */ + public RandomIndexWriter(Random r, Directory dir, Version v, Analyzer a) throws IOException { + this(r, dir, LuceneTestCase.newIndexWriterConfig(r, v, a)); + } + + /** create a RandomIndexWriter with the provided config */ + public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c) throws IOException { + this.r = r; + w = new MockIndexWriter(r, dir, c); + flushAt = _TestUtil.nextInt(r, 10, 1000); + codec = w.getConfig().getCodec(); + if (LuceneTestCase.VERBOSE) { + System.out.println("RIW config=" + w.getConfig()); + System.out.println("codec default=" + codec.getName()); + w.setInfoStream(System.out); + } + /* TODO: find some what to make that random... + * This must be fixed across all fixed bytes + * fields in one index. so if you open another writer + * this might change if I use r.nextInt(x) + * maybe we can peek at the existing files here? + */ + fixedBytesLength = 37; + docValuesFieldPrefix = r.nextLong(); + switchDoDocValues(); + } + + private void switchDoDocValues() { + // randomly enable / disable docValues + doDocValues = LuceneTestCase.rarely(r); + } + + /** + * Adds a Document. + * @see IndexWriter#addDocument(Iterable) + */ + public void addDocument(final Iterable doc) throws IOException { + if (doDocValues && doc instanceof Document) { + randomPerDocFieldValues(r, (Document) doc); + } + if (r.nextInt(5) == 3) { + // TODO: maybe, we should simply buffer up added docs + // (but we need to clone them), and only when + // getReader, commit, etc. are called, we do an + // addDocuments? Would be better testing. + w.addDocuments(new Iterable>() { + + @Override + public Iterator> iterator() { + return new Iterator>() { + boolean done; + + @Override + public boolean hasNext() { + return !done; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public Iterable next() { + if (done) { + throw new IllegalStateException(); + } + done = true; + return doc; + } + }; + } + }); + } else { + w.addDocument(doc); + } + + maybeCommit(); + } + + private void randomPerDocFieldValues(Random random, Document doc) { + + ValueType[] values = ValueType.values(); + ValueType type = values[random.nextInt(values.length)]; + String name = "random_" + type.name() + "" + docValuesFieldPrefix; + if ("Lucene3x".equals(codec.getName()) || doc.getField(name) != null) + return; + IndexDocValuesField docValuesField = new IndexDocValuesField(name); + switch (type) { + case BYTES_FIXED_DEREF: + case BYTES_FIXED_STRAIGHT: + case BYTES_FIXED_SORTED: + //make sure we use a valid unicode string with a fixed size byte length + final String randomUnicodeString = _TestUtil.randomFixedByteLengthUnicodeString(random, fixedBytesLength); + BytesRef fixedRef = new BytesRef(randomUnicodeString); + if (fixedRef.length > fixedBytesLength) { + fixedRef = new BytesRef(fixedRef.bytes, 0, fixedBytesLength); + } else { + fixedRef.grow(fixedBytesLength); + fixedRef.length = fixedBytesLength; + } + docValuesField.setBytes(fixedRef, type); + break; + case BYTES_VAR_DEREF: + case BYTES_VAR_STRAIGHT: + case BYTES_VAR_SORTED: + BytesRef ref = new BytesRef(_TestUtil.randomUnicodeString(random, 200)); + docValuesField.setBytes(ref, type); + break; + case FLOAT_32: + docValuesField.setFloat(random.nextFloat()); + break; + case FLOAT_64: + docValuesField.setFloat(random.nextDouble()); + break; + case VAR_INTS: + docValuesField.setInt(random.nextLong()); + break; + case FIXED_INTS_16: + docValuesField.setInt(random.nextInt(Short.MAX_VALUE)); + break; + case FIXED_INTS_32: + docValuesField.setInt(random.nextInt()); + break; + case FIXED_INTS_64: + docValuesField.setInt(random.nextLong()); + break; + case FIXED_INTS_8: + docValuesField.setInt(random.nextInt(128)); + break; + default: + throw new IllegalArgumentException("no such type: " + type); + } + + doc.add(docValuesField); + } + + private void maybeCommit() throws IOException { + if (docCount++ == flushAt) { + if (LuceneTestCase.VERBOSE) { + System.out.println("RIW.add/updateDocument: now doing a commit at docCount=" + docCount); + } + w.commit(); + flushAt += _TestUtil.nextInt(r, (int) (flushAtFactor * 10), (int) (flushAtFactor * 1000)); + if (flushAtFactor < 2e6) { + // gradually but exponentially increase time b/w flushes + flushAtFactor *= 1.05; + } + switchDoDocValues(); + } + } + + public void addDocuments(Iterable> docs) throws IOException { + w.addDocuments(docs); + maybeCommit(); + } + + public void updateDocuments(Term delTerm, Iterable> docs) throws IOException { + w.updateDocuments(delTerm, docs); + maybeCommit(); + } + + /** + * Updates a document. + * @see IndexWriter#updateDocument(Term, Iterable) + */ + public void updateDocument(Term t, final Iterable doc) throws IOException { + if (doDocValues) { + randomPerDocFieldValues(r, (Document) doc); + } + if (r.nextInt(5) == 3) { + w.updateDocuments(t, new Iterable>() { + + @Override + public Iterator> iterator() { + return new Iterator>() { + boolean done; + + @Override + public boolean hasNext() { + return !done; + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + + @Override + public Iterable next() { + if (done) { + throw new IllegalStateException(); + } + done = true; + return doc; + } + }; + } + }); + } else { + w.updateDocument(t, doc); + } + maybeCommit(); + } + + public void addIndexes(Directory... dirs) throws CorruptIndexException, IOException { + w.addIndexes(dirs); + } + + public void deleteDocuments(Term term) throws CorruptIndexException, IOException { + w.deleteDocuments(term); + } + + public void deleteDocuments(Query q) throws CorruptIndexException, IOException { + w.deleteDocuments(q); + } + + public void commit() throws CorruptIndexException, IOException { + w.commit(); + switchDoDocValues(); + } + + public int numDocs() throws IOException { + return w.numDocs(); + } + + public int maxDoc() { + return w.maxDoc(); + } + + public void deleteAll() throws IOException { + w.deleteAll(); + } + + public IndexReader getReader() throws IOException { + return getReader(true); + } + + private boolean doRandomOptimize = true; + private boolean doRandomOptimizeAssert = true; + + public void expungeDeletes(boolean doWait) throws IOException { + w.expungeDeletes(doWait); + } + + public void expungeDeletes() throws IOException { + w.expungeDeletes(); + } + + public void setDoRandomOptimize(boolean v) { + doRandomOptimize = v; + } + + public void setDoRandomOptimizeAssert(boolean v) { + doRandomOptimizeAssert = v; + } + + private void doRandomOptimize() throws IOException { + if (doRandomOptimize) { + final int segCount = w.getSegmentCount(); + if (r.nextBoolean() || segCount == 0) { + // full optimize + w.optimize(); + } else { + // partial optimize + final int limit = _TestUtil.nextInt(r, 1, segCount); + w.optimize(limit); + assert !doRandomOptimizeAssert || w.getSegmentCount() <= limit: "limit=" + limit + " actual=" + w.getSegmentCount(); + } + } + switchDoDocValues(); + } + + public IndexReader getReader(boolean applyDeletions) throws IOException { + getReaderCalled = true; + if (r.nextInt(4) == 2) { + doRandomOptimize(); + } + // If we are writing with PreFlexRW, force a full + // IndexReader.open so terms are sorted in codepoint + // order during searching: + if (!applyDeletions || !codec.getName().equals("Lucene3x") && r.nextBoolean()) { + if (LuceneTestCase.VERBOSE) { + System.out.println("RIW.getReader: use NRT reader"); + } + if (r.nextInt(5) == 1) { + w.commit(); + } + return w.getReader(applyDeletions); + } else { + if (LuceneTestCase.VERBOSE) { + System.out.println("RIW.getReader: open new reader"); + } + w.commit(); + switchDoDocValues(); + if (r.nextBoolean()) { + return IndexReader.open(w.getDirectory(), new KeepOnlyLastCommitDeletionPolicy(), r.nextBoolean(), _TestUtil.nextInt(r, 1, 10)); + } else { + return w.getReader(applyDeletions); + } + } + } + + /** + * Close this writer. + * @see IndexWriter#close() + */ + public void close() throws IOException { + // if someone isn't using getReader() API, we want to be sure to + // maybeOptimize since presumably they might open a reader on the dir. + if (getReaderCalled == false && r.nextInt(8) == 2) { + doRandomOptimize(); + } + w.close(); + } + + /** + * Forces an optimize. + *

+ * NOTE: this should be avoided in tests unless absolutely necessary, + * as it will result in less test coverage. + * @see IndexWriter#optimize() + */ + public void optimize() throws IOException { + w.optimize(); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java 2011-11-01 21:10:36.092609905 -0400 @@ -0,0 +1,650 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.util.Bits; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.LineFileDocs; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.NamedThreadFactory; +import org.apache.lucene.util._TestUtil; + +// TODO +// - mix in optimize, addIndexes +// - randomly mix in non-congruent docs + +/** Utility class that spawns multiple indexing and + * searching threads. */ +public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCase { + + protected final AtomicBoolean failed = new AtomicBoolean(); + protected final AtomicInteger addCount = new AtomicInteger(); + protected final AtomicInteger delCount = new AtomicInteger(); + protected final AtomicInteger packCount = new AtomicInteger(); + + protected Directory dir; + protected IndexWriter writer; + + private static class SubDocs { + public final String packID; + public final List subIDs; + public boolean deleted; + + public SubDocs(String packID, List subIDs) { + this.packID = packID; + this.subIDs = subIDs; + } + } + + // Called per-search + protected abstract IndexSearcher getCurrentSearcher() throws Exception; + + protected abstract IndexSearcher getFinalSearcher() throws Exception; + + protected void releaseSearcher(IndexSearcher s) throws Exception { + } + + // Called once to run searching + protected abstract void doSearching(ExecutorService es, long stopTime) throws Exception; + + protected Directory getDirectory(Directory in) { + return in; + } + + protected void updateDocuments(Term id, List> docs) throws Exception { + writer.updateDocuments(id, docs); + } + + protected void addDocuments(Term id, List> docs) throws Exception { + writer.addDocuments(docs); + } + + protected void addDocument(Term id, Iterable doc) throws Exception { + writer.addDocument(doc); + } + + protected void updateDocument(Term term, Iterable doc) throws Exception { + writer.updateDocument(term, doc); + } + + protected void deleteDocuments(Term term) throws Exception { + writer.deleteDocuments(term); + } + + protected void doAfterIndexingThreadDone() { + } + + private Thread[] launchIndexingThreads(final LineFileDocs docs, + int numThreads, + final long stopTime, + final Set delIDs, + final Set delPackIDs, + final List allSubDocs) + throws Exception { + final Thread[] threads = new Thread[numThreads]; + for(int thread=0;thread toDeleteIDs = new ArrayList(); + final List toDeleteSubDocs = new ArrayList(); + while(System.currentTimeMillis() < stopTime && !failed.get()) { + try { + + // Occasional longish pause if running + // nightly + if (LuceneTestCase.TEST_NIGHTLY && random.nextInt(6) == 3) { + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": now long sleep"); + } + Thread.sleep(_TestUtil.nextInt(random, 50, 500)); + } + + // Rate limit ingest rate: + if (random.nextInt(7) == 5) { + Thread.sleep(_TestUtil.nextInt(random, 1, 10)); + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": done sleep"); + } + } + + Document doc = docs.nextDoc(); + if (doc == null) { + break; + } + + // Maybe add randomly named field + final String addedField; + if (random.nextBoolean()) { + addedField = "extra" + random.nextInt(40); + doc.add(newField(addedField, "a random field", TextField.TYPE_STORED)); + } else { + addedField = null; + } + + if (random.nextBoolean()) { + + if (random.nextBoolean()) { + // Add/update doc block: + final String packID; + final SubDocs delSubDocs; + if (toDeleteSubDocs.size() > 0 && random.nextBoolean()) { + delSubDocs = toDeleteSubDocs.get(random.nextInt(toDeleteSubDocs.size())); + assert !delSubDocs.deleted; + toDeleteSubDocs.remove(delSubDocs); + // Update doc block, replacing prior packID + packID = delSubDocs.packID; + } else { + delSubDocs = null; + // Add doc block, using new packID + packID = packCount.getAndIncrement() + ""; + } + + final Field packIDField = newField("packID", packID, StringField.TYPE_STORED); + final List docIDs = new ArrayList(); + final SubDocs subDocs = new SubDocs(packID, docIDs); + final List docsList = new ArrayList(); + + allSubDocs.add(subDocs); + doc.add(packIDField); + docsList.add(_TestUtil.cloneDocument(doc)); + docIDs.add(doc.get("docid")); + + final int maxDocCount = _TestUtil.nextInt(random, 1, 10); + while(docsList.size() < maxDocCount) { + doc = docs.nextDoc(); + if (doc == null) { + break; + } + docsList.add(_TestUtil.cloneDocument(doc)); + docIDs.add(doc.get("docid")); + } + addCount.addAndGet(docsList.size()); + + final Term packIDTerm = new Term("packID", packID); + + if (delSubDocs != null) { + delSubDocs.deleted = true; + delIDs.addAll(delSubDocs.subIDs); + delCount.addAndGet(delSubDocs.subIDs.size()); + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": update pack packID=" + delSubDocs.packID + " count=" + docsList.size() + " docs=" + docIDs); + } + updateDocuments(packIDTerm, docsList); + } else { + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": add pack packID=" + packID + " count=" + docsList.size() + " docs=" + docIDs); + } + addDocuments(packIDTerm, docsList); + } + doc.removeField("packID"); + + if (random.nextInt(5) == 2) { + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": buffer del id:" + packID); + } + toDeleteSubDocs.add(subDocs); + } + + } else { + // Add single doc + final String docid = doc.get("docid"); + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": add doc docid:" + docid); + } + addDocument(new Term("docid", docid), doc); + addCount.getAndIncrement(); + + if (random.nextInt(5) == 3) { + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid")); + } + toDeleteIDs.add(docid); + } + } + } else { + + // Update single doc, but we never re-use + // and ID so the delete will never + // actually happen: + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": update doc id:" + doc.get("docid")); + } + final String docid = doc.get("docid"); + updateDocument(new Term("docid", docid), doc); + addCount.getAndIncrement(); + + if (random.nextInt(5) == 3) { + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid")); + } + toDeleteIDs.add(docid); + } + } + + if (random.nextInt(30) == 17) { + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": apply " + toDeleteIDs.size() + " deletes"); + } + for(String id : toDeleteIDs) { + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": del term=id:" + id); + } + deleteDocuments(new Term("docid", id)); + } + final int count = delCount.addAndGet(toDeleteIDs.size()); + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": tot " + count + " deletes"); + } + delIDs.addAll(toDeleteIDs); + toDeleteIDs.clear(); + + for(SubDocs subDocs : toDeleteSubDocs) { + assert !subDocs.deleted; + delPackIDs.add(subDocs.packID); + deleteDocuments(new Term("packID", subDocs.packID)); + subDocs.deleted = true; + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": del subs: " + subDocs.subIDs + " packID=" + subDocs.packID); + } + delIDs.addAll(subDocs.subIDs); + delCount.addAndGet(subDocs.subIDs.size()); + } + toDeleteSubDocs.clear(); + } + if (addedField != null) { + doc.removeField(addedField); + } + } catch (Throwable t) { + System.out.println(Thread.currentThread().getName() + ": hit exc"); + t.printStackTrace(); + failed.set(true); + throw new RuntimeException(t); + } + } + if (VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": indexing done"); + } + + doAfterIndexingThreadDone(); + } + }; + threads[thread].setDaemon(true); + threads[thread].start(); + } + + return threads; + } + + protected void runSearchThreads(final long stopTimeMS) throws Exception { + final int numThreads = _TestUtil.nextInt(random, 1, 5); + final Thread[] searchThreads = new Thread[numThreads]; + final AtomicInteger totHits = new AtomicInteger(); + + // silly starting guess: + final AtomicInteger totTermCount = new AtomicInteger(100); + + // TODO: we should enrich this to do more interesting searches + for(int thread=0;thread 0) { + smokeTestSearcher(s); + Fields fields = MultiFields.getFields(s.getIndexReader()); + if (fields == null) { + continue; + } + Terms terms = fields.terms("body"); + if (terms == null) { + continue; + } + TermsEnum termsEnum = terms.iterator(); + int seenTermCount = 0; + int shift; + int trigger; + if (totTermCount.get() < 10) { + shift = 0; + trigger = 1; + } else { + trigger = totTermCount.get()/10; + shift = random.nextInt(trigger); + } + BytesRef term = termsEnum.next(); + if (term == null) { + if (seenTermCount == 0) { + break; + } + totTermCount.set(seenTermCount); + seenTermCount = 0; + if (totTermCount.get() < 10) { + shift = 0; + trigger = 1; + } else { + trigger = totTermCount.get()/10; + //System.out.println("trigger " + trigger); + shift = random.nextInt(trigger); + } + termsEnum.seekCeil(new BytesRef("")); + continue; + } + seenTermCount++; + // search 10 terms + if (trigger == 0) { + trigger = 1; + } + if ((seenTermCount + shift) % trigger == 0) { + //if (VERBOSE) { + //System.out.println(Thread.currentThread().getName() + " now search body:" + term.utf8ToString()); + //} + totHits.addAndGet(runQuery(s, new TermQuery(new Term("body", term)))); + } + //if (VERBOSE) { + //System.out.println(Thread.currentThread().getName() + ": search done"); + //} + } + } finally { + releaseSearcher(s); + } + } catch (Throwable t) { + System.out.println(Thread.currentThread().getName() + ": hit exc"); + failed.set(true); + t.printStackTrace(System.out); + throw new RuntimeException(t); + } + } + } + }; + searchThreads[thread].setDaemon(true); + searchThreads[thread].start(); + } + + for(int thread=0;thread delIDs = Collections.synchronizedSet(new HashSet()); + final Set delPackIDs = Collections.synchronizedSet(new HashSet()); + final List allSubDocs = Collections.synchronizedList(new ArrayList()); + + final long stopTime = System.currentTimeMillis() + RUN_TIME_SEC*1000; + + final Thread[] indexThreads = launchIndexingThreads(docs, NUM_INDEX_THREADS, stopTime, delIDs, delPackIDs, allSubDocs); + + if (VERBOSE) { + System.out.println("TEST: DONE start indexing threads [" + (System.currentTimeMillis()-t0) + " ms]"); + } + + // Let index build up a bit + Thread.sleep(100); + + doSearching(es, stopTime); + + if (VERBOSE) { + System.out.println("TEST: all searching done [" + (System.currentTimeMillis()-t0) + " ms]"); + } + + for(int thread=0;thread saveMissCount); + } + + return docIdSet; + } + + @Override + public String toString() { + return "CachingWrapperFilterHelper("+filter+")"; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof CachingWrapperFilterHelper)) return false; + return this.filter.equals(o); + } + + @Override + public int hashCode() { + return this.filter.hashCode() ^ 0x5525aacb; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java 2011-11-01 21:10:36.008609908 -0400 @@ -0,0 +1,525 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.util.Locale; +import java.util.Set; +import java.util.TreeSet; +import java.util.Random; + +import junit.framework.Assert; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexReader.AtomicReaderContext; +import org.apache.lucene.store.Directory; + +public class CheckHits { + + /** + * Some explains methods calculate their values though a slightly + * different order of operations from the actual scoring method ... + * this allows for a small amount of relative variation + */ + public static float EXPLAIN_SCORE_TOLERANCE_DELTA = 0.001f; + + /** + * In general we use a relative epsilon, but some tests do crazy things + * like boost documents with 0, creating tiny tiny scores where the + * relative difference is large but the absolute difference is tiny. + * we ensure the the epsilon is always at least this big. + */ + public static float EXPLAIN_SCORE_TOLERANCE_MINIMUM = 1e-6f; + + /** + * Tests that all documents up to maxDoc which are *not* in the + * expected result set, have an explanation which indicates that + * the document does not match + */ + public static void checkNoMatchExplanations(Query q, String defaultFieldName, + IndexSearcher searcher, int[] results) + throws IOException { + + String d = q.toString(defaultFieldName); + Set ignore = new TreeSet(); + for (int i = 0; i < results.length; i++) { + ignore.add(Integer.valueOf(results[i])); + } + + int maxDoc = searcher.maxDoc(); + for (int doc = 0; doc < maxDoc; doc++) { + if (ignore.contains(Integer.valueOf(doc))) continue; + + Explanation exp = searcher.explain(q, doc); + Assert.assertNotNull("Explanation of [["+d+"]] for #"+doc+" is null", + exp); + Assert.assertFalse("Explanation of [["+d+"]] for #"+doc+ + " doesn't indicate non-match: " + exp.toString(), + exp.isMatch()); + } + + } + + /** + * Tests that a query matches the an expected set of documents using a + * HitCollector. + * + *

+ * Note that when using the HitCollector API, documents will be collected + * if they "match" regardless of what their score is. + *

+ * @param query the query to test + * @param searcher the searcher to test the query against + * @param defaultFieldName used for displaying the query in assertion messages + * @param results a list of documentIds that must match the query + * @see Searcher#search(Query,Collector) + * @see #checkHits + */ + public static void checkHitCollector(Random random, Query query, String defaultFieldName, + IndexSearcher searcher, int[] results) + throws IOException { + + QueryUtils.check(random,query,searcher); + + Set correct = new TreeSet(); + for (int i = 0; i < results.length; i++) { + correct.add(Integer.valueOf(results[i])); + } + final Set actual = new TreeSet(); + final Collector c = new SetCollector(actual); + + searcher.search(query, c); + Assert.assertEquals("Simple: " + query.toString(defaultFieldName), + correct, actual); + + for (int i = -1; i < 2; i++) { + actual.clear(); + IndexSearcher s = QueryUtils.wrapUnderlyingReader + (random, searcher, i); + s.search(query, c); + Assert.assertEquals("Wrap Reader " + i + ": " + + query.toString(defaultFieldName), + correct, actual); + FieldCache.DEFAULT.purge(s.getIndexReader()); // our wrapping can create insanity otherwise + s.close(); + } + } + + public static class SetCollector extends Collector { + final Set bag; + public SetCollector(Set bag) { + this.bag = bag; + } + private int base = 0; + @Override + public void setScorer(Scorer scorer) throws IOException {} + @Override + public void collect(int doc) { + bag.add(Integer.valueOf(doc + base)); + } + @Override + public void setNextReader(AtomicReaderContext context) { + base = context.docBase; + } + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + } + + /** + * Tests that a query matches the an expected set of documents using Hits. + * + *

+ * Note that when using the Hits API, documents will only be returned + * if they have a positive normalized score. + *

+ * @param query the query to test + * @param searcher the searcher to test the query against + * @param defaultFieldName used for displaing the query in assertion messages + * @param results a list of documentIds that must match the query + * @see Searcher#search(Query, int) + * @see #checkHitCollector + */ + public static void checkHits( + Random random, + Query query, + String defaultFieldName, + IndexSearcher searcher, + int[] results) + throws IOException { + + ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs; + + Set correct = new TreeSet(); + for (int i = 0; i < results.length; i++) { + correct.add(Integer.valueOf(results[i])); + } + + Set actual = new TreeSet(); + for (int i = 0; i < hits.length; i++) { + actual.add(Integer.valueOf(hits[i].doc)); + } + + Assert.assertEquals(query.toString(defaultFieldName), correct, actual); + + QueryUtils.check(random, query,searcher); + } + + /** Tests that a Hits has an expected order of documents */ + public static void checkDocIds(String mes, int[] results, ScoreDoc[] hits) + throws IOException { + Assert.assertEquals(mes + " nr of hits", hits.length, results.length); + for (int i = 0; i < results.length; i++) { + Assert.assertEquals(mes + " doc nrs for hit " + i, results[i], hits[i].doc); + } + } + + /** Tests that two queries have an expected order of documents, + * and that the two queries have the same score values. + */ + public static void checkHitsQuery( + Query query, + ScoreDoc[] hits1, + ScoreDoc[] hits2, + int[] results) + throws IOException { + + checkDocIds("hits1", results, hits1); + checkDocIds("hits2", results, hits2); + checkEqual(query, hits1, hits2); + } + + public static void checkEqual(Query query, ScoreDoc[] hits1, ScoreDoc[] hits2) throws IOException { + final float scoreTolerance = 1.0e-6f; + if (hits1.length != hits2.length) { + Assert.fail("Unequal lengths: hits1="+hits1.length+",hits2="+hits2.length); + } + for (int i = 0; i < hits1.length; i++) { + if (hits1[i].doc != hits2[i].doc) { + Assert.fail("Hit " + i + " docnumbers don't match\n" + + hits2str(hits1, hits2,0,0) + + "for query:" + query.toString()); + } + + if ((hits1[i].doc != hits2[i].doc) + || Math.abs(hits1[i].score - hits2[i].score) > scoreTolerance) + { + Assert.fail("Hit " + i + ", doc nrs " + hits1[i].doc + " and " + hits2[i].doc + + "\nunequal : " + hits1[i].score + + "\n and: " + hits2[i].score + + "\nfor query:" + query.toString()); + } + } + } + + public static String hits2str(ScoreDoc[] hits1, ScoreDoc[] hits2, int start, int end) throws IOException { + StringBuilder sb = new StringBuilder(); + int len1=hits1==null ? 0 : hits1.length; + int len2=hits2==null ? 0 : hits2.length; + if (end<=0) { + end = Math.max(len1,len2); + } + + sb.append("Hits length1=").append(len1).append("\tlength2=").append(len2); + + sb.append('\n'); + for (int i=start; i times others" (where is float). + float x = 0; + String descr = expl.getDescription().toLowerCase(Locale.ENGLISH); + boolean productOf = descr.endsWith("product of:"); + boolean sumOf = descr.endsWith("sum of:"); + boolean maxOf = descr.endsWith("max of:"); + boolean maxTimesOthers = false; + if (!(productOf || sumOf || maxOf)) { + // maybe 'max plus x times others' + int k1 = descr.indexOf("max plus "); + if (k1>=0) { + k1 += "max plus ".length(); + int k2 = descr.indexOf(" ",k1); + try { + x = Float.parseFloat(descr.substring(k1,k2).trim()); + if (descr.substring(k2).trim().equals("times others of:")) { + maxTimesOthers = true; + } + } catch (NumberFormatException e) { + } + } + } + // TODO: this is a TERRIBLE assertion!!!! + Assert.assertTrue( + q+": multi valued explanation description=\""+descr + +"\" must be 'max of plus x times others' or end with 'product of'" + +" or 'sum of:' or 'max of:' - "+expl, + productOf || sumOf || maxOf || maxTimesOthers); + float sum = 0; + float product = 1; + float max = 0; + for (int i=0; i maxDiff + || scorerDiff > maxDiff) { + StringBuilder sbord = new StringBuilder(); + for (int i = 0; i < order.length; i++) + sbord.append(order[i] == skip_op ? " skip()" : " next()"); + throw new RuntimeException("ERROR matching docs:" + "\n\t" + + (doc != scorerDoc ? "--> " : "") + "doc=" + doc + ", scorerDoc=" + scorerDoc + + "\n\t" + (!more ? "--> " : "") + "tscorer.more=" + more + + "\n\t" + (scoreDiff > maxDiff ? "--> " : "") + + "scorerScore=" + scorerScore + " scoreDiff=" + scoreDiff + + " maxDiff=" + maxDiff + "\n\t" + + (scorerDiff > maxDiff ? "--> " : "") + "scorerScore2=" + + scorerScore2 + " scorerDiff=" + scorerDiff + + "\n\thitCollector.doc=" + doc + " score=" + score + + "\n\t Scorer=" + scorer + "\n\t Query=" + q + " " + + q.getClass().getName() + "\n\t Searcher=" + s + + "\n\t Order=" + sbord + "\n\t Op=" + + (op == skip_op ? " skip()" : " next()")); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void setNextReader(AtomicReaderContext context) throws IOException { + // confirm that skipping beyond the last doc, on the + // previous reader, hits NO_MORE_DOCS + if (lastReader[0] != null) { + final IndexReader previousReader = lastReader[0]; + IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); + Weight w = indexSearcher.createNormalizedWeight(q); + AtomicReaderContext ctx = (AtomicReaderContext)indexSearcher.getTopReaderContext(); + Scorer scorer = w.scorer(ctx, true, false, ctx.reader.getLiveDocs()); + if (scorer != null) { + boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; + Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); + } + leafPtr++; + indexSearcher.close(); + } + lastReader[0] = context.reader; + assert readerContextArray[leafPtr].reader == context.reader; + this.scorer = null; + lastDoc[0] = -1; + } + + @Override + public boolean acceptsDocsOutOfOrder() { + return true; + } + }); + + if (lastReader[0] != null) { + // confirm that skipping beyond the last doc, on the + // previous reader, hits NO_MORE_DOCS + final IndexReader previousReader = lastReader[0]; + IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader, false); + Weight w = indexSearcher.createNormalizedWeight(q); + AtomicReaderContext ctx = (AtomicReaderContext)previousReader.getTopReaderContext(); + Scorer scorer = w.scorer(ctx, true, false, ctx.reader.getLiveDocs()); + if (scorer != null) { + boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; + Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); + } + indexSearcher.close(); + } + } + } + + // check that first skip on just created scorers always goes to the right doc + private static void checkFirstSkipTo(final Query q, final IndexSearcher s) throws IOException { + //System.out.println("checkFirstSkipTo: "+q); + final float maxDiff = 1e-3f; + final int lastDoc[] = {-1}; + final IndexReader lastReader[] = {null}; + final AtomicReaderContext[] context = ReaderUtil.leaves(s.getTopReaderContext()); + s.search(q,new Collector() { + private Scorer scorer; + private int leafPtr; + private Bits liveDocs; + @Override + public void setScorer(Scorer scorer) throws IOException { + this.scorer = scorer; + } + @Override + public void collect(int doc) throws IOException { + float score = scorer.score(); + try { + long startMS = System.currentTimeMillis(); + for (int i=lastDoc[0]+1; i<=doc; i++) { + Weight w = s.createNormalizedWeight(q); + Scorer scorer = w.scorer(context[leafPtr], true, false, liveDocs); + Assert.assertTrue("query collected "+doc+" but skipTo("+i+") says no more docs!",scorer.advance(i) != DocIdSetIterator.NO_MORE_DOCS); + Assert.assertEquals("query collected "+doc+" but skipTo("+i+") got to "+scorer.docID(),doc,scorer.docID()); + float skipToScore = scorer.score(); + Assert.assertEquals("unstable skipTo("+i+") score!",skipToScore,scorer.score(),maxDiff); + Assert.assertEquals("query assigned doc "+doc+" a score of <"+score+"> but skipTo("+i+") has <"+skipToScore+">!",score,skipToScore,maxDiff); + + // Hurry things along if they are going slow (eg + // if you got SimpleText codec this will kick in): + if (i < doc && System.currentTimeMillis() - startMS > 5) { + i = doc-1; + } + } + lastDoc[0] = doc; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Override + public void setNextReader(AtomicReaderContext context) throws IOException { + // confirm that skipping beyond the last doc, on the + // previous reader, hits NO_MORE_DOCS + if (lastReader[0] != null) { + final IndexReader previousReader = lastReader[0]; + IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); + Weight w = indexSearcher.createNormalizedWeight(q); + Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), true, false, previousReader.getLiveDocs()); + if (scorer != null) { + boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; + Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); + } + indexSearcher.close(); + leafPtr++; + } + + lastReader[0] = context.reader; + lastDoc[0] = -1; + liveDocs = context.reader.getLiveDocs(); + } + @Override + public boolean acceptsDocsOutOfOrder() { + return false; + } + }); + + if (lastReader[0] != null) { + // confirm that skipping beyond the last doc, on the + // previous reader, hits NO_MORE_DOCS + final IndexReader previousReader = lastReader[0]; + IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); + Weight w = indexSearcher.createNormalizedWeight(q); + Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), true, false, previousReader.getLiveDocs()); + if (scorer != null) { + boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; + Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); + } + indexSearcher.close(); + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/search/RandomSimilarityProvider.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/search/RandomSimilarityProvider.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/search/RandomSimilarityProvider.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/search/RandomSimilarityProvider.java 2011-11-01 21:10:36.008609908 -0400 @@ -0,0 +1,158 @@ +package org.apache.lucene.search; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; + +import org.apache.lucene.search.similarities.AfterEffect; +import org.apache.lucene.search.similarities.AfterEffectB; +import org.apache.lucene.search.similarities.AfterEffectL; +import org.apache.lucene.search.similarities.BM25Similarity; +import org.apache.lucene.search.similarities.BasicModel; +import org.apache.lucene.search.similarities.BasicModelBE; +import org.apache.lucene.search.similarities.BasicModelD; +import org.apache.lucene.search.similarities.BasicModelG; +import org.apache.lucene.search.similarities.BasicModelIF; +import org.apache.lucene.search.similarities.BasicModelIn; +import org.apache.lucene.search.similarities.BasicModelIne; +import org.apache.lucene.search.similarities.BasicModelP; +import org.apache.lucene.search.similarities.DFRSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarity; +import org.apache.lucene.search.similarities.DefaultSimilarityProvider; +import org.apache.lucene.search.similarities.Distribution; +import org.apache.lucene.search.similarities.DistributionLL; +import org.apache.lucene.search.similarities.DistributionSPL; +import org.apache.lucene.search.similarities.IBSimilarity; +import org.apache.lucene.search.similarities.LMDirichletSimilarity; +import org.apache.lucene.search.similarities.LMJelinekMercerSimilarity; +import org.apache.lucene.search.similarities.Lambda; +import org.apache.lucene.search.similarities.LambdaDF; +import org.apache.lucene.search.similarities.LambdaTTF; +import org.apache.lucene.search.similarities.Normalization; +import org.apache.lucene.search.similarities.NormalizationH1; +import org.apache.lucene.search.similarities.NormalizationH2; +import org.apache.lucene.search.similarities.NormalizationH3; +import org.apache.lucene.search.similarities.NormalizationZ; +import org.apache.lucene.search.similarities.Similarity; + +public class RandomSimilarityProvider extends DefaultSimilarityProvider { + final List knownSims; + Map previousMappings = new HashMap(); + final int perFieldSeed; + final boolean shouldCoord; + final boolean shouldQueryNorm; + + public RandomSimilarityProvider(Random random) { + perFieldSeed = random.nextInt(); + shouldCoord = random.nextBoolean(); + shouldQueryNorm = random.nextBoolean(); + knownSims = new ArrayList(allSims); + Collections.shuffle(knownSims, random); + } + + @Override + public float coord(int overlap, int maxOverlap) { + if (shouldCoord) { + return super.coord(overlap, maxOverlap); + } else { + return 1.0f; + } + } + + @Override + public float queryNorm(float sumOfSquaredWeights) { + if (shouldQueryNorm) { + return super.queryNorm(sumOfSquaredWeights); + } else { + return 1.0f; + } + } + + @Override + public synchronized Similarity get(String field) { + assert field != null; + Similarity sim = previousMappings.get(field); + if (sim == null) { + sim = knownSims.get(Math.abs(perFieldSeed ^ field.hashCode()) % knownSims.size()); + previousMappings.put(field, sim); + } + return sim; + } + + // all the similarities that we rotate through + /** The DFR basic models to test. */ + static BasicModel[] BASIC_MODELS = { + /* TODO: enable new BasicModelBE(), */ /* TODO: enable new BasicModelD(), */ new BasicModelG(), + new BasicModelIF(), new BasicModelIn(), new BasicModelIne(), + /* TODO: enable new BasicModelP() */ + }; + /** The DFR aftereffects to test. */ + static AfterEffect[] AFTER_EFFECTS = { + new AfterEffectB(), new AfterEffectL(), new AfterEffect.NoAfterEffect() + }; + /** The DFR normalizations to test. */ + static Normalization[] NORMALIZATIONS = { + new NormalizationH1(), new NormalizationH2(), + new NormalizationH3(), new NormalizationZ() + // TODO: if we enable NoNormalization, we have to deal with + // a couple tests (e.g. TestDocBoost, TestSort) that expect length normalization + // new Normalization.NoNormalization() + }; + /** The distributions for IB. */ + static Distribution[] DISTRIBUTIONS = { + new DistributionLL(), new DistributionSPL() + }; + /** Lambdas for IB. */ + static Lambda[] LAMBDAS = { + new LambdaDF(), new LambdaTTF() + }; + static List allSims; + static { + allSims = new ArrayList(); + allSims.add(new DefaultSimilarity()); + allSims.add(new BM25Similarity()); + for (BasicModel basicModel : BASIC_MODELS) { + for (AfterEffect afterEffect : AFTER_EFFECTS) { + for (Normalization normalization : NORMALIZATIONS) { + allSims.add(new DFRSimilarity(basicModel, afterEffect, normalization)); + } + } + } + for (Distribution distribution : DISTRIBUTIONS) { + for (Lambda lambda : LAMBDAS) { + for (Normalization normalization : NORMALIZATIONS) { + allSims.add(new IBSimilarity(distribution, lambda, normalization)); + } + } + } + /* TODO: enable Dirichlet + allSims.add(new LMDirichletSimilarity()); */ + allSims.add(new LMJelinekMercerSimilarity(0.1f)); + allSims.add(new LMJelinekMercerSimilarity(0.7f)); + } + + @Override + public synchronized String toString() { + return "RandomSimilarityProvider(queryNorm=" + shouldQueryNorm + ",coord=" + shouldCoord + "): " + previousMappings.toString(); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/store/MockDirectoryWrapper.java 2011-11-01 21:10:36.112609906 -0400 @@ -0,0 +1,695 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Closeable; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.ThrottledIndexOutput; +import org.apache.lucene.util._TestUtil; + +/** + * This is a Directory Wrapper that adds methods + * intended to be used only by unit tests. + * It also adds a number of features useful for testing: + *
    + *
  • Instances created by {@link LuceneTestCase#newDirectory()} are tracked + * to ensure they are closed by the test. + *
  • When a MockDirectoryWrapper is closed, it will throw an exception if + * it has any open files against it (with a stacktrace indicating where + * they were opened from). + *
  • When a MockDirectoryWrapper is closed, it runs CheckIndex to test if + * the index was corrupted. + *
  • MockDirectoryWrapper simulates some "features" of Windows, such as + * refusing to write/delete to open files. + *
+ */ + +public class MockDirectoryWrapper extends Directory { + final Directory delegate; + long maxSize; + + // Max actual bytes used. This is set by MockRAMOutputStream: + long maxUsedSize; + double randomIOExceptionRate; + Random randomState; + boolean noDeleteOpenFile = true; + boolean preventDoubleWrite = true; + boolean checkIndexOnClose = true; + boolean trackDiskUsage = false; + private Set unSyncedFiles; + private Set createdFiles; + private Set openFilesForWrite = new HashSet(); + Set openLocks = Collections.synchronizedSet(new HashSet()); + volatile boolean crashed; + private ThrottledIndexOutput throttledOutput; + private Throttling throttling = Throttling.SOMETIMES; + + final AtomicInteger inputCloneCount = new AtomicInteger(); + + // use this for tracking files for crash. + // additionally: provides debugging information in case you leave one open + private Map openFileHandles = Collections.synchronizedMap(new IdentityHashMap()); + + // NOTE: we cannot initialize the Map here due to the + // order in which our constructor actually does this + // member initialization vs when it calls super. It seems + // like super is called, then our members are initialized: + private Map openFiles; + + // Only tracked if noDeleteOpenFile is true: if an attempt + // is made to delete an open file, we enroll it here. + private Set openFilesDeleted; + + private synchronized void init() { + if (openFiles == null) { + openFiles = new HashMap(); + openFilesDeleted = new HashSet(); + } + + if (createdFiles == null) + createdFiles = new HashSet(); + if (unSyncedFiles == null) + unSyncedFiles = new HashSet(); + } + + public MockDirectoryWrapper(Random random, Directory delegate) { + this.delegate = delegate; + // must make a private random since our methods are + // called from different threads; else test failures may + // not be reproducible from the original seed + this.randomState = new Random(random.nextInt()); + this.throttledOutput = new ThrottledIndexOutput(ThrottledIndexOutput + .mBitsToBytes(40 + randomState.nextInt(10)), 5 + randomState.nextInt(5), null); + // force wrapping of lockfactory + try { + setLockFactory(new MockLockFactoryWrapper(this, delegate.getLockFactory())); + } catch (IOException e) { + throw new RuntimeException(e); + } + init(); + } + + public int getInputCloneCount() { + return inputCloneCount.get(); + } + + public void setTrackDiskUsage(boolean v) { + trackDiskUsage = v; + } + + /** If set to true, we throw an IOException if the same + * file is opened by createOutput, ever. */ + public void setPreventDoubleWrite(boolean value) { + preventDoubleWrite = value; + } + + public static enum Throttling { + /** always emulate a slow hard disk. could be very slow! */ + ALWAYS, + /** sometimes (2% of the time) emulate a slow hard disk. */ + SOMETIMES, + /** never throttle output */ + NEVER + } + + public void setThrottling(Throttling throttling) { + this.throttling = throttling; + } + + @Override + public synchronized void sync(Collection names) throws IOException { + maybeYield(); + for (String name : names) + maybeThrowDeterministicException(); + if (crashed) + throw new IOException("cannot sync after crash"); + unSyncedFiles.removeAll(names); + delegate.sync(names); + } + + @Override + public String toString() { + // NOTE: do not maybeYield here, since it consumes + // randomness and can thus (unexpectedly during + // debugging) change the behavior of a seed + // maybeYield(); + return "MockDirWrapper(" + delegate + ")"; + } + + public synchronized final long sizeInBytes() throws IOException { + if (delegate instanceof RAMDirectory) + return ((RAMDirectory) delegate).sizeInBytes(); + else { + // hack + long size = 0; + for (String file : delegate.listAll()) + size += delegate.fileLength(file); + return size; + } + } + + /** Simulates a crash of OS or machine by overwriting + * unsynced files. */ + public synchronized void crash() throws IOException { + crashed = true; + openFiles = new HashMap(); + openFilesForWrite = new HashSet(); + openFilesDeleted = new HashSet(); + Iterator it = unSyncedFiles.iterator(); + unSyncedFiles = new HashSet(); + // first force-close all files, so we can corrupt on windows etc. + // clone the file map, as these guys want to remove themselves on close. + Map m = new IdentityHashMap(openFileHandles); + for (Closeable f : m.keySet()) + try { + f.close(); + } catch (Exception ignored) {} + + int count = 0; + while(it.hasNext()) { + String name = it.next(); + if (count % 3 == 0) { + deleteFile(name, true); + } else if (count % 3 == 1) { + // Zero out file entirely + long length = fileLength(name); + byte[] zeroes = new byte[256]; + long upto = 0; + IndexOutput out = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState)); + while(upto < length) { + final int limit = (int) Math.min(length-upto, zeroes.length); + out.writeBytes(zeroes, 0, limit); + upto += limit; + } + out.close(); + } else if (count % 3 == 2) { + // Truncate the file: + IndexOutput out = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState)); + out.setLength(fileLength(name)/2); + out.close(); + } + count++; + } + } + + public synchronized void clearCrash() throws IOException { + crashed = false; + openLocks.clear(); + } + + public void setMaxSizeInBytes(long maxSize) { + this.maxSize = maxSize; + } + public long getMaxSizeInBytes() { + return this.maxSize; + } + + /** + * Returns the peek actual storage used (bytes) in this + * directory. + */ + public long getMaxUsedSizeInBytes() { + return this.maxUsedSize; + } + public void resetMaxUsedSizeInBytes() throws IOException { + this.maxUsedSize = getRecomputedActualSizeInBytes(); + } + + /** + * Emulate windows whereby deleting an open file is not + * allowed (raise IOException). + */ + public void setNoDeleteOpenFile(boolean value) { + this.noDeleteOpenFile = value; + } + public boolean getNoDeleteOpenFile() { + return noDeleteOpenFile; + } + + /** + * Set whether or not checkindex should be run + * on close + */ + public void setCheckIndexOnClose(boolean value) { + this.checkIndexOnClose = value; + } + + public boolean getCheckIndexOnClose() { + return checkIndexOnClose; + } + /** + * If 0.0, no exceptions will be thrown. Else this should + * be a double 0.0 - 1.0. We will randomly throw an + * IOException on the first write to an OutputStream based + * on this probability. + */ + public void setRandomIOExceptionRate(double rate) { + randomIOExceptionRate = rate; + } + public double getRandomIOExceptionRate() { + return randomIOExceptionRate; + } + + void maybeThrowIOException() throws IOException { + maybeThrowIOException(null); + } + + void maybeThrowIOException(String message) throws IOException { + if (randomIOExceptionRate > 0.0) { + int number = Math.abs(randomState.nextInt() % 1000); + if (number < randomIOExceptionRate*1000) { + if (LuceneTestCase.VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception" + (message == null ? "" : " (" + message + ")")); + new Throwable().printStackTrace(System.out); + } + throw new IOException("a random IOException" + (message == null ? "" : "(" + message + ")")); + } + } + } + + @Override + public synchronized void deleteFile(String name) throws IOException { + maybeYield(); + deleteFile(name, false); + } + + // sets the cause of the incoming ioe to be the stack + // trace when the offending file name was opened + private synchronized IOException fillOpenTrace(IOException ioe, String name, boolean input) { + for(Map.Entry ent : openFileHandles.entrySet()) { + if (input && ent.getKey() instanceof MockIndexInputWrapper && ((MockIndexInputWrapper) ent.getKey()).name.equals(name)) { + ioe.initCause(ent.getValue()); + break; + } else if (!input && ent.getKey() instanceof MockIndexOutputWrapper && ((MockIndexOutputWrapper) ent.getKey()).name.equals(name)) { + ioe.initCause(ent.getValue()); + break; + } + } + return ioe; + } + + private void maybeYield() { + if (randomState.nextBoolean()) { + Thread.yield(); + } + } + + private synchronized void deleteFile(String name, boolean forced) throws IOException { + maybeYield(); + + maybeThrowDeterministicException(); + + if (crashed && !forced) + throw new IOException("cannot delete after crash"); + + if (unSyncedFiles.contains(name)) + unSyncedFiles.remove(name); + if (!forced && noDeleteOpenFile) { + if (openFiles.containsKey(name)) { + openFilesDeleted.add(name); + throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true); + } else { + openFilesDeleted.remove(name); + } + } + delegate.deleteFile(name); + } + + public synchronized Set getOpenDeletedFiles() { + return new HashSet(openFilesDeleted); + } + + @Override + public synchronized IndexOutput createOutput(String name, IOContext context) throws IOException { + maybeYield(); + if (crashed) + throw new IOException("cannot createOutput after crash"); + init(); + synchronized(this) { + if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen")) + throw new IOException("file \"" + name + "\" was already written to"); + } + if (noDeleteOpenFile && openFiles.containsKey(name)) + throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite"); + + if (crashed) + throw new IOException("cannot createOutput after crash"); + unSyncedFiles.add(name); + createdFiles.add(name); + + if (delegate instanceof RAMDirectory) { + RAMDirectory ramdir = (RAMDirectory) delegate; + RAMFile file = new RAMFile(ramdir); + RAMFile existing = ramdir.fileMap.get(name); + + // Enforce write once: + if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite) + throw new IOException("file " + name + " already exists"); + else { + if (existing!=null) { + ramdir.sizeInBytes.getAndAdd(-existing.sizeInBytes); + existing.directory = null; + } + ramdir.fileMap.put(name, file); + } + } + + //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name); + IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name, LuceneTestCase.newIOContext(randomState)), name); + addFileHandle(io, name, Handle.Output); + openFilesForWrite.add(name); + + // throttling REALLY slows down tests, so don't do it very often for SOMETIMES. + if (throttling == Throttling.ALWAYS || + (throttling == Throttling.SOMETIMES && randomState.nextInt(50) == 0)) { + if (LuceneTestCase.VERBOSE) { + System.out.println("MockDirectoryWrapper: throttling indexOutput"); + } + return throttledOutput.newFromDelegate(io); + } else { + return io; + } + } + + private static enum Handle { + Input, Output, Slice + } + + synchronized void addFileHandle(Closeable c, String name, Handle handle) { + Integer v = openFiles.get(name); + if (v != null) { + v = Integer.valueOf(v.intValue()+1); + openFiles.put(name, v); + } else { + openFiles.put(name, Integer.valueOf(1)); + } + + openFileHandles.put(c, new RuntimeException("unclosed Index" + handle.name() + ": " + name)); + } + + @Override + public synchronized IndexInput openInput(String name, IOContext context) throws IOException { + maybeYield(); + if (!delegate.fileExists(name)) + throw new FileNotFoundException(name); + + // cannot open a file for input if it's still open for + // output, except for segments.gen and segments_N + if (openFilesForWrite.contains(name) && !name.startsWith("segments")) { + throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); + } + + IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name, LuceneTestCase.newIOContext(randomState))); + addFileHandle(ii, name, Handle.Input); + return ii; + } + + /** Provided for testing purposes. Use sizeInBytes() instead. */ + public synchronized final long getRecomputedSizeInBytes() throws IOException { + if (!(delegate instanceof RAMDirectory)) + return sizeInBytes(); + long size = 0; + for(final RAMFile file: ((RAMDirectory)delegate).fileMap.values()) { + size += file.getSizeInBytes(); + } + return size; + } + + /** Like getRecomputedSizeInBytes(), but, uses actual file + * lengths rather than buffer allocations (which are + * quantized up to nearest + * RAMOutputStream.BUFFER_SIZE (now 1024) bytes. + */ + + public final synchronized long getRecomputedActualSizeInBytes() throws IOException { + if (!(delegate instanceof RAMDirectory)) + return sizeInBytes(); + long size = 0; + for (final RAMFile file : ((RAMDirectory)delegate).fileMap.values()) + size += file.length; + return size; + } + + @Override + public synchronized void close() throws IOException { + maybeYield(); + if (openFiles == null) { + openFiles = new HashMap(); + openFilesDeleted = new HashSet(); + } + if (noDeleteOpenFile && openFiles.size() > 0) { + // print the first one as its very verbose otherwise + Exception cause = null; + Iterator stacktraces = openFileHandles.values().iterator(); + if (stacktraces.hasNext()) + cause = stacktraces.next(); + // RuntimeException instead of IOException because + // super() does not throw IOException currently: + throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause); + } + if (noDeleteOpenFile && openLocks.size() > 0) { + throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks); + } + open = false; + if (checkIndexOnClose) { + if (LuceneTestCase.VERBOSE) { + System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex"); + } + if (IndexReader.indexExists(this)) { + _TestUtil.checkIndex(this); + } + } + delegate.close(); + } + + synchronized void removeOpenFile(Closeable c, String name) { + Integer v = openFiles.get(name); + // Could be null when crash() was called + if (v != null) { + if (v.intValue() == 1) { + openFiles.remove(name); + openFilesDeleted.remove(name); + } else { + v = Integer.valueOf(v.intValue()-1); + openFiles.put(name, v); + } + } + + openFileHandles.remove(c); + } + + public synchronized void removeIndexOutput(IndexOutput out, String name) { + openFilesForWrite.remove(name); + removeOpenFile(out, name); + } + + public synchronized void removeIndexInput(IndexInput in, String name) { + removeOpenFile(in, name); + } + + boolean open = true; + + public synchronized boolean isOpen() { + return open; + } + + /** + * Objects that represent fail-able conditions. Objects of a derived + * class are created and registered with the mock directory. After + * register, each object will be invoked once for each first write + * of a file, giving the object a chance to throw an IOException. + */ + public static class Failure { + /** + * eval is called on the first write of every new file. + */ + public void eval(MockDirectoryWrapper dir) throws IOException { } + + /** + * reset should set the state of the failure to its default + * (freshly constructed) state. Reset is convenient for tests + * that want to create one failure object and then reuse it in + * multiple cases. This, combined with the fact that Failure + * subclasses are often anonymous classes makes reset difficult to + * do otherwise. + * + * A typical example of use is + * Failure failure = new Failure() { ... }; + * ... + * mock.failOn(failure.reset()) + */ + public Failure reset() { return this; } + + protected boolean doFail; + + public void setDoFail() { + doFail = true; + } + + public void clearDoFail() { + doFail = false; + } + } + + ArrayList failures; + + /** + * add a Failure object to the list of objects to be evaluated + * at every potential failure point + */ + synchronized public void failOn(Failure fail) { + if (failures == null) { + failures = new ArrayList(); + } + failures.add(fail); + } + + /** + * Iterate through the failures list, giving each object a + * chance to throw an IOE + */ + synchronized void maybeThrowDeterministicException() throws IOException { + if (failures != null) { + for(int i = 0; i < failures.size(); i++) { + failures.get(i).eval(this); + } + } + } + + @Override + public synchronized String[] listAll() throws IOException { + maybeYield(); + return delegate.listAll(); + } + + @Override + public synchronized boolean fileExists(String name) throws IOException { + maybeYield(); + return delegate.fileExists(name); + } + + @Override + public synchronized long fileModified(String name) throws IOException { + maybeYield(); + return delegate.fileModified(name); + } + + @Override + public synchronized long fileLength(String name) throws IOException { + maybeYield(); + return delegate.fileLength(name); + } + + @Override + public synchronized Lock makeLock(String name) { + maybeYield(); + return delegate.makeLock(name); + } + + @Override + public synchronized void clearLock(String name) throws IOException { + maybeYield(); + delegate.clearLock(name); + } + + @Override + public synchronized void setLockFactory(LockFactory lockFactory) throws IOException { + maybeYield(); + delegate.setLockFactory(lockFactory); + } + + @Override + public synchronized LockFactory getLockFactory() { + maybeYield(); + return delegate.getLockFactory(); + } + + @Override + public synchronized String getLockID() { + maybeYield(); + return delegate.getLockID(); + } + + @Override + public synchronized void copy(Directory to, String src, String dest, IOContext context) throws IOException { + maybeYield(); + // randomize the IOContext here? + delegate.copy(to, src, dest, context); + } + + @Override + public IndexInputSlicer createSlicer(final String name, IOContext context) + throws IOException { + maybeYield(); + if (!delegate.fileExists(name)) + throw new FileNotFoundException(name); + // cannot open a file for input if it's still open for + // output, except for segments.gen and segments_N + if (openFilesForWrite.contains(name) && !name.startsWith("segments")) { + throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); + } + + final IndexInputSlicer delegateHandle = delegate.createSlicer(name, context); + final IndexInputSlicer handle = new IndexInputSlicer() { + + private boolean isClosed; + @Override + public void close() throws IOException { + if (!isClosed) { + delegateHandle.close(); + MockDirectoryWrapper.this.removeOpenFile(this, name); + isClosed = true; + } + } + + @Override + public IndexInput openSlice(long offset, long length) throws IOException { + maybeYield(); + IndexInput ii = new MockIndexInputWrapper(MockDirectoryWrapper.this, name, delegateHandle.openSlice(offset, length)); + addFileHandle(ii, name, Handle.Input); + return ii; + } + + @Override + public IndexInput openFullSlice() throws IOException { + maybeYield(); + IndexInput ii = new MockIndexInputWrapper(MockDirectoryWrapper.this, name, delegateHandle.openFullSlice()); + addFileHandle(ii, name, Handle.Input); + return ii; + } + + }; + addFileHandle(handle, name, Handle.Slice); + return handle; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexInputWrapper.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexInputWrapper.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexInputWrapper.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexInputWrapper.java 2011-11-02 12:49:35.428580881 -0400 @@ -0,0 +1,180 @@ +package org.apache.lucene.store; + +import java.io.IOException; +import java.util.Map; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Used by MockDirectoryWrapper to create an input stream that + * keeps track of when it's been closed. + */ + +public class MockIndexInputWrapper extends IndexInput { + private MockDirectoryWrapper dir; + final String name; + private IndexInput delegate; + private boolean isClone; + private boolean closed; + + /** Construct an empty output buffer. */ + public MockIndexInputWrapper(MockDirectoryWrapper dir, String name, IndexInput delegate) { + this.name = name; + this.dir = dir; + this.delegate = delegate; + } + + @Override + public void close() throws IOException { + try { + // turn on the following to look for leaks closing inputs, + // after fixing TestTransactions + // dir.maybeThrowDeterministicException(); + } finally { + closed = true; + delegate.close(); + // Pending resolution on LUCENE-686 we may want to + // remove the conditional check so we also track that + // all clones get closed: + if (!isClone) { + dir.removeIndexInput(this, name); + } + } + } + + private void ensureOpen() { + if (closed) { + throw new RuntimeException("Abusing closed IndexInput!"); + } + } + + @Override + public Object clone() { + ensureOpen(); + dir.inputCloneCount.incrementAndGet(); + IndexInput iiclone = (IndexInput) delegate.clone(); + MockIndexInputWrapper clone = new MockIndexInputWrapper(dir, name, iiclone); + clone.isClone = true; + // Pending resolution on LUCENE-686 we may want to + // uncomment this code so that we also track that all + // clones get closed: + /* + synchronized(dir.openFiles) { + if (dir.openFiles.containsKey(name)) { + Integer v = (Integer) dir.openFiles.get(name); + v = Integer.valueOf(v.intValue()+1); + dir.openFiles.put(name, v); + } else { + throw new RuntimeException("BUG: cloned file was not open?"); + } + } + */ + return clone; + } + + @Override + public long getFilePointer() { + ensureOpen(); + return delegate.getFilePointer(); + } + + @Override + public void seek(long pos) throws IOException { + ensureOpen(); + delegate.seek(pos); + } + + @Override + public long length() { + ensureOpen(); + return delegate.length(); + } + + @Override + public byte readByte() throws IOException { + ensureOpen(); + return delegate.readByte(); + } + + @Override + public void readBytes(byte[] b, int offset, int len) throws IOException { + ensureOpen(); + delegate.readBytes(b, offset, len); + } + + @Override + public void copyBytes(IndexOutput out, long numBytes) throws IOException { + ensureOpen(); + delegate.copyBytes(out, numBytes); + } + + @Override + public void readBytes(byte[] b, int offset, int len, boolean useBuffer) + throws IOException { + ensureOpen(); + delegate.readBytes(b, offset, len, useBuffer); + } + + @Override + public short readShort() throws IOException { + ensureOpen(); + return delegate.readShort(); + } + + @Override + public int readInt() throws IOException { + ensureOpen(); + return delegate.readInt(); + } + + @Override + public long readLong() throws IOException { + ensureOpen(); + return delegate.readLong(); + } + + @Override + public String readString() throws IOException { + ensureOpen(); + return delegate.readString(); + } + + @Override + public Map readStringStringMap() throws IOException { + ensureOpen(); + return delegate.readStringStringMap(); + } + + @Override + public int readVInt() throws IOException { + ensureOpen(); + return delegate.readVInt(); + } + + @Override + public long readVLong() throws IOException { + ensureOpen(); + return delegate.readVLong(); + } + + @Override + public String toString() { + return "MockIndexInputWrapper(" + delegate + ")"; + } +} + diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexOutputWrapper.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexOutputWrapper.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexOutputWrapper.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/store/MockIndexOutputWrapper.java 2011-11-01 21:10:36.112609906 -0400 @@ -0,0 +1,164 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.util.LuceneTestCase; + +/** + * Used by MockRAMDirectory to create an output stream that + * will throw an IOException on fake disk full, track max + * disk space actually used, and maybe throw random + * IOExceptions. + */ + +public class MockIndexOutputWrapper extends IndexOutput { + private MockDirectoryWrapper dir; + private final IndexOutput delegate; + private boolean first=true; + final String name; + + byte[] singleByte = new byte[1]; + + /** Construct an empty output buffer. */ + public MockIndexOutputWrapper(MockDirectoryWrapper dir, IndexOutput delegate, String name) { + this.dir = dir; + this.name = name; + this.delegate = delegate; + } + + @Override + public void close() throws IOException { + try { + dir.maybeThrowDeterministicException(); + } finally { + delegate.close(); + if (dir.trackDiskUsage) { + // Now compute actual disk usage & track the maxUsedSize + // in the MockDirectoryWrapper: + long size = dir.getRecomputedActualSizeInBytes(); + if (size > dir.maxUsedSize) { + dir.maxUsedSize = size; + } + } + dir.removeIndexOutput(this, name); + } + } + + @Override + public void flush() throws IOException { + dir.maybeThrowDeterministicException(); + delegate.flush(); + } + + @Override + public void writeByte(byte b) throws IOException { + singleByte[0] = b; + writeBytes(singleByte, 0, 1); + } + + @Override + public void writeBytes(byte[] b, int offset, int len) throws IOException { + long freeSpace = dir.maxSize == 0 ? 0 : dir.maxSize - dir.sizeInBytes(); + long realUsage = 0; + + // If MockRAMDir crashed since we were opened, then + // don't write anything: + if (dir.crashed) + throw new IOException("MockRAMDirectory was crashed; cannot write to " + name); + + // Enforce disk full: + if (dir.maxSize != 0 && freeSpace <= len) { + // Compute the real disk free. This will greatly slow + // down our test but makes it more accurate: + realUsage = dir.getRecomputedActualSizeInBytes(); + freeSpace = dir.maxSize - realUsage; + } + + if (dir.maxSize != 0 && freeSpace <= len) { + if (freeSpace > 0) { + realUsage += freeSpace; + delegate.writeBytes(b, offset, (int) freeSpace); + } + if (realUsage > dir.maxUsedSize) { + dir.maxUsedSize = realUsage; + } + String message = "fake disk full at " + dir.getRecomputedActualSizeInBytes() + " bytes when writing " + name + " (file length=" + delegate.length(); + if (freeSpace > 0) { + message += "; wrote " + freeSpace + " of " + len + " bytes"; + } + message += ")"; + if (LuceneTestCase.VERBOSE) { + System.out.println(Thread.currentThread().getName() + ": MDW: now throw fake disk full"); + new Throwable().printStackTrace(System.out); + } + throw new IOException(message); + } else { + if (dir.randomState.nextInt(200) == 0) { + final int half = len/2; + delegate.writeBytes(b, offset, half); + Thread.yield(); + delegate.writeBytes(b, offset+half, len-half); + } else { + delegate.writeBytes(b, offset, len); + } + } + + dir.maybeThrowDeterministicException(); + + if (first) { + // Maybe throw random exception; only do this on first + // write to a new file: + first = false; + dir.maybeThrowIOException(name); + } + } + + @Override + public long getFilePointer() { + return delegate.getFilePointer(); + } + + @Override + public void seek(long pos) throws IOException { + delegate.seek(pos); + } + + @Override + public long length() throws IOException { + return delegate.length(); + } + + @Override + public void setLength(long length) throws IOException { + delegate.setLength(length); + } + + @Override + public void copyBytes(DataInput input, long numBytes) throws IOException { + delegate.copyBytes(input, numBytes); + // TODO: we may need to check disk full here as well + dir.maybeThrowDeterministicException(); + } + + @Override + public String toString() { + return "MockIndexOutputWrapper(" + delegate + ")"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockLockFactoryWrapper.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/store/MockLockFactoryWrapper.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/store/MockLockFactoryWrapper.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/store/MockLockFactoryWrapper.java 2011-11-01 21:10:36.112609906 -0400 @@ -0,0 +1,87 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +public class MockLockFactoryWrapper extends LockFactory { + MockDirectoryWrapper dir; + LockFactory delegate; + + public MockLockFactoryWrapper(MockDirectoryWrapper dir, LockFactory delegate) { + this.dir = dir; + this.delegate = delegate; + } + + @Override + public void setLockPrefix(String lockPrefix) { + delegate.setLockPrefix(lockPrefix); + } + + @Override + public String getLockPrefix() { + return delegate.getLockPrefix(); + } + + @Override + public Lock makeLock(String lockName) { + return new MockLock(delegate.makeLock(lockName), lockName); + } + + @Override + public void clearLock(String lockName) throws IOException { + delegate.clearLock(lockName); + dir.openLocks.remove(lockName); + } + + @Override + public String toString() { + return "MockLockFactoryWrapper(" + delegate.toString() + ")"; + } + + private class MockLock extends Lock { + private Lock delegateLock; + private String name; + + MockLock(Lock delegate, String name) { + this.delegateLock = delegate; + this.name = name; + } + + @Override + public boolean obtain() throws IOException { + if (delegateLock.obtain()) { + dir.openLocks.add(name); + return true; + } else { + return false; + } + } + + @Override + public void release() throws IOException { + delegateLock.release(); + dir.openLocks.remove(name); + } + + @Override + public boolean isLocked() throws IOException { + return delegateLock.isLocked(); + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/store/_TestHelper.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/store/_TestHelper.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/store/_TestHelper.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/store/_TestHelper.java 2011-11-01 21:10:36.112609906 -0400 @@ -0,0 +1,65 @@ +package org.apache.lucene.store; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput; + +/** This class provides access to package-level features defined in the + * store package. It is used for testing only. + */ +public class _TestHelper { + + /** Returns true if the instance of the provided input stream is actually + * an SimpleFSIndexInput. + */ + public static boolean isSimpleFSIndexInput(IndexInput is) { + return is instanceof SimpleFSIndexInput; + } + + /** Returns true if the provided input stream is an SimpleFSIndexInput and + * is a clone, that is it does not own its underlying file descriptor. + */ + public static boolean isSimpleFSIndexInputClone(IndexInput is) { + if (isSimpleFSIndexInput(is)) { + return ((SimpleFSIndexInput) is).isClone; + } else { + return false; + } + } + + /** Given an instance of SimpleFSDirectory.SimpleFSIndexInput, this method returns + * true if the underlying file descriptor is valid, and false otherwise. + * This can be used to determine if the OS file has been closed. + * The descriptor becomes invalid when the non-clone instance of the + * SimpleFSIndexInput that owns this descriptor is closed. However, the + * descriptor may possibly become invalid in other ways as well. + */ + public static boolean isSimpleFSIndexInputOpen(IndexInput is) + throws IOException + { + if (isSimpleFSIndexInput(is)) { + SimpleFSIndexInput fis = (SimpleFSIndexInput) is; + return fis.isFDValid(); + } else { + return false; + } + } + +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/automaton/AutomatonTestUtil.java 2011-11-01 21:10:39.308609906 -0400 @@ -0,0 +1,411 @@ +package org.apache.lucene.util.automaton; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.IdentityHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; + +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.UnicodeUtil; +import org.apache.lucene.util._TestUtil; + +public class AutomatonTestUtil { + /** Returns random string, including full unicode range. */ + public static String randomRegexp(Random r) { + while (true) { + String regexp = randomRegexpString(r); + // we will also generate some undefined unicode queries + if (!UnicodeUtil.validUTF16String(regexp)) + continue; + try { + new RegExp(regexp, RegExp.NONE); + return regexp; + } catch (Exception e) {} + } + } + + private static String randomRegexpString(Random r) { + final int end = r.nextInt(20); + if (end == 0) { + // allow 0 length + return ""; + } + final char[] buffer = new char[end]; + for (int i = 0; i < end; i++) { + int t = r.nextInt(15); + if (0 == t && i < end - 1) { + // Make a surrogate pair + // High surrogate + buffer[i++] = (char) _TestUtil.nextInt(r, 0xd800, 0xdbff); + // Low surrogate + buffer[i] = (char) _TestUtil.nextInt(r, 0xdc00, 0xdfff); + } + else if (t <= 1) buffer[i] = (char) r.nextInt(0x80); + else if (2 == t) buffer[i] = (char) _TestUtil.nextInt(r, 0x80, 0x800); + else if (3 == t) buffer[i] = (char) _TestUtil.nextInt(r, 0x800, 0xd7ff); + else if (4 == t) buffer[i] = (char) _TestUtil.nextInt(r, 0xe000, 0xffff); + else if (5 == t) buffer[i] = '.'; + else if (6 == t) buffer[i] = '?'; + else if (7 == t) buffer[i] = '*'; + else if (8 == t) buffer[i] = '+'; + else if (9 == t) buffer[i] = '('; + else if (10 == t) buffer[i] = ')'; + else if (11 == t) buffer[i] = '-'; + else if (12 == t) buffer[i] = '['; + else if (13 == t) buffer[i] = ']'; + else if (14 == t) buffer[i] = '|'; + } + return new String(buffer, 0, end); + } + + /** picks a random int code point, avoiding surrogates; + * throws IllegalArgumentException if this transition only + * accepts surrogates */ + private static int getRandomCodePoint(final Random r, final Transition t) { + final int code; + if (t.max < UnicodeUtil.UNI_SUR_HIGH_START || + t.min > UnicodeUtil.UNI_SUR_HIGH_END) { + // easy: entire range is before or after surrogates + code = t.min+r.nextInt(t.max-t.min+1); + } else if (t.min >= UnicodeUtil.UNI_SUR_HIGH_START) { + if (t.max > UnicodeUtil.UNI_SUR_LOW_END) { + // after surrogates + code = 1+UnicodeUtil.UNI_SUR_LOW_END+r.nextInt(t.max-UnicodeUtil.UNI_SUR_LOW_END); + } else { + throw new IllegalArgumentException("transition accepts only surrogates: " + t); + } + } else if (t.max <= UnicodeUtil.UNI_SUR_LOW_END) { + if (t.min < UnicodeUtil.UNI_SUR_HIGH_START) { + // before surrogates + code = t.min + r.nextInt(UnicodeUtil.UNI_SUR_HIGH_START - t.min); + } else { + throw new IllegalArgumentException("transition accepts only surrogates: " + t); + } + } else { + // range includes all surrogates + int gap1 = UnicodeUtil.UNI_SUR_HIGH_START - t.min; + int gap2 = t.max - UnicodeUtil.UNI_SUR_LOW_END; + int c = r.nextInt(gap1+gap2); + if (c < gap1) { + code = t.min + c; + } else { + code = UnicodeUtil.UNI_SUR_LOW_END + c - gap1 + 1; + } + } + + assert code >= t.min && code <= t.max && (code < UnicodeUtil.UNI_SUR_HIGH_START || code > UnicodeUtil.UNI_SUR_LOW_END): + "code=" + code + " min=" + t.min + " max=" + t.max; + return code; + } + + public static class RandomAcceptedStrings { + + private final Map leadsToAccept; + private final Automaton a; + + private static class ArrivingTransition { + final State from; + final Transition t; + public ArrivingTransition(State from, Transition t) { + this.from = from; + this.t = t; + } + } + + public RandomAcceptedStrings(Automaton a) { + this.a = a; + if (a.isSingleton()) { + leadsToAccept = null; + return; + } + + // must use IdentityHashmap because two Transitions w/ + // different start nodes can be considered the same + leadsToAccept = new IdentityHashMap(); + final Map> allArriving = new HashMap>(); + + final LinkedList q = new LinkedList(); + final Set seen = new HashSet(); + + // reverse map the transitions, so we can quickly look + // up all arriving transitions to a given state + for(State s: a.getNumberedStates()) { + for(int i=0;i tl = allArriving.get(t.to); + if (tl == null) { + tl = new ArrayList(); + allArriving.put(t.to, tl); + } + tl.add(new ArrivingTransition(s, t)); + } + if (s.accept) { + q.add(s); + seen.add(s); + } + } + + // Breadth-first search, from accept states, + // backwards: + while(!q.isEmpty()) { + final State s = q.removeFirst(); + List arriving = allArriving.get(s); + if (arriving != null) { + for(ArrivingTransition at : arriving) { + final State from = at.from; + if (!seen.contains(from)) { + q.add(from); + seen.add(from); + leadsToAccept.put(at.t, Boolean.TRUE); + } + } + } + } + } + + public int[] getRandomAcceptedString(Random r) { + + final List soFar = new ArrayList(); + if (a.isSingleton()) { + // accepts only one + final String s = a.singleton; + + int charUpto = 0; + while(charUpto < s.length()) { + final int cp = s.codePointAt(charUpto); + charUpto += Character.charCount(cp); + soFar.add(cp); + } + } else { + + State s = a.initial; + + while(true) { + + if (s.accept) { + if (s.numTransitions == 0) { + // stop now + break; + } else { + if (r.nextBoolean()) { + break; + } + } + } + + if (s.numTransitions == 0) { + throw new RuntimeException("this automaton has dead states"); + } + + boolean cheat = r.nextBoolean(); + + final Transition t; + if (cheat) { + // pick a transition that we know is the fastest + // path to an accept state + List toAccept = new ArrayList(); + for(int i=0;i initialset = new HashSet(); + initialset.add(a.initial); + determinizeSimple(a, initialset); + } + + /** + * Simple, original brics implementation of determinize() + * Determinizes the given automaton using the given set of initial states. + */ + public static void determinizeSimple(Automaton a, Set initialset) { + int[] points = a.getStartPoints(); + // subset construction + Map, Set> sets = new HashMap, Set>(); + LinkedList> worklist = new LinkedList>(); + Map, State> newstate = new HashMap, State>(); + sets.put(initialset, initialset); + worklist.add(initialset); + a.initial = new State(); + newstate.put(initialset, a.initial); + while (worklist.size() > 0) { + Set s = worklist.removeFirst(); + State r = newstate.get(s); + for (State q : s) + if (q.accept) { + r.accept = true; + break; + } + for (int n = 0; n < points.length; n++) { + Set p = new HashSet(); + for (State q : s) + for (Transition t : q.getTransitions()) + if (t.min <= points[n] && points[n] <= t.max) + p.add(t.to); + if (!sets.containsKey(p)) { + sets.put(p, p); + worklist.add(p); + newstate.put(p, new State()); + } + State q = newstate.get(p); + int min = points[n]; + int max; + if (n + 1 < points.length) + max = points[n + 1] - 1; + else + max = Character.MAX_CODE_POINT; + r.addTransition(new Transition(min, max, q)); + } + } + a.deterministic = true; + a.clearNumberedStates(); + a.removeDeadTransitions(); + } + + /** + * Returns true if the language of this automaton is finite. + *

+ * WARNING: this method is slow, it will blow up if the automaton is large. + * this is only used to test the correctness of our faster implementation. + */ + public static boolean isFiniteSlow(Automaton a) { + if (a.isSingleton()) return true; + return isFiniteSlow(a.initial, new HashSet()); + } + + /** + * Checks whether there is a loop containing s. (This is sufficient since + * there are never transitions to dead states.) + */ + // TODO: not great that this is recursive... in theory a + // large automata could exceed java's stack + private static boolean isFiniteSlow(State s, HashSet path) { + path.add(s); + for (Transition t : s.getTransitions()) + if (path.contains(t.to) || !isFiniteSlow(t.to, path)) return false; + path.remove(s); + return true; + } + + + /** + * Checks that an automaton has no detached states that are unreachable + * from the initial state. + */ + public static void assertNoDetachedStates(Automaton a) { + int numStates = a.getNumberOfStates(); + a.clearNumberedStates(); // force recomputation of cached numbered states + assert numStates == a.getNumberOfStates() : "automaton has " + (numStates - a.getNumberOfStates()) + " detached states"; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java 2011-11-01 21:10:39.308609906 -0400 @@ -0,0 +1,361 @@ +package org.apache.lucene.util.automaton; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.*; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.CharsRef; +import org.apache.lucene.util.UnicodeUtil; + +/** + * Builds a minimal deterministic automaton that accepts a set of strings. The + * algorithm requires sorted input data, but is very fast (nearly linear with + * the input size). + */ +public final class DaciukMihovAutomatonBuilder { + /** + * DFSA state with char labels on transitions. + */ + final static class State { + + /** An empty set of labels. */ + private final static int[] NO_LABELS = new int[0]; + + /** An empty set of states. */ + private final static State[] NO_STATES = new State[0]; + + /** + * Labels of outgoing transitions. Indexed identically to {@link #states}. + * Labels must be sorted lexicographically. + */ + int[] labels = NO_LABELS; + + /** + * States reachable from outgoing transitions. Indexed identically to + * {@link #labels}. + */ + State[] states = NO_STATES; + + /** + * true if this state corresponds to the end of at least one + * input sequence. + */ + boolean is_final; + + /** + * Returns the target state of a transition leaving this state and labeled + * with label. If no such transition exists, returns + * null. + */ + public State getState(int label) { + final int index = Arrays.binarySearch(labels, label); + return index >= 0 ? states[index] : null; + } + + /** + * Returns an array of outgoing transition labels. The array is sorted in + * lexicographic order and indexes correspond to states returned from + * {@link #getStates()}. + */ + public int[] getTransitionLabels() { + return this.labels; + } + + /** + * Returns an array of outgoing transitions from this state. The returned + * array must not be changed. + */ + public State[] getStates() { + return this.states; + } + + /** + * Two states are equal if: + *

    + *
  • they have an identical number of outgoing transitions, labeled with + * the same labels
    • + *
  • corresponding outgoing transitions lead to the same states (to states + * with an identical right-language). + *
+ */ + @Override + public boolean equals(Object obj) { + final State other = (State) obj; + return is_final == other.is_final + && Arrays.equals(this.labels, other.labels) + && referenceEquals(this.states, other.states); + } + + /** + * Return true if this state has any children (outgoing + * transitions). + */ + public boolean hasChildren() { + return labels.length > 0; + } + + /** + * Is this state a final state in the automaton? + */ + public boolean isFinal() { + return is_final; + } + + /** + * Compute the hash code of the current status of this state. + */ + @Override + public int hashCode() { + int hash = is_final ? 1 : 0; + + hash ^= hash * 31 + this.labels.length; + for (int c : this.labels) + hash ^= hash * 31 + c; + + /* + * Compare the right-language of this state using reference-identity of + * outgoing states. This is possible because states are interned (stored + * in registry) and traversed in post-order, so any outgoing transitions + * are already interned. + */ + for (State s : this.states) { + hash ^= System.identityHashCode(s); + } + + return hash; + } + + /** + * Create a new outgoing transition labeled label and return + * the newly created target state for this transition. + */ + State newState(int label) { + assert Arrays.binarySearch(labels, label) < 0 : "State already has transition labeled: " + + label; + + labels = copyOf(labels, labels.length + 1); + states = copyOf(states, states.length + 1); + + labels[labels.length - 1] = label; + return states[states.length - 1] = new State(); + } + + /** + * Return the most recent transitions's target state. + */ + State lastChild() { + assert hasChildren() : "No outgoing transitions."; + return states[states.length - 1]; + } + + /** + * Return the associated state if the most recent transition is labeled with + * label. + */ + State lastChild(int label) { + final int index = labels.length - 1; + State s = null; + if (index >= 0 && labels[index] == label) { + s = states[index]; + } + assert s == getState(label); + return s; + } + + /** + * Replace the last added outgoing transition's target state with the given + * state. + */ + void replaceLastChild(State state) { + assert hasChildren() : "No outgoing transitions."; + states[states.length - 1] = state; + } + + /** + * JDK1.5-replacement of {@link Arrays#copyOf(int[], int)} + */ + private static int[] copyOf(int[] original, int newLength) { + int[] copy = new int[newLength]; + System.arraycopy(original, 0, copy, 0, + Math.min(original.length, newLength)); + return copy; + } + + /** + * JDK1.5-replacement of {@link Arrays#copyOf(char[], int)} + */ + public static State[] copyOf(State[] original, int newLength) { + State[] copy = new State[newLength]; + System.arraycopy(original, 0, copy, 0, + Math.min(original.length, newLength)); + return copy; + } + + /** + * Compare two lists of objects for reference-equality. + */ + private static boolean referenceEquals(Object[] a1, Object[] a2) { + if (a1.length != a2.length) return false; + + for (int i = 0; i < a1.length; i++) + if (a1[i] != a2[i]) return false; + + return true; + } + } + + /** + * "register" for state interning. + */ + private HashMap register = new HashMap(); + + /** + * Root automaton state. + */ + private State root = new State(); + + /** + * Previous sequence added to the automaton in {@link #add(CharSequence)}. + */ + private CharsRef previous; + + private static final Comparator comparator = CharsRef.getUTF16SortedAsUTF8Comparator(); + + /** + * Add another character sequence to this automaton. The sequence must be + * lexicographically larger or equal compared to any previous sequences added + * to this automaton (the input must be sorted). + */ + public void add(CharsRef current) { + assert register != null : "Automaton already built."; + assert previous == null + || comparator.compare(previous, current) <= 0 : "Input must be sorted: " + + previous + " >= " + current; + assert setPrevious(current); + + // Descend in the automaton (find matching prefix). + int pos = 0, max = current.length(); + State next, state = root; + while (pos < max && (next = state.lastChild(Character.codePointAt(current, pos))) != null) { + state = next; + // todo, optimize me + pos += Character.charCount(Character.codePointAt(current, pos)); + } + + if (state.hasChildren()) replaceOrRegister(state); + + addSuffix(state, current, pos); + } + + /** + * Finalize the automaton and return the root state. No more strings can be + * added to the builder after this call. + * + * @return Root automaton state. + */ + public State complete() { + if (this.register == null) throw new IllegalStateException(); + + if (root.hasChildren()) replaceOrRegister(root); + + register = null; + return root; + } + + /** + * Internal recursive traversal for conversion. + */ + private static org.apache.lucene.util.automaton.State convert(State s, + IdentityHashMap visited) { + org.apache.lucene.util.automaton.State converted = visited.get(s); + if (converted != null) return converted; + + converted = new org.apache.lucene.util.automaton.State(); + converted.setAccept(s.is_final); + + visited.put(s, converted); + int i = 0; + int[] labels = s.labels; + for (DaciukMihovAutomatonBuilder.State target : s.states) { + converted.addTransition(new Transition(labels[i++], convert(target, + visited))); + } + + return converted; + } + + /** + * Build a minimal, deterministic automaton from a sorted list of strings. + */ + public static Automaton build(Collection input) { + final DaciukMihovAutomatonBuilder builder = new DaciukMihovAutomatonBuilder(); + + CharsRef scratch = new CharsRef(); + for (BytesRef b : input) { + UnicodeUtil.UTF8toUTF16(b, scratch); + builder.add(scratch); + } + + Automaton a = new Automaton(); + a.initial = convert(builder.complete(), new IdentityHashMap()); + a.deterministic = true; + return a; + } + + /** + * Copy current into an internal buffer. + */ + private boolean setPrevious(CharsRef current) { + // don't need to copy, once we fix https://issues.apache.org/jira/browse/LUCENE-3277 + // still, called only from assert + previous = new CharsRef(current); + return true; + } + + /** + * Replace last child of state with an already registered state + * or register the last child state. + */ + private void replaceOrRegister(State state) { + final State child = state.lastChild(); + + if (child.hasChildren()) replaceOrRegister(child); + + final State registered = register.get(child); + if (registered != null) { + state.replaceLastChild(registered); + } else { + register.put(child, child); + } + } + + /** + * Add a suffix of current starting at fromIndex + * (inclusive) to state state. + */ + private void addSuffix(State state, CharSequence current, int fromIndex) { + final int len = current.length(); + while (fromIndex < len) { + int cp = Character.codePointAt(current, fromIndex); + state = state.newState(cp); + fromIndex += Character.charCount(cp); + } + state.is_final = true; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/LineFileDocs.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/LineFileDocs.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/LineFileDocs.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/LineFileDocs.java 2011-11-01 21:10:39.320609906 -0400 @@ -0,0 +1,186 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.Closeable; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.io.InputStream; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.zip.GZIPInputStream; +import java.util.Random; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.document.StringField; +import org.apache.lucene.document.TextField; + +/** Minimal port of contrib/benchmark's LneDocSource + + * DocMaker, so tests can enum docs from a line file created + * by contrib/benchmark's WriteLineDoc task */ +public class LineFileDocs implements Closeable { + + private BufferedReader reader; + private final static int BUFFER_SIZE = 1 << 16; // 64K + private final AtomicInteger id = new AtomicInteger(); + private final String path; + + /** If forever is true, we rewind the file at EOF (repeat + * the docs over and over) */ + public LineFileDocs(Random random, String path) throws IOException { + this.path = path; + open(random); + } + + public LineFileDocs(Random random) throws IOException { + this(random, LuceneTestCase.TEST_LINE_DOCS_FILE); + } + + public synchronized void close() throws IOException { + if (reader != null) { + reader.close(); + reader = null; + } + } + + private synchronized void open(Random random) throws IOException { + InputStream is = getClass().getResourceAsStream(path); + if (is == null) { + // if its not in classpath, we load it as absolute filesystem path (e.g. Hudson's home dir) + is = new FileInputStream(path); + } + File file = new File(path); + long size; + if (file.exists()) { + size = file.length(); + } else { + size = is.available(); + } + if (path.endsWith(".gz")) { + is = new GZIPInputStream(is); + // guestimate: + size *= 2.8; + } + + reader = new BufferedReader(new InputStreamReader(is, "UTF-8"), BUFFER_SIZE); + + // Override sizes for currently "known" line files: + if (path.equals("europarl.lines.txt.gz")) { + size = 15129506L; + } else if (path.equals("/home/hudson/lucene-data/enwiki.random.lines.txt.gz")) { + size = 3038178822L; + } + + // Randomly seek to starting point: + if (random != null && size > 3) { + final long seekTo = (random.nextLong()&Long.MAX_VALUE) % (size/3); + if (LuceneTestCase.VERBOSE) { + System.out.println("TEST: LineFileDocs: seek to fp=" + seekTo + " on open"); + } + reader.skip(seekTo); + reader.readLine(); + } + } + + public synchronized void reset(Random random) throws IOException { + close(); + open(random); + id.set(0); + } + + private final static char SEP = '\t'; + + private static final class DocState { + final Document doc; + final Field titleTokenized; + final Field title; + final Field body; + final Field id; + final Field date; + + public DocState() { + doc = new Document(); + + title = new StringField("title", ""); + doc.add(title); + + FieldType ft = new FieldType(TextField.TYPE_STORED); + ft.setStoreTermVectors(true); + ft.setStoreTermVectorOffsets(true); + ft.setStoreTermVectorPositions(true); + + titleTokenized = new Field("titleTokenized", "", ft); + doc.add(titleTokenized); + + body = new Field("body", "", ft); + doc.add(body); + + id = new Field("docid", "", StringField.TYPE_STORED); + doc.add(id); + + date = new Field("date", "", StringField.TYPE_STORED); + doc.add(date); + } + } + + private final ThreadLocal threadDocs = new ThreadLocal(); + + /** Note: Document instance is re-used per-thread */ + public Document nextDoc() throws IOException { + String line; + synchronized(this) { + line = reader.readLine(); + if (line == null) { + // Always rewind at end: + if (LuceneTestCase.VERBOSE) { + System.out.println("TEST: LineFileDocs: now rewind file..."); + } + close(); + open(null); + line = reader.readLine(); + } + } + + DocState docState = threadDocs.get(); + if (docState == null) { + docState = new DocState(); + threadDocs.set(docState); + } + + int spot = line.indexOf(SEP); + if (spot == -1) { + throw new RuntimeException("line: [" + line + "] is in an invalid format !"); + } + int spot2 = line.indexOf(SEP, 1 + spot); + if (spot2 == -1) { + throw new RuntimeException("line: [" + line + "] is in an invalid format !"); + } + + docState.body.setValue(line.substring(1+spot2, line.length())); + final String title = line.substring(0, spot); + docState.title.setValue(title); + docState.titleTokenized.setValue(title); + docState.date.setValue(line.substring(1+spot, spot2)); + docState.id.setValue(Integer.toString(id.getAndIncrement())); + return docState.doc; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitDividingSelector.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitDividingSelector.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitDividingSelector.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitDividingSelector.java 2011-11-01 21:10:39.320609906 -0400 @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +package org.apache.lucene.util; +import java.io.File; + +import org.apache.tools.ant.BuildException; +import org.apache.tools.ant.types.Parameter; +import org.apache.tools.ant.types.selectors.BaseExtendSelector; + +/** Divides filesets into equal groups */ +public class LuceneJUnitDividingSelector extends BaseExtendSelector { + private int counter; + /** Number of total parts to split. */ + private int divisor; + /** Current part to accept. */ + private int part; + + @Override + public void setParameters(Parameter[] pParameters) { + super.setParameters(pParameters); + for (int j = 0; j < pParameters.length; j++) { + Parameter p = pParameters[j]; + if ("divisor".equalsIgnoreCase(p.getName())) { + divisor = Integer.parseInt(p.getValue()); + } + else if ("part".equalsIgnoreCase(p.getName())) { + part = Integer.parseInt(p.getValue()); + } + else { + throw new BuildException("unknown " + p.getName()); + } + } + } + + @Override + public void verifySettings() { + super.verifySettings(); + if (divisor <= 0 || part <= 0) { + throw new BuildException("part or divisor not set"); + } + if (part > divisor) { + throw new BuildException("part must be <= divisor"); + } + } + + @Override + public boolean isSelected(File dir, String name, File path) { + counter = counter % divisor + 1; + return counter == part; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitResultFormatter.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitResultFormatter.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitResultFormatter.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/LuceneJUnitResultFormatter.java 2011-11-01 21:10:39.320609906 -0400 @@ -0,0 +1,293 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.lucene.util; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.OutputStream; +import java.text.NumberFormat; +import java.util.logging.LogManager; + +import junit.framework.AssertionFailedError; +import junit.framework.Test; + +import org.apache.lucene.store.LockReleaseFailedException; +import org.apache.lucene.store.NativeFSLockFactory; +import org.apache.tools.ant.taskdefs.optional.junit.JUnitResultFormatter; +import org.apache.tools.ant.taskdefs.optional.junit.JUnitTest; +import org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner; +import org.apache.tools.ant.util.FileUtils; +import org.apache.tools.ant.util.StringUtils; +import org.junit.Ignore; + +/** + * Just like BriefJUnitResultFormatter "brief" bundled with ant, + * except all formatted text is buffered until the test suite is finished. + * At this point, the output is written at once in synchronized fashion. + * This way tests can run in parallel without interleaving output. + */ +public class LuceneJUnitResultFormatter implements JUnitResultFormatter { + private static final double ONE_SECOND = 1000.0; + + private static final NativeFSLockFactory lockFactory; + + /** Where to write the log to. */ + private OutputStream out; + + /** Formatter for timings. */ + private NumberFormat numberFormat = NumberFormat.getInstance(); + + /** Output suite has written to System.out */ + private String systemOutput = null; + + /** Output suite has written to System.err */ + private String systemError = null; + + /** Buffer output until the end of the test */ + private ByteArrayOutputStream sb; // use a BOS for our mostly ascii-output + + private static final org.apache.lucene.store.Lock lock; + + static { + File lockDir = new File(System.getProperty("java.io.tmpdir"), + "lucene_junit_lock"); + lockDir.mkdirs(); + if (!lockDir.exists()) { + throw new RuntimeException("Could not make Lock directory:" + lockDir); + } + try { + lockFactory = new NativeFSLockFactory(lockDir); + lock = lockFactory.makeLock("junit_lock"); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** Constructor for LuceneJUnitResultFormatter. */ + public LuceneJUnitResultFormatter() { + } + + /** + * Sets the stream the formatter is supposed to write its results to. + * @param out the output stream to write to + */ + public void setOutput(OutputStream out) { + this.out = out; + } + + /** + * @see JUnitResultFormatter#setSystemOutput(String) + */ + /** {@inheritDoc}. */ + public void setSystemOutput(String out) { + systemOutput = out; + } + + /** + * @see JUnitResultFormatter#setSystemError(String) + */ + /** {@inheritDoc}. */ + public void setSystemError(String err) { + systemError = err; + } + + + /** + * The whole testsuite started. + * @param suite the test suite + */ + public synchronized void startTestSuite(JUnitTest suite) { + if (out == null) { + return; // Quick return - no output do nothing. + } + sb = new ByteArrayOutputStream(); // don't reuse, so its gc'ed + try { + LogManager.getLogManager().readConfiguration(); + } catch (Exception e) {} + append("Testsuite: "); + append(suite.getName()); + append(StringUtils.LINE_SEP); + } + + /** + * The whole testsuite ended. + * @param suite the test suite + */ + public synchronized void endTestSuite(JUnitTest suite) { + append("Tests run: "); + append(suite.runCount()); + append(", Failures: "); + append(suite.failureCount()); + append(", Errors: "); + append(suite.errorCount()); + append(", Time elapsed: "); + append(numberFormat.format(suite.getRunTime() / ONE_SECOND)); + append(" sec"); + append(StringUtils.LINE_SEP); + append(StringUtils.LINE_SEP); + + // append the err and output streams to the log + if (systemOutput != null && systemOutput.length() > 0) { + append("------------- Standard Output ---------------") + .append(StringUtils.LINE_SEP) + .append(systemOutput) + .append("------------- ---------------- ---------------") + .append(StringUtils.LINE_SEP); + } + + // HACK: junit gives us no way to do this in LuceneTestCase + try { + Class clazz = Class.forName(suite.getName()); + Ignore ignore = clazz.getAnnotation(Ignore.class); + if (ignore != null) { + if (systemError == null) systemError = ""; + systemError += "NOTE: Ignoring test class '" + clazz.getSimpleName() + "': " + + ignore.value() + StringUtils.LINE_SEP; + } + } catch (ClassNotFoundException e) { /* no problem */ } + // END HACK + + if (systemError != null && systemError.length() > 0) { + append("------------- Standard Error -----------------") + .append(StringUtils.LINE_SEP) + .append(systemError) + .append("------------- ---------------- ---------------") + .append(StringUtils.LINE_SEP); + } + + if (out != null) { + try { + lock.obtain(5000); + try { + sb.writeTo(out); + out.flush(); + } finally { + try { + lock.release(); + } catch(LockReleaseFailedException e) { + // well lets pretend its released anyway + } + } + } catch (IOException e) { + throw new RuntimeException("unable to write results", e); + } finally { + if (out != System.out && out != System.err) { + FileUtils.close(out); + } + } + } + } + + /** + * A test started. + * @param test a test + */ + public void startTest(Test test) { + } + + /** + * A test ended. + * @param test a test + */ + public void endTest(Test test) { + } + + /** + * Interface TestListener for JUnit <= 3.4. + * + *

A Test failed. + * @param test a test + * @param t the exception thrown by the test + */ + public void addFailure(Test test, Throwable t) { + formatError("\tFAILED", test, t); + } + + /** + * Interface TestListener for JUnit > 3.4. + * + *

A Test failed. + * @param test a test + * @param t the assertion failed by the test + */ + public void addFailure(Test test, AssertionFailedError t) { + addFailure(test, (Throwable) t); + } + + /** + * A test caused an error. + * @param test a test + * @param error the error thrown by the test + */ + public void addError(Test test, Throwable error) { + formatError("\tCaused an ERROR", test, error); + } + + /** + * Format the test for printing.. + * @param test a test + * @return the formatted testname + */ + protected String formatTest(Test test) { + if (test == null) { + return "Null Test: "; + } else { + return "Testcase: " + test.toString() + ":"; + } + } + + /** + * Format an error and print it. + * @param type the type of error + * @param test the test that failed + * @param error the exception that the test threw + */ + protected synchronized void formatError(String type, Test test, + Throwable error) { + if (test != null) { + endTest(test); + } + + append(formatTest(test) + type); + append(StringUtils.LINE_SEP); + append(error.getMessage()); + append(StringUtils.LINE_SEP); + String strace = JUnitTestRunner.getFilteredTrace(error); + append(strace); + append(StringUtils.LINE_SEP); + append(StringUtils.LINE_SEP); + } + + public LuceneJUnitResultFormatter append(String s) { + if (s == null) + s = "(null)"; + try { + sb.write(s.getBytes()); // intentionally use default charset, its a console. + } catch (IOException e) { + throw new RuntimeException(e); + } + return this; + } + + public LuceneJUnitResultFormatter append(long l) { + return append(Long.toString(l)); + } +} + diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java 2011-11-02 11:51:05.900582691 -0400 @@ -0,0 +1,1343 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.lang.reflect.Constructor; +import java.util.*; +import java.util.Map.Entry; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.FieldType; +import org.apache.lucene.index.*; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xCodec; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockPostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; +import org.apache.lucene.index.codecs.mockrandom.MockRandomPostingsFormat; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; +import org.apache.lucene.index.codecs.preflexrw.PreFlexRWPostingsFormat; +import org.apache.lucene.index.codecs.pulsing.PulsingPostingsFormat; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.FieldCache; +import org.apache.lucene.search.FieldCache.CacheEntry; +import org.apache.lucene.search.AssertingIndexSearcher; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.RandomSimilarityProvider; +import org.apache.lucene.search.similarities.SimilarityProvider; +import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; +import org.apache.lucene.store.FlushInfo; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.MergeInfo; +import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.store.MockDirectoryWrapper.Throttling; +import org.apache.lucene.util.FieldCacheSanityChecker.Insanity; +import org.junit.*; +import org.junit.rules.MethodRule; +import org.junit.rules.TestWatchman; +import org.junit.runner.RunWith; +import org.junit.runners.model.FrameworkMethod; +import org.junit.runners.model.Statement; + +/** + * Base class for all Lucene unit tests, Junit3 or Junit4 variant. + *

+ *

+ *

+ * If you + * override either setUp() or + * tearDown() in your unit test, make sure you + * call super.setUp() and + * super.tearDown() + *

+ * + * @After - replaces setup + * @Before - replaces teardown + * @Test - any public method with this annotation is a test case, regardless + * of its name + *

+ *

+ * See Junit4 documentation for a complete list of features. + *

+ * Import from org.junit rather than junit.framework. + *

+ * You should be able to use this class anywhere you used LuceneTestCase + * if you annotate your derived class correctly with the annotations above + * @see #assertSaneFieldCaches(String) + */ + +@RunWith(LuceneTestCaseRunner.class) +public abstract class LuceneTestCase extends Assert { + + /** + * true iff tests are run in verbose mode. Note: if it is false, tests are not + * expected to print any messages. + */ + public static final boolean VERBOSE = Boolean.getBoolean("tests.verbose"); + + /** Use this constant when creating Analyzers and any other version-dependent stuff. + *

NOTE: Change this when development starts for new Lucene version: + */ + public static final Version TEST_VERSION_CURRENT = Version.LUCENE_40; + + /** + * If this is set, it is the only method that should run. + */ + static final String TEST_METHOD; + + /** Create indexes in this directory, optimally use a subdir, named after the test */ + public static final File TEMP_DIR; + static { + String method = System.getProperty("testmethod", "").trim(); + TEST_METHOD = method.length() == 0 ? null : method; + String s = System.getProperty("tempDir", System.getProperty("java.io.tmpdir")); + if (s == null) + throw new RuntimeException("To run tests, you need to define system property 'tempDir' or 'java.io.tmpdir'."); + TEMP_DIR = new File(s); + TEMP_DIR.mkdirs(); + } + + /** set of directories we created, in afterclass we try to clean these up */ + private static final Map tempDirs = Collections.synchronizedMap(new HashMap()); + + // by default we randomly pick a different codec for + // each test case (non-J4 tests) and each test class (J4 + // tests) + /** Gets the postingsFormat to run tests with. */ + public static final String TEST_POSTINGSFORMAT = System.getProperty("tests.postingsformat", "random"); + /** Gets the locale to run tests with */ + public static final String TEST_LOCALE = System.getProperty("tests.locale", "random"); + /** Gets the timezone to run tests with */ + public static final String TEST_TIMEZONE = System.getProperty("tests.timezone", "random"); + /** Gets the directory to run tests with */ + public static final String TEST_DIRECTORY = System.getProperty("tests.directory", "random"); + /** Get the number of times to run tests */ + public static final int TEST_ITER = Integer.parseInt(System.getProperty("tests.iter", "1")); + /** Get the minimum number of times to run tests until a failure happens */ + public static final int TEST_ITER_MIN = Integer.parseInt(System.getProperty("tests.iter.min", Integer.toString(TEST_ITER))); + /** Get the random seed for tests */ + public static final String TEST_SEED = System.getProperty("tests.seed", "random"); + /** whether or not nightly tests should run */ + public static final boolean TEST_NIGHTLY = Boolean.parseBoolean(System.getProperty("tests.nightly", "false")); + /** the line file used by LineFileDocs */ + public static final String TEST_LINE_DOCS_FILE = System.getProperty("tests.linedocsfile", "europarl.lines.txt.gz"); + /** whether or not to clean threads between test invocations: "false", "perMethod", "perClass" */ + public static final String TEST_CLEAN_THREADS = System.getProperty("tests.cleanthreads", "perClass"); + /** whether or not to clean threads between test invocations: "false", "perMethod", "perClass" */ + public static final Throttling TEST_THROTTLING = TEST_NIGHTLY ? Throttling.SOMETIMES : Throttling.NEVER; + + private static final Pattern codecWithParam = Pattern.compile("(.*)\\(\\s*(\\d+)\\s*\\)"); + + /** + * A random multiplier which you should use when writing random tests: + * multiply it by the number of iterations + */ + public static final int RANDOM_MULTIPLIER = Integer.parseInt(System.getProperty("tests.multiplier", "1")); + + /** @lucene.internal */ + public static boolean PREFLEX_IMPERSONATION_IS_ACTIVE; + + private int savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount(); + + private volatile Thread.UncaughtExceptionHandler savedUncaughtExceptionHandler = null; + + /** Used to track if setUp and tearDown are called correctly from subclasses */ + private static State state = State.INITIAL; + + private static enum State { + INITIAL, // no tests ran yet + SETUP, // test has called setUp() + RANTEST, // test is running + TEARDOWN // test has called tearDown() + } + + /** + * Some tests expect the directory to contain a single segment, and want to do tests on that segment's reader. + * This is an utility method to help them. + */ + public static SegmentReader getOnlySegmentReader(IndexReader reader) { + if (reader instanceof SegmentReader) + return (SegmentReader) reader; + + IndexReader[] subReaders = reader.getSequentialSubReaders(); + if (subReaders.length != 1) + throw new IllegalArgumentException(reader + " has " + subReaders.length + " segments instead of exactly one"); + + return (SegmentReader) subReaders[0]; + } + + private static class UncaughtExceptionEntry { + public final Thread thread; + public final Throwable exception; + + public UncaughtExceptionEntry(Thread thread, Throwable exception) { + this.thread = thread; + this.exception = exception; + } + } + private List uncaughtExceptions = Collections.synchronizedList(new ArrayList()); + + // default codec + private static Codec savedCodec; + + private static SimilarityProvider similarityProvider; + + private static Locale locale; + private static Locale savedLocale; + private static TimeZone timeZone; + private static TimeZone savedTimeZone; + + protected static Map stores; + + /** @deprecated (4.0) until we fix no-fork problems in solr tests */ + @Deprecated + static List testClassesRun = new ArrayList(); + + private static void initRandom() { + assert !random.initialized; + staticSeed = "random".equals(TEST_SEED) ? seedRand.nextLong() : ThreeLongs.fromString(TEST_SEED).l1; + random.setSeed(staticSeed); + random.initialized = true; + } + + @Deprecated + private static boolean icuTested = false; + + @BeforeClass + public static void beforeClassLuceneTestCaseJ4() { + initRandom(); + state = State.INITIAL; + tempDirs.clear(); + stores = Collections.synchronizedMap(new IdentityHashMap()); + + // enable this by default, for IDE consistency with ant tests (as its the default from ant) + // TODO: really should be in solr base classes, but some extend LTC directly. + // we do this in beforeClass, because some tests currently disable it + if (System.getProperty("solr.directoryFactory") == null) { + System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockDirectoryFactory"); + } + + // if verbose: print some debugging stuff about which codecs are loaded + if (VERBOSE) { + Set codecs = Codec.availableCodecs(); + for (String codec : codecs) { + System.out.println("Loaded codec: '" + codec + "': " + Codec.forName(codec).getClass().getName()); + } + + Set postingsFormats = PostingsFormat.availablePostingsFormats(); + for (String postingsFormat : postingsFormats) { + System.out.println("Loaded postingsFormat: '" + postingsFormat + "': " + PostingsFormat.forName(postingsFormat).getClass().getName()); + } + } + + PREFLEX_IMPERSONATION_IS_ACTIVE = false; + savedCodec = Codec.getDefault(); + final Codec codec; + if ("Lucene3x".equals(TEST_POSTINGSFORMAT) || ("random".equals(TEST_POSTINGSFORMAT) && random.nextInt(4) == 0)) { // preflex-only setup + codec = new PreFlexRWCodec(); + PREFLEX_IMPERSONATION_IS_ACTIVE = true; + } else if ("random".equals(TEST_POSTINGSFORMAT)) { + codec = new RandomCodec(random, useNoMemoryExpensiveCodec); + } else { + codec = new Lucene40Codec() { + private final PostingsFormat format = PostingsFormat.forName(TEST_POSTINGSFORMAT); + + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return format; + } + + @Override + public String toString() { + return super.toString() + ": " + format.toString(); + } + }; + } + + Codec.setDefault(codec); + + savedLocale = Locale.getDefault(); + + // START hack to init ICU safely before we randomize locales. + // ICU fails during classloading when a special Java7-only locale is the default + // see: http://bugs.icu-project.org/trac/ticket/8734 + if (!icuTested) { + icuTested = true; + try { + Locale.setDefault(Locale.US); + Class.forName("com.ibm.icu.util.ULocale"); + } catch (ClassNotFoundException cnfe) { + // ignore if no ICU is in classpath + } + } + // END hack + + locale = TEST_LOCALE.equals("random") ? randomLocale(random) : localeForName(TEST_LOCALE); + Locale.setDefault(locale); + savedTimeZone = TimeZone.getDefault(); + timeZone = TEST_TIMEZONE.equals("random") ? randomTimeZone(random) : TimeZone.getTimeZone(TEST_TIMEZONE); + TimeZone.setDefault(timeZone); + similarityProvider = new RandomSimilarityProvider(random); + testsFailed = false; + } + + @AfterClass + public static void afterClassLuceneTestCaseJ4() { + State oldState = state; // capture test execution state + state = State.INITIAL; // set the state for subsequent tests + + Throwable problem = null; + try { + if (!testsFailed) { + assertTrue("ensure your setUp() calls super.setUp() and your tearDown() calls super.tearDown()!!!", + oldState == State.INITIAL || oldState == State.TEARDOWN); + } + } catch (Throwable t) { + if (problem == null) problem = t; + } + + if (! "false".equals(TEST_CLEAN_THREADS)) { + int rogueThreads = threadCleanup("test class"); + if (rogueThreads > 0) { + // TODO: fail here once the leaks are fixed. + System.err.println("RESOURCE LEAK: test class left " + rogueThreads + " thread(s) running"); + } + } + + String codecDescription = Codec.getDefault().toString(); + Codec.setDefault(savedCodec); + Locale.setDefault(savedLocale); + TimeZone.setDefault(savedTimeZone); + System.clearProperty("solr.solr.home"); + System.clearProperty("solr.data.dir"); + + try { + // now look for unclosed resources + if (!testsFailed) { + checkResourcesAfterClass(); + } + } catch (Throwable t) { + if (problem == null) problem = t; + } + + stores = null; + + try { + // clear out any temp directories if we can + if (!testsFailed) { + clearTempDirectoriesAfterClass(); + } + } catch (Throwable t) { + if (problem == null) problem = t; + } + + // if we had afterClass failures, get some debugging information + if (problem != null) { + reportPartialFailureInfo(); + } + + // if verbose or tests failed, report some information back + if (VERBOSE || testsFailed || problem != null) { + printDebuggingInformation(codecDescription); + } + + // reset seed + random.setSeed(0L); + random.initialized = false; + + if (problem != null) { + throw new RuntimeException(problem); + } + } + + /** print some useful debugging information about the environment */ + private static void printDebuggingInformation(String codecDescription) { + System.err.println("NOTE: test params are: codec=" + codecDescription + + ", sim=" + similarityProvider + + ", locale=" + locale + + ", timezone=" + (timeZone == null ? "(null)" : timeZone.getID())); + System.err.println("NOTE: all tests run in this JVM:"); + System.err.println(Arrays.toString(testClassesRun.toArray())); + System.err.println("NOTE: " + System.getProperty("os.name") + " " + + System.getProperty("os.version") + " " + + System.getProperty("os.arch") + "/" + + System.getProperty("java.vendor") + " " + + System.getProperty("java.version") + " " + + (Constants.JRE_IS_64BIT ? "(64-bit)" : "(32-bit)") + "/" + + "cpus=" + Runtime.getRuntime().availableProcessors() + "," + + "threads=" + Thread.activeCount() + "," + + "free=" + Runtime.getRuntime().freeMemory() + "," + + "total=" + Runtime.getRuntime().totalMemory()); + } + + /** check that directories and their resources were closed */ + private static void checkResourcesAfterClass() { + for (MockDirectoryWrapper d : stores.keySet()) { + if (d.isOpen()) { + StackTraceElement elements[] = stores.get(d); + // Look for the first class that is not LuceneTestCase that requested + // a Directory. The first two items are of Thread's, so skipping over + // them. + StackTraceElement element = null; + for (int i = 2; i < elements.length; i++) { + StackTraceElement ste = elements[i]; + if (ste.getClassName().indexOf("LuceneTestCase") == -1) { + element = ste; + break; + } + } + fail("directory of test was not closed, opened from: " + element); + } + } + } + + /** clear temp directories: this will fail if its not successful */ + private static void clearTempDirectoriesAfterClass() { + for (Entry entry : tempDirs.entrySet()) { + try { + _TestUtil.rmDir(entry.getKey()); + } catch (IOException e) { + e.printStackTrace(); + System.err.println("path " + entry.getKey() + " allocated from"); + // first two STE's are Java's + StackTraceElement[] elements = entry.getValue(); + for (int i = 2; i < elements.length; i++) { + StackTraceElement ste = elements[i]; + // print only our code's stack information + if (ste.getClassName().indexOf("org.apache.lucene") == -1) break; + System.err.println("\t" + ste); + } + fail("could not remove temp dir: " + entry.getKey()); + } + } + } + + protected static boolean testsFailed; /* true if any tests failed */ + + // This is how we get control when errors occur. + // Think of this as start/end/success/failed + // events. + @Rule + public final TestWatchman intercept = new TestWatchman() { + + @Override + public void failed(Throwable e, FrameworkMethod method) { + // org.junit.internal.AssumptionViolatedException in older releases + // org.junit.Assume.AssumptionViolatedException in recent ones + if (e.getClass().getName().endsWith("AssumptionViolatedException")) { + if (e.getCause() instanceof _TestIgnoredException) + e = e.getCause(); + System.err.print("NOTE: Assume failed in '" + method.getName() + "' (ignored):"); + if (VERBOSE) { + System.err.println(); + e.printStackTrace(System.err); + } else { + System.err.print(" "); + System.err.println(e.getMessage()); + } + } else { + testsFailed = true; + reportAdditionalFailureInfo(); + } + super.failed(e, method); + } + + @Override + public void starting(FrameworkMethod method) { + // set current method name for logging + LuceneTestCase.this.name = method.getName(); + State s = state; // capture test execution state + state = State.RANTEST; // set the state for subsequent tests + if (!testsFailed) { + assertTrue("ensure your setUp() calls super.setUp()!!!", s == State.SETUP); + } + super.starting(method); + } + }; + + /** + * The thread executing the current test case. + * @see #isTestThread() + */ + volatile Thread testCaseThread; + + /** @see #testCaseThread */ + @Rule + public final MethodRule setTestThread = new MethodRule() { + public Statement apply(final Statement s, FrameworkMethod fm, Object target) { + return new Statement() { + public void evaluate() throws Throwable { + try { + LuceneTestCase.this.testCaseThread = Thread.currentThread(); + s.evaluate(); + } finally { + LuceneTestCase.this.testCaseThread = null; + } + } + }; + } + }; + + @Before + public void setUp() throws Exception { + seed = "random".equals(TEST_SEED) ? seedRand.nextLong() : ThreeLongs.fromString(TEST_SEED).l2; + random.setSeed(seed); + State s = state; // capture test execution state + state = State.SETUP; // set the state for subsequent tests + + savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler(); + Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + public void uncaughtException(Thread t, Throwable e) { + testsFailed = true; + uncaughtExceptions.add(new UncaughtExceptionEntry(t, e)); + if (savedUncaughtExceptionHandler != null) + savedUncaughtExceptionHandler.uncaughtException(t, e); + } + }); + + savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount(); + + if (!testsFailed) { + assertTrue("ensure your tearDown() calls super.tearDown()!!!", (s == State.INITIAL || s == State.TEARDOWN)); + } + + if (useNoMemoryExpensiveCodec) { + String defFormat = _TestUtil.getPostingsFormat("thisCodeMakesAbsolutelyNoSenseCanWeDeleteIt"); + // Stupid: assumeFalse in setUp() does not print any information, because + // TestWatchman does not watch test during setUp() - getName() is also not defined... + // => print info directly and use assume without message: + if ("SimpleText".equals(defFormat) || "Memory".equals(defFormat)) { + System.err.println("NOTE: A test method in " + getClass().getSimpleName() + " was ignored, as it uses too much memory with " + defFormat + "."); + Assume.assumeTrue(false); + } + } + } + + /** + * Forcible purges all cache entries from the FieldCache. + *

+ * This method will be called by tearDown to clean up FieldCache.DEFAULT. + * If a (poorly written) test has some expectation that the FieldCache + * will persist across test methods (ie: a static IndexReader) this + * method can be overridden to do nothing. + *

+ * + * @see FieldCache#purgeAllCaches() + */ + protected void purgeFieldCache(final FieldCache fc) { + fc.purgeAllCaches(); + } + + protected String getTestLabel() { + return getClass().getName() + "." + getName(); + } + + /** + * Returns true if and only if the calling thread is the primary thread + * executing the test case. + */ + protected boolean isTestThread() { + assertNotNull("Test case thread not set?", testCaseThread); + return Thread.currentThread() == testCaseThread; + } + + @After + public void tearDown() throws Exception { + State oldState = state; // capture test execution state + state = State.TEARDOWN; // set the state for subsequent tests + + // NOTE: with junit 4.7, we don't get a reproduceWith because our Watchman + // does not know if something fails in tearDown. so we ensure this happens ourselves for now. + // we can remove this if we upgrade to 4.8 + Throwable problem = null; + + try { + if (!testsFailed) { + // Note: we allow a test to go straight from SETUP -> TEARDOWN (without ever entering the RANTEST state) + // because if you assume() inside setUp(), it skips the test and the TestWatchman has no way to know... + assertTrue("ensure your setUp() calls super.setUp()!!!", oldState == State.RANTEST || oldState == State.SETUP); + } + } catch (Throwable t) { + if (problem == null) problem = t; + } + + BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount); + + // this won't throw any exceptions or fail the test + // if we change this, then change this logic + checkRogueThreadsAfter(); + // restore the default uncaught exception handler + Thread.setDefaultUncaughtExceptionHandler(savedUncaughtExceptionHandler); + + try { + checkUncaughtExceptionsAfter(); + } catch (Throwable t) { + if (problem == null) problem = t; + } + + try { + // calling assertSaneFieldCaches here isn't as useful as having test + // classes call it directly from the scope where the index readers + // are used, because they could be gc'ed just before this tearDown + // method is called. + // + // But it's better then nothing. + // + // If you are testing functionality that you know for a fact + // "violates" FieldCache sanity, then you should either explicitly + // call purgeFieldCache at the end of your test method, or refactor + // your Test class so that the inconsistant FieldCache usages are + // isolated in distinct test methods + assertSaneFieldCaches(getTestLabel()); + } catch (Throwable t) { + if (problem == null) problem = t; + } + + purgeFieldCache(FieldCache.DEFAULT); + + if (problem != null) { + testsFailed = true; + reportAdditionalFailureInfo(); + throw new RuntimeException(problem); + } + } + + /** check if the test still has threads running, we don't want them to + * fail in a subsequent test and pass the blame to the wrong test */ + private void checkRogueThreadsAfter() { + if ("perMethod".equals(TEST_CLEAN_THREADS)) { + int rogueThreads = threadCleanup("test method: '" + getName() + "'"); + if (!testsFailed && rogueThreads > 0) { + System.err.println("RESOURCE LEAK: test method: '" + getName() + + "' left " + rogueThreads + " thread(s) running"); + // TODO: fail, but print seed for now + if (uncaughtExceptions.isEmpty()) { + reportAdditionalFailureInfo(); + } + } + } + } + + /** see if any other threads threw uncaught exceptions, and fail the test if so */ + private void checkUncaughtExceptionsAfter() { + if (!uncaughtExceptions.isEmpty()) { + System.err.println("The following exceptions were thrown by threads:"); + for (UncaughtExceptionEntry entry : uncaughtExceptions) { + System.err.println("*** Thread: " + entry.thread.getName() + " ***"); + entry.exception.printStackTrace(System.err); + } + fail("Some threads threw uncaught exceptions!"); + } + } + + private final static int THREAD_STOP_GRACE_MSEC = 50; + // jvm-wide list of 'rogue threads' we found, so they only get reported once. + private final static IdentityHashMap rogueThreads = new IdentityHashMap(); + + static { + // just a hack for things like eclipse test-runner threads + for (Thread t : Thread.getAllStackTraces().keySet()) { + rogueThreads.put(t, true); + } + + if (TEST_ITER > 1) { + System.out.println("WARNING: you are using -Dtests.iter=n where n > 1, not all tests support this option."); + System.out.println("Some may crash or fail: this is not a bug."); + } + } + + /** + * Looks for leftover running threads, trying to kill them off, + * so they don't fail future tests. + * returns the number of rogue threads that it found. + */ + private static int threadCleanup(String context) { + // educated guess + Thread[] stillRunning = new Thread[Thread.activeCount()+1]; + int threadCount = 0; + int rogueCount = 0; + + if ((threadCount = Thread.enumerate(stillRunning)) > 1) { + while (threadCount == stillRunning.length) { + // truncated response + stillRunning = new Thread[stillRunning.length*2]; + threadCount = Thread.enumerate(stillRunning); + } + + for (int i = 0; i < threadCount; i++) { + Thread t = stillRunning[i]; + + if (t.isAlive() && + !rogueThreads.containsKey(t) && + t != Thread.currentThread() && + /* its ok to keep your searcher across test cases */ + (t.getName().startsWith("LuceneTestCase") && context.startsWith("test method")) == false) { + System.err.println("WARNING: " + context + " left thread running: " + t); + rogueThreads.put(t, true); + rogueCount++; + if (t.getName().startsWith("LuceneTestCase")) { + System.err.println("PLEASE CLOSE YOUR INDEXSEARCHERS IN YOUR TEST!!!!"); + continue; + } else { + // wait on the thread to die of natural causes + try { + t.join(THREAD_STOP_GRACE_MSEC); + } catch (InterruptedException e) { e.printStackTrace(); } + } + // try to stop the thread: + t.setUncaughtExceptionHandler(null); + Thread.setDefaultUncaughtExceptionHandler(null); + if (!t.getName().startsWith("SyncThread")) // avoid zookeeper jre crash + t.interrupt(); + } + } + } + return rogueCount; + } + + /** + * Asserts that FieldCacheSanityChecker does not detect any + * problems with FieldCache.DEFAULT. + *

+ * If any problems are found, they are logged to System.err + * (allong with the msg) when the Assertion is thrown. + *

+ *

+ * This method is called by tearDown after every test method, + * however IndexReaders scoped inside test methods may be garbage + * collected prior to this method being called, causing errors to + * be overlooked. Tests are encouraged to keep their IndexReaders + * scoped at the class level, or to explicitly call this method + * directly in the same scope as the IndexReader. + *

+ * + * @see org.apache.lucene.util.FieldCacheSanityChecker + */ + protected void assertSaneFieldCaches(final String msg) { + final CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries(); + Insanity[] insanity = null; + try { + try { + insanity = FieldCacheSanityChecker.checkSanity(entries); + } catch (RuntimeException e) { + dumpArray(msg + ": FieldCache", entries, System.err); + throw e; + } + + assertEquals(msg + ": Insane FieldCache usage(s) found", + 0, insanity.length); + insanity = null; + } finally { + + // report this in the event of any exception/failure + // if no failure, then insanity will be null anyway + if (null != insanity) { + dumpArray(msg + ": Insane FieldCache usage(s)", insanity, System.err); + } + + } + } + + /** + * Returns a number of at least i + *

+ * The actual number returned will be influenced by whether {@link #TEST_NIGHTLY} + * is active and {@link #RANDOM_MULTIPLIER}, but also with some random fudge. + */ + public static int atLeast(Random random, int i) { + int min = (TEST_NIGHTLY ? 3*i : i) * RANDOM_MULTIPLIER; + int max = min+(min/2); + return _TestUtil.nextInt(random, min, max); + } + + public static int atLeast(int i) { + return atLeast(random, i); + } + + /** + * Returns true if something should happen rarely, + *

+ * The actual number returned will be influenced by whether {@link #TEST_NIGHTLY} + * is active and {@link #RANDOM_MULTIPLIER}. + */ + public static boolean rarely(Random random) { + int p = TEST_NIGHTLY ? 10 : 5; + p += (p * Math.log(RANDOM_MULTIPLIER)); + int min = 100 - Math.min(p, 50); // never more than 50 + return random.nextInt(100) >= min; + } + + public static boolean rarely() { + return rarely(random); + } + + public static boolean usually(Random random) { + return !rarely(random); + } + + public static boolean usually() { + return usually(random); + } + + public static void assumeTrue(String msg, boolean b) { + Assume.assumeNoException(b ? null : new _TestIgnoredException(msg)); + } + + public static void assumeFalse(String msg, boolean b) { + assumeTrue(msg, !b); + } + + public static void assumeNoException(String msg, Exception e) { + Assume.assumeNoException(e == null ? null : new _TestIgnoredException(msg, e)); + } + + public static Set asSet(T... args) { + return new HashSet(Arrays.asList(args)); + } + + /** + * Convenience method for logging an iterator. + * + * @param label String logged before/after the items in the iterator + * @param iter Each next() is toString()ed and logged on it's own line. If iter is null this is logged differnetly then an empty iterator. + * @param stream Stream to log messages to. + */ + public static void dumpIterator(String label, Iterator iter, + PrintStream stream) { + stream.println("*** BEGIN " + label + " ***"); + if (null == iter) { + stream.println(" ... NULL ..."); + } else { + while (iter.hasNext()) { + stream.println(iter.next().toString()); + } + } + stream.println("*** END " + label + " ***"); + } + + /** + * Convenience method for logging an array. Wraps the array in an iterator and delegates + * + * @see #dumpIterator(String,Iterator,PrintStream) + */ + public static void dumpArray(String label, Object[] objs, + PrintStream stream) { + Iterator iter = (null == objs) ? null : Arrays.asList(objs).iterator(); + dumpIterator(label, iter, stream); + } + + /** create a new index writer config with random defaults */ + public static IndexWriterConfig newIndexWriterConfig(Version v, Analyzer a) { + return newIndexWriterConfig(random, v, a); + } + + /** create a new index writer config with random defaults using the specified random */ + public static IndexWriterConfig newIndexWriterConfig(Random r, Version v, Analyzer a) { + IndexWriterConfig c = new IndexWriterConfig(v, a); + c.setSimilarityProvider(similarityProvider); + if (r.nextBoolean()) { + c.setMergeScheduler(new SerialMergeScheduler()); + } + if (r.nextBoolean()) { + if (rarely(r)) { + // crazy value + c.setMaxBufferedDocs(_TestUtil.nextInt(r, 2, 7)); + } else { + // reasonable value + c.setMaxBufferedDocs(_TestUtil.nextInt(r, 8, 1000)); + } + } + if (r.nextBoolean()) { + if (rarely(r)) { + // crazy value + c.setTermIndexInterval(r.nextBoolean() ? _TestUtil.nextInt(r, 1, 31) : _TestUtil.nextInt(r, 129, 1000)); + } else { + // reasonable value + c.setTermIndexInterval(_TestUtil.nextInt(r, 32, 128)); + } + } + if (r.nextBoolean()) { + c.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(_TestUtil.nextInt(r, 1, 20))); + } + + if (r.nextBoolean()) { + c.setMergePolicy(newTieredMergePolicy()); + } else if (r.nextBoolean()) { + c.setMergePolicy(newLogMergePolicy()); + } else { + c.setMergePolicy(new MockRandomMergePolicy(r)); + } + + c.setReaderPooling(r.nextBoolean()); + c.setReaderTermsIndexDivisor(_TestUtil.nextInt(r, 1, 4)); + return c; + } + + public static LogMergePolicy newLogMergePolicy() { + return newLogMergePolicy(random); + } + + public static TieredMergePolicy newTieredMergePolicy() { + return newTieredMergePolicy(random); + } + + public static LogMergePolicy newLogMergePolicy(Random r) { + LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy(); + logmp.setUseCompoundFile(r.nextBoolean()); + logmp.setCalibrateSizeByDeletes(r.nextBoolean()); + if (rarely(r)) { + logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 4)); + } else { + logmp.setMergeFactor(_TestUtil.nextInt(r, 5, 50)); + } + return logmp; + } + + public static TieredMergePolicy newTieredMergePolicy(Random r) { + TieredMergePolicy tmp = new TieredMergePolicy(); + if (rarely(r)) { + tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 2, 4)); + tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 2, 4)); + } else { + tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 5, 50)); + tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 5, 50)); + } + tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0); + tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0); + tmp.setExpungeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0); + tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 2, 20)); + tmp.setUseCompoundFile(r.nextBoolean()); + tmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8); + tmp.setReclaimDeletesWeight(r.nextDouble()*4); + return tmp; + } + + public static LogMergePolicy newLogMergePolicy(boolean useCFS) { + LogMergePolicy logmp = newLogMergePolicy(); + logmp.setUseCompoundFile(useCFS); + return logmp; + } + + public static LogMergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) { + LogMergePolicy logmp = newLogMergePolicy(); + logmp.setUseCompoundFile(useCFS); + logmp.setMergeFactor(mergeFactor); + return logmp; + } + + public static LogMergePolicy newLogMergePolicy(int mergeFactor) { + LogMergePolicy logmp = newLogMergePolicy(); + logmp.setMergeFactor(mergeFactor); + return logmp; + } + + /** + * Returns a new Directory instance. Use this when the test does not + * care about the specific Directory implementation (most tests). + *

+ * The Directory is wrapped with {@link MockDirectoryWrapper}. + * By default this means it will be picky, such as ensuring that you + * properly close it and all open files in your test. It will emulate + * some features of Windows, such as not allowing open files to be + * overwritten. + */ + public static MockDirectoryWrapper newDirectory() throws IOException { + return newDirectory(random); + } + + /** + * Returns a new Directory instance, using the specified random. + * See {@link #newDirectory()} for more information. + */ + public static MockDirectoryWrapper newDirectory(Random r) throws IOException { + Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); + MockDirectoryWrapper dir = new MockDirectoryWrapper(r, impl); + stores.put(dir, Thread.currentThread().getStackTrace()); + dir.setThrottling(TEST_THROTTLING); + return dir; + } + + /** + * Returns a new Directory instance, with contents copied from the + * provided directory. See {@link #newDirectory()} for more + * information. + */ + public static MockDirectoryWrapper newDirectory(Directory d) throws IOException { + return newDirectory(random, d); + } + + /** Returns a new FSDirectory instance over the given file, which must be a folder. */ + public static MockDirectoryWrapper newFSDirectory(File f) throws IOException { + return newFSDirectory(f, null); + } + + /** Returns a new FSDirectory instance over the given file, which must be a folder. */ + public static MockDirectoryWrapper newFSDirectory(File f, LockFactory lf) throws IOException { + String fsdirClass = TEST_DIRECTORY; + if (fsdirClass.equals("random")) { + fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)]; + } + + if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store + fsdirClass = "org.apache.lucene.store." + fsdirClass; + } + + Class clazz; + try { + try { + clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class); + } catch (ClassCastException e) { + // TEST_DIRECTORY is not a sub-class of FSDirectory, so draw one at random + fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)]; + + if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store + fsdirClass = "org.apache.lucene.store." + fsdirClass; + } + + clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class); + } + MockDirectoryWrapper dir = new MockDirectoryWrapper(random, newFSDirectoryImpl(clazz, f)); + if (lf != null) { + dir.setLockFactory(lf); + } + stores.put(dir, Thread.currentThread().getStackTrace()); + dir.setThrottling(TEST_THROTTLING); + return dir; + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** + * Returns a new Directory instance, using the specified random + * with contents copied from the provided directory. See + * {@link #newDirectory()} for more information. + */ + public static MockDirectoryWrapper newDirectory(Random r, Directory d) throws IOException { + Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); + for (String file : d.listAll()) { + d.copy(impl, file, file, newIOContext(r)); + } + MockDirectoryWrapper dir = new MockDirectoryWrapper(r, impl); + stores.put(dir, Thread.currentThread().getStackTrace()); + dir.setThrottling(TEST_THROTTLING); + return dir; + } + + public static Field newField(String name, String value, FieldType type) { + return newField(random, name, value, type); + } + + public static Field newField(Random random, String name, String value, FieldType type) { + if (usually(random) || !type.indexed()) { + // most of the time, don't modify the params + return new Field(name, value, type); + } + + FieldType newType = new FieldType(type); + if (!newType.stored() && random.nextBoolean()) { + newType.setStored(true); // randomly store it + } + + if (!newType.storeTermVectors() && random.nextBoolean()) { + newType.setStoreTermVectors(true); + if (!newType.storeTermVectorOffsets()) { + newType.setStoreTermVectorOffsets(random.nextBoolean()); + } + if (!newType.storeTermVectorPositions()) { + newType.setStoreTermVectorPositions(random.nextBoolean()); + } + } + + // TODO: we need to do this, but smarter, ie, most of + // the time we set the same value for a given field but + // sometimes (rarely) we change it up: + /* + if (newType.omitNorms()) { + newType.setOmitNorms(random.nextBoolean()); + } + */ + + return new Field(name, value, newType); + } + + /** return a random Locale from the available locales on the system */ + public static Locale randomLocale(Random random) { + Locale locales[] = Locale.getAvailableLocales(); + return locales[random.nextInt(locales.length)]; + } + + /** return a random TimeZone from the available timezones on the system */ + public static TimeZone randomTimeZone(Random random) { + String tzIds[] = TimeZone.getAvailableIDs(); + return TimeZone.getTimeZone(tzIds[random.nextInt(tzIds.length)]); + } + + /** return a Locale object equivalent to its programmatic name */ + public static Locale localeForName(String localeName) { + String elements[] = localeName.split("\\_"); + switch(elements.length) { + case 3: return new Locale(elements[0], elements[1], elements[2]); + case 2: return new Locale(elements[0], elements[1]); + case 1: return new Locale(elements[0]); + default: throw new IllegalArgumentException("Invalid Locale: " + localeName); + } + } + + private static final String FS_DIRECTORIES[] = { + "SimpleFSDirectory", + "NIOFSDirectory", + "MMapDirectory" + }; + + private static final String CORE_DIRECTORIES[] = { + "RAMDirectory", + FS_DIRECTORIES[0], FS_DIRECTORIES[1], FS_DIRECTORIES[2] + }; + + public static String randomDirectory(Random random) { + if (rarely(random)) { + return CORE_DIRECTORIES[random.nextInt(CORE_DIRECTORIES.length)]; + } else { + return "RAMDirectory"; + } + } + + private static Directory newFSDirectoryImpl( + Class clazz, File file) + throws IOException { + FSDirectory d = null; + try { + // Assuming every FSDirectory has a ctor(File), but not all may take a + // LockFactory too, so setting it afterwards. + Constructor ctor = clazz.getConstructor(File.class); + d = ctor.newInstance(file); + } catch (Exception e) { + d = FSDirectory.open(file); + } + return d; + } + + /** Registers a temp file that will be deleted when tests are done. */ + public static void registerTempFile(File tmpFile) { + tempDirs.put(tmpFile.getAbsoluteFile(), Thread.currentThread().getStackTrace()); + } + + static Directory newDirectoryImpl(Random random, String clazzName) { + if (clazzName.equals("random")) + clazzName = randomDirectory(random); + if (clazzName.indexOf(".") == -1) // if not fully qualified, assume .store + clazzName = "org.apache.lucene.store." + clazzName; + try { + final Class clazz = Class.forName(clazzName).asSubclass(Directory.class); + // If it is a FSDirectory type, try its ctor(File) + if (FSDirectory.class.isAssignableFrom(clazz)) { + final File tmpFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR); + tmpFile.delete(); + tmpFile.mkdir(); + registerTempFile(tmpFile); + return newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), tmpFile); + } + + // try empty ctor + return clazz.newInstance(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + /** create a new searcher over the reader. + * This searcher might randomly use threads. */ + public static IndexSearcher newSearcher(IndexReader r) throws IOException { + return newSearcher(r, true); + } + + /** create a new searcher over the reader. + * This searcher might randomly use threads. + * if maybeWrap is true, this searcher might wrap the reader + * with one that returns null for getSequentialSubReaders. + */ + public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) throws IOException { + if (random.nextBoolean()) { + if (maybeWrap && rarely()) { + r = new SlowMultiReaderWrapper(r); + } + IndexSearcher ret = random.nextBoolean() ? new AssertingIndexSearcher(random, r) : new AssertingIndexSearcher(random, r.getTopReaderContext()); + ret.setSimilarityProvider(similarityProvider); + return ret; + } else { + int threads = 0; + final ExecutorService ex = (random.nextBoolean()) ? null + : Executors.newFixedThreadPool(threads = _TestUtil.nextInt(random, 1, 8), + new NamedThreadFactory("LuceneTestCase")); + if (ex != null && VERBOSE) { + System.out.println("NOTE: newSearcher using ExecutorService with " + threads + " threads"); + } + IndexSearcher ret = random.nextBoolean() ? + new AssertingIndexSearcher(random, r, ex) { + @Override + public void close() throws IOException { + super.close(); + shutdownExecutorService(ex); + } + } : new AssertingIndexSearcher(random, r.getTopReaderContext(), ex) { + @Override + public void close() throws IOException { + super.close(); + shutdownExecutorService(ex); + } + }; + ret.setSimilarityProvider(similarityProvider); + return ret; + } + } + + static void shutdownExecutorService(ExecutorService ex) { + if (ex != null) { + ex.shutdown(); + try { + ex.awaitTermination(1000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } + } + + public String getName() { + return this.name; + } + + /** Gets a resource from the classpath as {@link File}. This method should only be used, + * if a real file is needed. To get a stream, code should prefer + * {@link Class#getResourceAsStream} using {@code this.getClass()}. + */ + + protected File getDataFile(String name) throws IOException { + try { + return new File(this.getClass().getResource(name).toURI()); + } catch (Exception e) { + throw new IOException("Cannot find resource: " + name); + } + } + + // We get here from InterceptTestCaseEvents on the 'failed' event.... + public static void reportPartialFailureInfo() { + System.err.println("NOTE: reproduce with (hopefully): ant test -Dtestcase=" + testClassesRun.get(testClassesRun.size()-1) + + " -Dtests.seed=" + new ThreeLongs(staticSeed, 0L, LuceneTestCaseRunner.runnerSeed) + + reproduceWithExtraParams()); + } + + // We get here from InterceptTestCaseEvents on the 'failed' event.... + public void reportAdditionalFailureInfo() { + System.err.println("NOTE: reproduce with: ant test -Dtestcase=" + getClass().getSimpleName() + + " -Dtestmethod=" + getName() + " -Dtests.seed=" + new ThreeLongs(staticSeed, seed, LuceneTestCaseRunner.runnerSeed) + + reproduceWithExtraParams()); + } + + // extra params that were overridden needed to reproduce the command + private static String reproduceWithExtraParams() { + StringBuilder sb = new StringBuilder(); + if (!TEST_POSTINGSFORMAT.equals("random")) sb.append(" -Dtests.postingsformat=").append(TEST_POSTINGSFORMAT); + if (!TEST_LOCALE.equals("random")) sb.append(" -Dtests.locale=").append(TEST_LOCALE); + if (!TEST_TIMEZONE.equals("random")) sb.append(" -Dtests.timezone=").append(TEST_TIMEZONE); + if (!TEST_DIRECTORY.equals("random")) sb.append(" -Dtests.directory=").append(TEST_DIRECTORY); + if (RANDOM_MULTIPLIER > 1) sb.append(" -Dtests.multiplier=").append(RANDOM_MULTIPLIER); + if (TEST_NIGHTLY) sb.append(" -Dtests.nightly=true"); + return sb.toString(); + } + + public static IOContext newIOContext(Random random) { + final int randomNumDocs = random.nextInt(4192); + final int size = random.nextInt(512) * randomNumDocs; + final IOContext context; + switch (random.nextInt(5)) { + case 0: + context = IOContext.DEFAULT; + break; + case 1: + context = IOContext.READ; + break; + case 2: + context = IOContext.READONCE; + break; + case 3: + context = new IOContext(new MergeInfo(randomNumDocs, size, true, false)); + break; + case 4: + context = new IOContext(new FlushInfo(randomNumDocs, size)); + break; + default: + context = IOContext.DEFAULT; + } + return context; + } + + // initialized by the TestRunner + static boolean useNoMemoryExpensiveCodec; + + // recorded seed: for beforeClass + private static long staticSeed; + // seed for individual test methods, changed in @before + private long seed; + + static final Random seedRand = new Random(); + protected static final SmartRandom random = new SmartRandom(0); + + private String name = ""; + + /** + * Annotation for tests that should only be run during nightly builds. + */ + @Documented + @Inherited + @Retention(RetentionPolicy.RUNTIME) + public @interface Nightly {} + + /** + * Annotation for test classes that should only use codecs that are not memory expensive (avoid SimpleText, MemoryCodec). + */ + @Documented + @Inherited + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + public @interface UseNoMemoryExpensiveCodec {} + + @Ignore("just a hack") + public final void alwaysIgnoredTestMethod() {} +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java 2011-11-01 21:10:39.320609906 -0400 @@ -0,0 +1,191 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import java.util.Random; + +import org.apache.lucene.util.LuceneTestCase.Nightly; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runner.Description; +import org.junit.runner.manipulation.Filter; +import org.junit.runner.manipulation.NoTestsRemainException; +import org.junit.runner.notification.Failure; +import org.junit.runner.notification.RunListener; +import org.junit.runner.notification.RunNotifier; +import org.junit.runners.BlockJUnit4ClassRunner; +import org.junit.runners.model.FrameworkMethod; +import org.junit.runners.model.InitializationError; + +// please don't reorganize these into a wildcard! +import static org.apache.lucene.util.LuceneTestCase.TEST_ITER; +import static org.apache.lucene.util.LuceneTestCase.TEST_ITER_MIN; +import static org.apache.lucene.util.LuceneTestCase.TEST_METHOD; +import static org.apache.lucene.util.LuceneTestCase.TEST_SEED; +import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY; +import static org.apache.lucene.util.LuceneTestCase.VERBOSE; + +/** optionally filters the tests to be run by TEST_METHOD */ +public class LuceneTestCaseRunner extends BlockJUnit4ClassRunner { + private List testMethods; + static final long runnerSeed; + static { + runnerSeed = "random".equals(TEST_SEED) ? LuceneTestCase.seedRand.nextLong() : ThreeLongs.fromString(TEST_SEED).l3; + } + + @Override + protected List computeTestMethods() { + if (testMethods != null) + return testMethods; + + Random r = new Random(runnerSeed); + + LuceneTestCase.testClassesRun.add(getTestClass().getJavaClass().getSimpleName()); + testMethods = new ArrayList(); + for (Method m : getTestClass().getJavaClass().getMethods()) { + // check if the current test's class has methods annotated with @Ignore + final Ignore ignored = m.getAnnotation(Ignore.class); + if (ignored != null && !m.getName().equals("alwaysIgnoredTestMethod")) { + System.err.println("NOTE: Ignoring test method '" + m.getName() + "': " + ignored.value()); + } + // add methods starting with "test" + final int mod = m.getModifiers(); + if (m.getAnnotation(Test.class) != null || + (m.getName().startsWith("test") && + !Modifier.isAbstract(mod) && + m.getParameterTypes().length == 0 && + m.getReturnType() == Void.TYPE)) + { + if (Modifier.isStatic(mod)) + throw new RuntimeException("Test methods must not be static."); + testMethods.add(new FrameworkMethod(m)); + } + } + + if (testMethods.isEmpty()) { + throw new RuntimeException("No runnable methods!"); + } + + if (TEST_NIGHTLY == false) { + if (getTestClass().getJavaClass().isAnnotationPresent(Nightly.class)) { + /* the test class is annotated with nightly, remove all methods */ + String className = getTestClass().getJavaClass().getSimpleName(); + System.err.println("NOTE: Ignoring nightly-only test class '" + className + "'"); + testMethods.clear(); + } else { + /* remove all nightly-only methods */ + for (int i = 0; i < testMethods.size(); i++) { + final FrameworkMethod m = testMethods.get(i); + if (m.getAnnotation(Nightly.class) != null) { + System.err.println("NOTE: Ignoring nightly-only test method '" + m.getName() + "'"); + testMethods.remove(i--); + } + } + } + /* dodge a possible "no-runnable methods" exception by adding a fake ignored test */ + if (testMethods.isEmpty()) { + try { + testMethods.add(new FrameworkMethod(LuceneTestCase.class.getMethod("alwaysIgnoredTestMethod"))); + } catch (Exception e) { throw new RuntimeException(e); } + } + } + // sort the test methods first before shuffling them, so that the shuffle is consistent + // across different implementations that might order the methods different originally. + Collections.sort(testMethods, new Comparator() { + @Override + public int compare(FrameworkMethod f1, FrameworkMethod f2) { + return f1.getName().compareTo(f2.getName()); + } + }); + Collections.shuffle(testMethods, r); + return testMethods; + } + + @Override + protected void runChild(FrameworkMethod arg0, RunNotifier arg1) { + if (VERBOSE) { + System.out.println("\nNOTE: running test " + arg0.getName()); + } + + // only print iteration info if the user requested more than one iterations + final boolean verbose = VERBOSE && TEST_ITER > 1; + + final int currentIter[] = new int[1]; + arg1.addListener(new RunListener() { + @Override + public void testFailure(Failure failure) throws Exception { + if (verbose) { + System.out.println("\nNOTE: iteration " + currentIter[0] + " failed! "); + } + } + }); + for (int i = 0; i < TEST_ITER; i++) { + currentIter[0] = i; + if (verbose) { + System.out.println("\nNOTE: running iter=" + (1+i) + " of " + TEST_ITER); + } + super.runChild(arg0, arg1); + if (LuceneTestCase.testsFailed) { + if (i >= TEST_ITER_MIN - 1) { // XXX is this still off-by-one? + break; + } + } + } + } + + public LuceneTestCaseRunner(Class clazz) throws InitializationError { + super(clazz); + + // This TestRunner can handle only LuceneTestCase subclasses + if (!LuceneTestCase.class.isAssignableFrom(clazz)) { + throw new UnsupportedOperationException("LuceneTestCaseRunner can only be used with LuceneTestCase."); + } + + final boolean useNoMemoryExpensiveCodec = LuceneTestCase.useNoMemoryExpensiveCodec = + clazz.isAnnotationPresent(UseNoMemoryExpensiveCodec.class); + if (useNoMemoryExpensiveCodec) { + System.err.println("NOTE: Using no memory expensive codecs (Memory, SimpleText) for " + + clazz.getSimpleName() + "."); + } + + // evil we cannot init our random here, because super() calls computeTestMethods!!!!; + Filter f = new Filter() { + + @Override + public String describe() { return "filters according to TEST_METHOD"; } + + @Override + public boolean shouldRun(Description d) { + return TEST_METHOD == null || d.getMethodName().equals(TEST_METHOD); + } + }; + + try { + f.apply(this); + } catch (NoTestsRemainException e) { + throw new RuntimeException(e); + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/SmartRandom.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/SmartRandom.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/SmartRandom.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/SmartRandom.java 2011-11-01 21:10:39.320609906 -0400 @@ -0,0 +1,43 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Random; + +/** + * A random that tracks if its been initialized properly, + * and throws an exception if it hasn't. + */ +public class SmartRandom extends Random { + boolean initialized; + + SmartRandom(long seed) { + super(seed); + } + + @Override + protected int next(int bits) { + if (!initialized) { + System.err.println("!!! WARNING: test is using random from static initializer !!!"); + Thread.dumpStack(); + // I wish, but it causes JRE crashes + // throw new IllegalStateException("you cannot use this random from a static initializer in your test"); + } + return super.next(bits); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/_TestIgnoredException.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/_TestIgnoredException.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/_TestIgnoredException.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/_TestIgnoredException.java 2011-11-01 21:10:39.320609906 -0400 @@ -0,0 +1,51 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.PrintStream; + +/** Replacement for Assume jUnit class, so we can add a message with explanation */ +final class _TestIgnoredException extends RuntimeException { + + _TestIgnoredException(String msg) { + super(msg); + } + + _TestIgnoredException(String msg, Throwable t) { + super(msg, t); + } + + @Override + public String getMessage() { + StringBuilder sb = new StringBuilder(super.getMessage()); + if (getCause() != null) + sb.append(" - ").append(getCause()); + return sb.toString(); + } + + // only this one is called by our code, exception is not used outside this class: + @Override + public void printStackTrace(PrintStream s) { + if (getCause() != null) { + s.println(super.toString() + " - Caused by:"); + getCause().printStackTrace(s); + } else { + super.printStackTrace(s); + } + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java 2011-11-02 21:33:03.036564703 -0400 @@ -0,0 +1,517 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.BufferedOutputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.PrintStream; +import java.lang.reflect.Method; +import java.util.Enumeration; +import java.util.HashMap; +import java.util.Map; +import java.util.Random; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.CheckIndex; +import org.apache.lucene.index.ConcurrentMergeScheduler; +import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexableField; +import org.apache.lucene.index.LogMergePolicy; +import org.apache.lucene.index.MergePolicy; +import org.apache.lucene.index.MergeScheduler; +import org.apache.lucene.index.TieredMergePolicy; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.search.FieldDoc; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; +import org.apache.lucene.store.Directory; +import org.junit.Assert; + +public class _TestUtil { + + /** Returns temp dir, based on String arg in its name; + * does not create the directory. */ + public static File getTempDir(String desc) { + try { + File f = createTempFile(desc, "tmp", LuceneTestCase.TEMP_DIR); + f.delete(); + LuceneTestCase.registerTempFile(f); + return f; + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Deletes a directory and everything underneath it. + */ + public static void rmDir(File dir) throws IOException { + if (dir.exists()) { + for (File f : dir.listFiles()) { + if (f.isDirectory()) { + rmDir(f); + } else { + if (!f.delete()) { + throw new IOException("could not delete " + f); + } + } + } + if (!dir.delete()) { + throw new IOException("could not delete " + dir); + } + } + } + + /** + * Convenience method: Unzip zipName + ".zip" under destDir, removing destDir first + */ + public static void unzip(File zipName, File destDir) throws IOException { + + ZipFile zipFile = new ZipFile(zipName); + + Enumeration entries = zipFile.entries(); + + rmDir(destDir); + + destDir.mkdir(); + LuceneTestCase.registerTempFile(destDir); + + while (entries.hasMoreElements()) { + ZipEntry entry = entries.nextElement(); + + InputStream in = zipFile.getInputStream(entry); + File targetFile = new File(destDir, entry.getName()); + if (entry.isDirectory()) { + // allow unzipping with directory structure + targetFile.mkdirs(); + } else { + if (targetFile.getParentFile()!=null) { + // be on the safe side: do not rely on that directories are always extracted + // before their children (although this makes sense, but is it guaranteed?) + targetFile.getParentFile().mkdirs(); + } + OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile)); + + byte[] buffer = new byte[8192]; + int len; + while((len = in.read(buffer)) >= 0) { + out.write(buffer, 0, len); + } + + in.close(); + out.close(); + } + } + + zipFile.close(); + } + + public static void syncConcurrentMerges(IndexWriter writer) { + syncConcurrentMerges(writer.getConfig().getMergeScheduler()); + } + + public static void syncConcurrentMerges(MergeScheduler ms) { + if (ms instanceof ConcurrentMergeScheduler) + ((ConcurrentMergeScheduler) ms).sync(); + } + + /** This runs the CheckIndex tool on the index in. If any + * issues are hit, a RuntimeException is thrown; else, + * true is returned. */ + public static CheckIndex.Status checkIndex(Directory dir) throws IOException { + ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); + CheckIndex checker = new CheckIndex(dir); + checker.setInfoStream(new PrintStream(bos), false); + CheckIndex.Status indexStatus = checker.checkIndex(null); + if (indexStatus == null || indexStatus.clean == false) { + System.out.println("CheckIndex failed"); + System.out.println(bos.toString()); + throw new RuntimeException("CheckIndex failed"); + } else { + return indexStatus; + } + } + + // NOTE: only works for TMP and LMP!! + public static void setUseCompoundFile(MergePolicy mp, boolean v) { + if (mp instanceof TieredMergePolicy) { + ((TieredMergePolicy) mp).setUseCompoundFile(v); + } else if (mp instanceof LogMergePolicy) { + ((LogMergePolicy) mp).setUseCompoundFile(v); + } + } + + /** start and end are BOTH inclusive */ + public static int nextInt(Random r, int start, int end) { + return start + r.nextInt(end-start+1); + } + + public static String randomSimpleString(Random r) { + final int end = r.nextInt(10); + if (end == 0) { + // allow 0 length + return ""; + } + final char[] buffer = new char[end]; + for (int i = 0; i < end; i++) { + buffer[i] = (char) _TestUtil.nextInt(r, 97, 102); + } + return new String(buffer, 0, end); + } + + /** Returns random string, including full unicode range. */ + public static String randomUnicodeString(Random r) { + return randomUnicodeString(r, 20); + } + + /** + * Returns a random string up to a certain length. + */ + public static String randomUnicodeString(Random r, int maxLength) { + final int end = r.nextInt(maxLength); + if (end == 0) { + // allow 0 length + return ""; + } + final char[] buffer = new char[end]; + randomFixedLengthUnicodeString(r, buffer, 0, buffer.length); + return new String(buffer, 0, end); + } + + /** + * Fills provided char[] with valid random unicode code + * unit sequence. + */ + public static void randomFixedLengthUnicodeString(Random random, char[] chars, int offset, int length) { + int i = offset; + final int end = offset + length; + while(i < end) { + final int t = random.nextInt(5); + if (0 == t && i < length - 1) { + // Make a surrogate pair + // High surrogate + chars[i++] = (char) nextInt(random, 0xd800, 0xdbff); + // Low surrogate + chars[i++] = (char) nextInt(random, 0xdc00, 0xdfff); + } else if (t <= 1) { + chars[i++] = (char) random.nextInt(0x80); + } else if (2 == t) { + chars[i++] = (char) nextInt(random, 0x80, 0x7ff); + } else if (3 == t) { + chars[i++] = (char) nextInt(random, 0x800, 0xd7ff); + } else if (4 == t) { + chars[i++] = (char) nextInt(random, 0xe000, 0xffff); + } + } + } + + private static final int[] blockStarts = { + 0x0000, 0x0080, 0x0100, 0x0180, 0x0250, 0x02B0, 0x0300, 0x0370, 0x0400, + 0x0500, 0x0530, 0x0590, 0x0600, 0x0700, 0x0750, 0x0780, 0x07C0, 0x0800, + 0x0900, 0x0980, 0x0A00, 0x0A80, 0x0B00, 0x0B80, 0x0C00, 0x0C80, 0x0D00, + 0x0D80, 0x0E00, 0x0E80, 0x0F00, 0x1000, 0x10A0, 0x1100, 0x1200, 0x1380, + 0x13A0, 0x1400, 0x1680, 0x16A0, 0x1700, 0x1720, 0x1740, 0x1760, 0x1780, + 0x1800, 0x18B0, 0x1900, 0x1950, 0x1980, 0x19E0, 0x1A00, 0x1A20, 0x1B00, + 0x1B80, 0x1C00, 0x1C50, 0x1CD0, 0x1D00, 0x1D80, 0x1DC0, 0x1E00, 0x1F00, + 0x2000, 0x2070, 0x20A0, 0x20D0, 0x2100, 0x2150, 0x2190, 0x2200, 0x2300, + 0x2400, 0x2440, 0x2460, 0x2500, 0x2580, 0x25A0, 0x2600, 0x2700, 0x27C0, + 0x27F0, 0x2800, 0x2900, 0x2980, 0x2A00, 0x2B00, 0x2C00, 0x2C60, 0x2C80, + 0x2D00, 0x2D30, 0x2D80, 0x2DE0, 0x2E00, 0x2E80, 0x2F00, 0x2FF0, 0x3000, + 0x3040, 0x30A0, 0x3100, 0x3130, 0x3190, 0x31A0, 0x31C0, 0x31F0, 0x3200, + 0x3300, 0x3400, 0x4DC0, 0x4E00, 0xA000, 0xA490, 0xA4D0, 0xA500, 0xA640, + 0xA6A0, 0xA700, 0xA720, 0xA800, 0xA830, 0xA840, 0xA880, 0xA8E0, 0xA900, + 0xA930, 0xA960, 0xA980, 0xAA00, 0xAA60, 0xAA80, 0xABC0, 0xAC00, 0xD7B0, + 0xE000, 0xF900, 0xFB00, 0xFB50, 0xFE00, 0xFE10, + 0xFE20, 0xFE30, 0xFE50, 0xFE70, 0xFF00, 0xFFF0, + 0x10000, 0x10080, 0x10100, 0x10140, 0x10190, 0x101D0, 0x10280, 0x102A0, + 0x10300, 0x10330, 0x10380, 0x103A0, 0x10400, 0x10450, 0x10480, 0x10800, + 0x10840, 0x10900, 0x10920, 0x10A00, 0x10A60, 0x10B00, 0x10B40, 0x10B60, + 0x10C00, 0x10E60, 0x11080, 0x12000, 0x12400, 0x13000, 0x1D000, 0x1D100, + 0x1D200, 0x1D300, 0x1D360, 0x1D400, 0x1F000, 0x1F030, 0x1F100, 0x1F200, + 0x20000, 0x2A700, 0x2F800, 0xE0000, 0xE0100, 0xF0000, 0x100000 + }; + + private static final int[] blockEnds = { + 0x007F, 0x00FF, 0x017F, 0x024F, 0x02AF, 0x02FF, 0x036F, 0x03FF, 0x04FF, + 0x052F, 0x058F, 0x05FF, 0x06FF, 0x074F, 0x077F, 0x07BF, 0x07FF, 0x083F, + 0x097F, 0x09FF, 0x0A7F, 0x0AFF, 0x0B7F, 0x0BFF, 0x0C7F, 0x0CFF, 0x0D7F, + 0x0DFF, 0x0E7F, 0x0EFF, 0x0FFF, 0x109F, 0x10FF, 0x11FF, 0x137F, 0x139F, + 0x13FF, 0x167F, 0x169F, 0x16FF, 0x171F, 0x173F, 0x175F, 0x177F, 0x17FF, + 0x18AF, 0x18FF, 0x194F, 0x197F, 0x19DF, 0x19FF, 0x1A1F, 0x1AAF, 0x1B7F, + 0x1BBF, 0x1C4F, 0x1C7F, 0x1CFF, 0x1D7F, 0x1DBF, 0x1DFF, 0x1EFF, 0x1FFF, + 0x206F, 0x209F, 0x20CF, 0x20FF, 0x214F, 0x218F, 0x21FF, 0x22FF, 0x23FF, + 0x243F, 0x245F, 0x24FF, 0x257F, 0x259F, 0x25FF, 0x26FF, 0x27BF, 0x27EF, + 0x27FF, 0x28FF, 0x297F, 0x29FF, 0x2AFF, 0x2BFF, 0x2C5F, 0x2C7F, 0x2CFF, + 0x2D2F, 0x2D7F, 0x2DDF, 0x2DFF, 0x2E7F, 0x2EFF, 0x2FDF, 0x2FFF, 0x303F, + 0x309F, 0x30FF, 0x312F, 0x318F, 0x319F, 0x31BF, 0x31EF, 0x31FF, 0x32FF, + 0x33FF, 0x4DBF, 0x4DFF, 0x9FFF, 0xA48F, 0xA4CF, 0xA4FF, 0xA63F, 0xA69F, + 0xA6FF, 0xA71F, 0xA7FF, 0xA82F, 0xA83F, 0xA87F, 0xA8DF, 0xA8FF, 0xA92F, + 0xA95F, 0xA97F, 0xA9DF, 0xAA5F, 0xAA7F, 0xAADF, 0xABFF, 0xD7AF, 0xD7FF, + 0xF8FF, 0xFAFF, 0xFB4F, 0xFDFF, 0xFE0F, 0xFE1F, + 0xFE2F, 0xFE4F, 0xFE6F, 0xFEFF, 0xFFEF, 0xFFFF, + 0x1007F, 0x100FF, 0x1013F, 0x1018F, 0x101CF, 0x101FF, 0x1029F, 0x102DF, + 0x1032F, 0x1034F, 0x1039F, 0x103DF, 0x1044F, 0x1047F, 0x104AF, 0x1083F, + 0x1085F, 0x1091F, 0x1093F, 0x10A5F, 0x10A7F, 0x10B3F, 0x10B5F, 0x10B7F, + 0x10C4F, 0x10E7F, 0x110CF, 0x123FF, 0x1247F, 0x1342F, 0x1D0FF, 0x1D1FF, + 0x1D24F, 0x1D35F, 0x1D37F, 0x1D7FF, 0x1F02F, 0x1F09F, 0x1F1FF, 0x1F2FF, + 0x2A6DF, 0x2B73F, 0x2FA1F, 0xE007F, 0xE01EF, 0xFFFFF, 0x10FFFF + }; + + /** Returns random string of length between 0-20 codepoints, all codepoints within the same unicode block. */ + public static String randomRealisticUnicodeString(Random r) { + return randomRealisticUnicodeString(r, 20); + } + + /** Returns random string of length up to maxLength codepoints , all codepoints within the same unicode block. */ + public static String randomRealisticUnicodeString(Random r, int maxLength) { + return randomRealisticUnicodeString(r, 0, 20); + } + + /** Returns random string of length between min and max codepoints, all codepoints within the same unicode block. */ + public static String randomRealisticUnicodeString(Random r, int minLength, int maxLength) { + final int end = minLength + r.nextInt(maxLength); + final int block = r.nextInt(blockStarts.length); + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < end; i++) + sb.appendCodePoint(nextInt(r, blockStarts[block], blockEnds[block])); + return sb.toString(); + } + + /** Returns random string, with a given UTF-8 byte length*/ + public static String randomFixedByteLengthUnicodeString(Random r, int length) { + + final char[] buffer = new char[length*3]; + int bytes = length; + int i = 0; + for (; i < buffer.length && bytes != 0; i++) { + int t; + if (bytes >= 4) { + t = r.nextInt(5); + } else if (bytes >= 3) { + t = r.nextInt(4); + } else if (bytes >= 2) { + t = r.nextInt(2); + } else { + t = 0; + } + if (t == 0) { + buffer[i] = (char) r.nextInt(0x80); + bytes--; + } else if (1 == t) { + buffer[i] = (char) nextInt(r, 0x80, 0x7ff); + bytes -= 2; + } else if (2 == t) { + buffer[i] = (char) nextInt(r, 0x800, 0xd7ff); + bytes -= 3; + } else if (3 == t) { + buffer[i] = (char) nextInt(r, 0xe000, 0xffff); + bytes -= 3; + } else if (4 == t) { + // Make a surrogate pair + // High surrogate + buffer[i++] = (char) nextInt(r, 0xd800, 0xdbff); + // Low surrogate + buffer[i] = (char) nextInt(r, 0xdc00, 0xdfff); + bytes -= 4; + } + + } + return new String(buffer, 0, i); + } + + + /** Return a Codec that can read any of the + * default codecs and formats, but always writes in the specified + * format. */ + public static Codec alwaysPostingsFormat(final PostingsFormat format) { + return new Lucene40Codec() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + return format; + } + }; + } + + // TODO: generalize all 'test-checks-for-crazy-codecs' to + // annotations (LUCENE-3489) + public static String getPostingsFormat(String field) { + PostingsFormat p = Codec.getDefault().postingsFormat(); + if (p instanceof PerFieldPostingsFormat) { + return ((PerFieldPostingsFormat)p).getPostingsFormatForField(field).getName(); + } else { + return p.getName(); + } + } + + public static boolean anyFilesExceptWriteLock(Directory dir) throws IOException { + String[] files = dir.listAll(); + if (files.length > 1 || (files.length == 1 && !files[0].equals("write.lock"))) { + return true; + } else { + return false; + } + } + + /** just tries to configure things to keep the open file + * count lowish */ + public static void reduceOpenFiles(IndexWriter w) { + // keep number of open files lowish + MergePolicy mp = w.getConfig().getMergePolicy(); + if (mp instanceof LogMergePolicy) { + LogMergePolicy lmp = (LogMergePolicy) mp; + lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor())); + } else if (mp instanceof TieredMergePolicy) { + TieredMergePolicy tmp = (TieredMergePolicy) mp; + tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce())); + tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier())); + } + MergeScheduler ms = w.getConfig().getMergeScheduler(); + if (ms instanceof ConcurrentMergeScheduler) { + ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2); + ((ConcurrentMergeScheduler) ms).setMaxMergeCount(3); + } + } + + /** Checks some basic behaviour of an AttributeImpl + * @param reflectedValues contains a map with "AttributeClass#key" as values + */ + public static void assertAttributeReflection(final AttributeImpl att, Map reflectedValues) { + final Map map = new HashMap(); + att.reflectWith(new AttributeReflector() { + public void reflect(Class attClass, String key, Object value) { + map.put(attClass.getName() + '#' + key, value); + } + }); + Assert.assertEquals("Reflection does not produce same map", reflectedValues, map); + } + + public static void keepFullyDeletedSegments(IndexWriter w) { + try { + // Carefully invoke what is a package-private (test + // only, internal) method on IndexWriter: + Method m = IndexWriter.class.getDeclaredMethod("keepFullyDeletedSegments"); + m.setAccessible(true); + m.invoke(w); + } catch (Exception e) { + // Should not happen? + throw new RuntimeException(e); + } + } + + /** Adds field info for a Document. */ + public static void add(Document doc, FieldInfos fieldInfos) { + for (IndexableField field : doc) { + fieldInfos.addOrUpdate(field.name(), field.fieldType(), false, field.docValuesType()); + } + } + + /** + * insecure, fast version of File.createTempFile + * uses Random instead of SecureRandom. + */ + public static File createTempFile(String prefix, String suffix, File directory) + throws IOException { + // Force a prefix null check first + if (prefix.length() < 3) { + throw new IllegalArgumentException("prefix must be 3"); + } + String newSuffix = suffix == null ? ".tmp" : suffix; + File result; + do { + result = genTempFile(prefix, newSuffix, directory); + } while (!result.createNewFile()); + return result; + } + + /* Temp file counter */ + private static int counter = 0; + + /* identify for differnt VM processes */ + private static int counterBase = 0; + + private static class TempFileLocker {}; + private static TempFileLocker tempFileLocker = new TempFileLocker(); + + private static File genTempFile(String prefix, String suffix, File directory) { + int identify = 0; + + synchronized (tempFileLocker) { + if (counter == 0) { + int newInt = new Random().nextInt(); + counter = ((newInt / 65535) & 0xFFFF) + 0x2710; + counterBase = counter; + } + identify = counter++; + } + + StringBuilder newName = new StringBuilder(); + newName.append(prefix); + newName.append(counterBase); + newName.append(identify); + newName.append(suffix); + return new File(directory, newName.toString()); + } + + public static void assertEquals(TopDocs expected, TopDocs actual) { + Assert.assertEquals("wrong total hits", expected.totalHits, actual.totalHits); + Assert.assertEquals("wrong maxScore", expected.getMaxScore(), actual.getMaxScore(), 0.0); + Assert.assertEquals("wrong hit count", expected.scoreDocs.length, actual.scoreDocs.length); + for(int hitIDX=0;hitIDX + *

  • The test class's random seed: this is what the test sees in its beforeClass methods + *
  • The test method's random seed: this is what the test method sees starting in its befores + *
  • The test runner's random seed (controls the shuffling of test methods) + * + */ +class ThreeLongs { + public final long l1, l2, l3; + + public ThreeLongs(long l1, long l2, long l3) { + this.l1 = l1; + this.l2 = l2; + this.l3 = l3; + } + + @Override + public String toString() { + return Long.toString(l1, 16) + ":" + Long.toString(l2, 16) + ":" + Long.toString(l3, 16); + } + + public static ThreeLongs fromString(String s) { + String parts[] = s.split(":"); + assert parts.length == 3; + return new ThreeLongs(Long.parseLong(parts[0], 16), Long.parseLong(parts[1], 16), Long.parseLong(parts[2], 16)); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/ThrottledIndexOutput.java lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/ThrottledIndexOutput.java --- lucene-clean-trunk/lucene/src/test-framework/java/org/apache/lucene/util/ThrottledIndexOutput.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/lucene/src/test-framework/java/org/apache/lucene/util/ThrottledIndexOutput.java 2011-11-01 21:10:39.320609906 -0400 @@ -0,0 +1,149 @@ +package org.apache.lucene.util; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import java.io.IOException; + +import org.apache.lucene.store.DataInput; +import org.apache.lucene.store.IndexOutput; + +public class ThrottledIndexOutput extends IndexOutput { + public static final int DEFAULT_MIN_WRITTEN_BYTES = 1024; + private final int bytesPerSecond; + private IndexOutput delegate; + private long flushDelayMillis; + private long closeDelayMillis; + private long seekDelayMillis; + private long pendingBytes; + private long minBytesWritten; + private long timeElapsed; + private final byte[] bytes = new byte[1]; + + public ThrottledIndexOutput newFromDelegate(IndexOutput output) { + return new ThrottledIndexOutput(bytesPerSecond, flushDelayMillis, + closeDelayMillis, seekDelayMillis, minBytesWritten, output); + } + + public ThrottledIndexOutput(int bytesPerSecond, long delayInMillis, + IndexOutput delegate) { + this(bytesPerSecond, delayInMillis, delayInMillis, delayInMillis, + DEFAULT_MIN_WRITTEN_BYTES, delegate); + } + + public ThrottledIndexOutput(int bytesPerSecond, long delays, + int minBytesWritten, IndexOutput delegate) { + this(bytesPerSecond, delays, delays, delays, minBytesWritten, delegate); + } + + public static final int mBitsToBytes(int mbits) { + return mbits * 125000; + } + + public ThrottledIndexOutput(int bytesPerSecond, long flushDelayMillis, + long closeDelayMillis, long seekDelayMillis, long minBytesWritten, + IndexOutput delegate) { + assert bytesPerSecond > 0; + this.delegate = delegate; + this.bytesPerSecond = bytesPerSecond; + this.flushDelayMillis = flushDelayMillis; + this.closeDelayMillis = closeDelayMillis; + this.seekDelayMillis = seekDelayMillis; + this.minBytesWritten = minBytesWritten; + } + + @Override + public void flush() throws IOException { + sleep(flushDelayMillis); + delegate.flush(); + } + + @Override + public void close() throws IOException { + try { + sleep(closeDelayMillis + getDelay(true)); + } finally { + delegate.close(); + } + } + + @Override + public long getFilePointer() { + return delegate.getFilePointer(); + } + + @Override + public void seek(long pos) throws IOException { + sleep(seekDelayMillis); + delegate.seek(pos); + } + + @Override + public long length() throws IOException { + return delegate.length(); + } + + @Override + public void writeByte(byte b) throws IOException { + bytes[0] = b; + writeBytes(bytes, 0, 1); + } + + @Override + public void writeBytes(byte[] b, int offset, int length) throws IOException { + final long before = System.nanoTime(); + delegate.writeBytes(b, offset, length); + timeElapsed += System.nanoTime() - before; + pendingBytes += length; + sleep(getDelay(false)); + + } + + protected long getDelay(boolean closing) { + if (pendingBytes > 0 && (closing || pendingBytes > minBytesWritten)) { + long actualBps = (timeElapsed / pendingBytes) * 1000000000l; // nano to sec + if (actualBps > bytesPerSecond) { + long expected = (pendingBytes * 1000l / bytesPerSecond) ; + final long delay = expected - (timeElapsed / 1000000l) ; + pendingBytes = 0; + timeElapsed = 0; + return delay; + } + } + return 0; + + } + + private static final void sleep(long ms) { + if (ms <= 0) + return; + try { + Thread.sleep(ms); + } catch (InterruptedException e) { + throw new ThreadInterruptedException(e); + } + } + + @Override + public void setLength(long length) throws IOException { + delegate.setLength(length); + } + + @Override + public void copyBytes(DataInput input, long numBytes) throws IOException { + delegate.copyBytes(input, numBytes); + } +} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java 2011-11-01 18:41:12.544614524 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,334 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.StringReader; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Random; - -import org.apache.lucene.analysis.tokenattributes.*; -import org.apache.lucene.util.Attribute; -import org.apache.lucene.util.AttributeImpl; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util._TestUtil; - -/** - * Base class for all Lucene unit tests that use TokenStreams. - *

    - * When writing unit tests for analysis components, its highly recommended - * to use the helper methods here (especially in conjunction with {@link MockAnalyzer} or - * {@link MockTokenizer}), as they contain many assertions and checks to - * catch bugs. - * - * @see MockAnalyzer - * @see MockTokenizer - */ -public abstract class BaseTokenStreamTestCase extends LuceneTestCase { - // some helpers to test Analyzers and TokenStreams: - - public static interface CheckClearAttributesAttribute extends Attribute { - boolean getAndResetClearCalled(); - } - - public static final class CheckClearAttributesAttributeImpl extends AttributeImpl implements CheckClearAttributesAttribute { - private boolean clearCalled = false; - - public boolean getAndResetClearCalled() { - try { - return clearCalled; - } finally { - clearCalled = false; - } - } - - @Override - public void clear() { - clearCalled = true; - } - - @Override - public boolean equals(Object other) { - return ( - other instanceof CheckClearAttributesAttributeImpl && - ((CheckClearAttributesAttributeImpl) other).clearCalled == this.clearCalled - ); - } - - @Override - public int hashCode() { - return 76137213 ^ Boolean.valueOf(clearCalled).hashCode(); - } - - @Override - public void copyTo(AttributeImpl target) { - ((CheckClearAttributesAttributeImpl) target).clear(); - } - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[], Integer finalOffset) throws IOException { - assertNotNull(output); - CheckClearAttributesAttribute checkClearAtt = ts.addAttribute(CheckClearAttributesAttribute.class); - - assertTrue("has no CharTermAttribute", ts.hasAttribute(CharTermAttribute.class)); - CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class); - - OffsetAttribute offsetAtt = null; - if (startOffsets != null || endOffsets != null || finalOffset != null) { - assertTrue("has no OffsetAttribute", ts.hasAttribute(OffsetAttribute.class)); - offsetAtt = ts.getAttribute(OffsetAttribute.class); - } - - TypeAttribute typeAtt = null; - if (types != null) { - assertTrue("has no TypeAttribute", ts.hasAttribute(TypeAttribute.class)); - typeAtt = ts.getAttribute(TypeAttribute.class); - } - - PositionIncrementAttribute posIncrAtt = null; - if (posIncrements != null) { - assertTrue("has no PositionIncrementAttribute", ts.hasAttribute(PositionIncrementAttribute.class)); - posIncrAtt = ts.getAttribute(PositionIncrementAttribute.class); - } - - ts.reset(); - for (int i = 0; i < output.length; i++) { - // extra safety to enforce, that the state is not preserved and also assign bogus values - ts.clearAttributes(); - termAtt.setEmpty().append("bogusTerm"); - if (offsetAtt != null) offsetAtt.setOffset(14584724,24683243); - if (typeAtt != null) typeAtt.setType("bogusType"); - if (posIncrAtt != null) posIncrAtt.setPositionIncrement(45987657); - - checkClearAtt.getAndResetClearCalled(); // reset it, because we called clearAttribute() before - assertTrue("token "+i+" does not exist", ts.incrementToken()); - assertTrue("clearAttributes() was not called correctly in TokenStream chain", checkClearAtt.getAndResetClearCalled()); - - assertEquals("term "+i, output[i], termAtt.toString()); - if (startOffsets != null) - assertEquals("startOffset "+i, startOffsets[i], offsetAtt.startOffset()); - if (endOffsets != null) - assertEquals("endOffset "+i, endOffsets[i], offsetAtt.endOffset()); - if (types != null) - assertEquals("type "+i, types[i], typeAtt.type()); - if (posIncrements != null) - assertEquals("posIncrement "+i, posIncrements[i], posIncrAtt.getPositionIncrement()); - - // we can enforce some basic things about a few attributes even if the caller doesn't check: - if (offsetAtt != null) { - assertTrue("startOffset must be >= 0", offsetAtt.startOffset() >= 0); - assertTrue("endOffset must be >= 0", offsetAtt.endOffset() >= 0); - assertTrue("endOffset must be >= startOffset", offsetAtt.endOffset() >= offsetAtt.startOffset()); - } - if (posIncrAtt != null) { - assertTrue("posIncrement must be >= 0", posIncrAtt.getPositionIncrement() >= 0); - } - } - assertFalse("end of stream", ts.incrementToken()); - ts.end(); - if (finalOffset != null) - assertEquals("finalOffset ", finalOffset.intValue(), offsetAtt.endOffset()); - if (offsetAtt != null) { - assertTrue("finalOffset must be >= 0", offsetAtt.endOffset() >= 0); - } - ts.close(); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException { - assertTokenStreamContents(ts, output, startOffsets, endOffsets, types, posIncrements, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output) throws IOException { - assertTokenStreamContents(ts, output, null, null, null, null, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, String[] types) throws IOException { - assertTokenStreamContents(ts, output, null, null, types, null, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int[] posIncrements) throws IOException { - assertTokenStreamContents(ts, output, null, null, null, posIncrements, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[]) throws IOException { - assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, null, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], Integer finalOffset) throws IOException { - assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, null, finalOffset); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements) throws IOException { - assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, posIncrements, null); - } - - public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements, Integer finalOffset) throws IOException { - assertTokenStreamContents(ts, output, startOffsets, endOffsets, null, posIncrements, finalOffset); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException { - assertTokenStreamContents(a.tokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements, input.length()); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output) throws IOException { - assertAnalyzesTo(a, input, output, null, null, null, null); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output, String[] types) throws IOException { - assertAnalyzesTo(a, input, output, null, null, types, null); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int[] posIncrements) throws IOException { - assertAnalyzesTo(a, input, output, null, null, null, posIncrements); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[]) throws IOException { - assertAnalyzesTo(a, input, output, startOffsets, endOffsets, null, null); - } - - public static void assertAnalyzesTo(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements) throws IOException { - assertAnalyzesTo(a, input, output, startOffsets, endOffsets, null, posIncrements); - } - - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException { - assertTokenStreamContents(a.tokenStream("dummy", new StringReader(input)), output, startOffsets, endOffsets, types, posIncrements, input.length()); - } - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output) throws IOException { - assertAnalyzesToReuse(a, input, output, null, null, null, null); - } - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, String[] types) throws IOException { - assertAnalyzesToReuse(a, input, output, null, null, types, null); - } - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int[] posIncrements) throws IOException { - assertAnalyzesToReuse(a, input, output, null, null, null, posIncrements); - } - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[]) throws IOException { - assertAnalyzesToReuse(a, input, output, startOffsets, endOffsets, null, null); - } - - public static void assertAnalyzesToReuse(Analyzer a, String input, String[] output, int startOffsets[], int endOffsets[], int[] posIncrements) throws IOException { - assertAnalyzesToReuse(a, input, output, startOffsets, endOffsets, null, posIncrements); - } - - // simple utility method for testing stemmers - - public static void checkOneTerm(Analyzer a, final String input, final String expected) throws IOException { - assertAnalyzesTo(a, input, new String[]{expected}); - } - - public static void checkOneTermReuse(Analyzer a, final String input, final String expected) throws IOException { - assertAnalyzesToReuse(a, input, new String[]{expected}); - } - - // simple utility method for blasting tokenstreams with data to make sure they don't do anything crazy - - public static void checkRandomData(Random random, Analyzer a, int iterations) throws IOException { - checkRandomData(random, a, iterations, 20); - } - - public static void checkRandomData(Random random, Analyzer a, int iterations, int maxWordLength) throws IOException { - for (int i = 0; i < iterations; i++) { - String text; - switch(_TestUtil.nextInt(random, 0, 3)) { - case 0: - text = _TestUtil.randomSimpleString(random); - break; - case 1: - text = _TestUtil.randomRealisticUnicodeString(random, maxWordLength); - break; - default: - text = _TestUtil.randomUnicodeString(random, maxWordLength); - } - - if (VERBOSE) { - System.out.println("NOTE: BaseTokenStreamTestCase: get first token stream now text=" + text); - } - - TokenStream ts = a.tokenStream("dummy", new StringReader(text)); - assertTrue("has no CharTermAttribute", ts.hasAttribute(CharTermAttribute.class)); - CharTermAttribute termAtt = ts.getAttribute(CharTermAttribute.class); - OffsetAttribute offsetAtt = ts.hasAttribute(OffsetAttribute.class) ? ts.getAttribute(OffsetAttribute.class) : null; - PositionIncrementAttribute posIncAtt = ts.hasAttribute(PositionIncrementAttribute.class) ? ts.getAttribute(PositionIncrementAttribute.class) : null; - TypeAttribute typeAtt = ts.hasAttribute(TypeAttribute.class) ? ts.getAttribute(TypeAttribute.class) : null; - List tokens = new ArrayList(); - List types = new ArrayList(); - List positions = new ArrayList(); - List startOffsets = new ArrayList(); - List endOffsets = new ArrayList(); - ts.reset(); - while (ts.incrementToken()) { - tokens.add(termAtt.toString()); - if (typeAtt != null) types.add(typeAtt.type()); - if (posIncAtt != null) positions.add(posIncAtt.getPositionIncrement()); - if (offsetAtt != null) { - startOffsets.add(offsetAtt.startOffset()); - endOffsets.add(offsetAtt.endOffset()); - } - } - ts.end(); - ts.close(); - // verify reusing is "reproducable" and also get the normal tokenstream sanity checks - if (!tokens.isEmpty()) { - if (VERBOSE) { - System.out.println("NOTE: BaseTokenStreamTestCase: re-run analysis"); - } - if (typeAtt != null && posIncAtt != null && offsetAtt != null) { - // offset + pos + type - assertAnalyzesToReuse(a, text, - tokens.toArray(new String[tokens.size()]), - toIntArray(startOffsets), - toIntArray(endOffsets), - types.toArray(new String[types.size()]), - toIntArray(positions)); - } else if (posIncAtt != null && offsetAtt != null) { - // offset + pos - assertAnalyzesToReuse(a, text, - tokens.toArray(new String[tokens.size()]), - toIntArray(startOffsets), - toIntArray(endOffsets), - toIntArray(positions)); - } else if (offsetAtt != null) { - // offset - assertAnalyzesToReuse(a, text, - tokens.toArray(new String[tokens.size()]), - toIntArray(startOffsets), - toIntArray(endOffsets)); - } else { - // terms only - assertAnalyzesToReuse(a, text, - tokens.toArray(new String[tokens.size()])); - } - } - } - } - - static int[] toIntArray(List list) { - int ret[] = new int[list.size()]; - int offset = 0; - for (Integer i : list) { - ret[offset++] = i; - } - return ret; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/CollationTestBase.java lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/CollationTestBase.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/CollationTestBase.java 2011-11-01 18:41:12.544614524 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/CollationTestBase.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,308 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute; -import org.apache.lucene.store.RAMDirectory; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.Term; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.TermRangeFilter; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TermRangeQuery; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IndexableBinaryStringTools; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util._TestUtil; - -import java.io.StringReader; -import java.io.IOException; -import java.util.HashMap; -import java.util.Map; - -public abstract class CollationTestBase extends LuceneTestCase { - - protected String firstRangeBeginningOriginal = "\u062F"; - protected String firstRangeEndOriginal = "\u0698"; - - protected String secondRangeBeginningOriginal = "\u0633"; - protected String secondRangeEndOriginal = "\u0638"; - - /** - * Convenience method to perform the same function as CollationKeyFilter. - * - * @param keyBits the result from - * collator.getCollationKey(original).toByteArray() - * @return The encoded collation key for the original String - * @deprecated only for testing deprecated filters - */ - @Deprecated - protected String encodeCollationKey(byte[] keyBits) { - // Ensure that the backing char[] array is large enough to hold the encoded - // Binary String - int encodedLength = IndexableBinaryStringTools.getEncodedLength(keyBits, 0, keyBits.length); - char[] encodedBegArray = new char[encodedLength]; - IndexableBinaryStringTools.encode(keyBits, 0, keyBits.length, encodedBegArray, 0, encodedLength); - return new String(encodedBegArray); - } - - public void testFarsiRangeFilterCollating(Analyzer analyzer, BytesRef firstBeg, - BytesRef firstEnd, BytesRef secondBeg, - BytesRef secondEnd) throws Exception { - RAMDirectory ramDir = new RAMDirectory(); - IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig( - TEST_VERSION_CURRENT, analyzer)); - Document doc = new Document(); - doc.add(new Field("content", "\u0633\u0627\u0628", TextField.TYPE_STORED)); - doc.add(new Field("body", "body", StringField.TYPE_STORED)); - writer.addDocument(doc); - writer.close(); - IndexSearcher searcher = new IndexSearcher(ramDir, true); - Query query = new TermQuery(new Term("body","body")); - - // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi - // orders the U+0698 character before the U+0633 character, so the single - // index Term below should NOT be returned by a TermRangeFilter with a Farsi - // Collator (or an Arabic one for the case when Farsi searcher not - // supported). - ScoreDoc[] result = searcher.search - (query, new TermRangeFilter("content", firstBeg, firstEnd, true, true), 1).scoreDocs; - assertEquals("The index Term should not be included.", 0, result.length); - - result = searcher.search - (query, new TermRangeFilter("content", secondBeg, secondEnd, true, true), 1).scoreDocs; - assertEquals("The index Term should be included.", 1, result.length); - - searcher.close(); - } - - public void testFarsiRangeQueryCollating(Analyzer analyzer, BytesRef firstBeg, - BytesRef firstEnd, BytesRef secondBeg, - BytesRef secondEnd) throws Exception { - RAMDirectory ramDir = new RAMDirectory(); - IndexWriter writer = new IndexWriter(ramDir, new IndexWriterConfig( - TEST_VERSION_CURRENT, analyzer)); - Document doc = new Document(); - - // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi - // orders the U+0698 character before the U+0633 character, so the single - // index Term below should NOT be returned by a TermRangeQuery with a Farsi - // Collator (or an Arabic one for the case when Farsi is not supported). - doc.add(new Field("content", "\u0633\u0627\u0628", TextField.TYPE_STORED)); - writer.addDocument(doc); - writer.close(); - IndexSearcher searcher = new IndexSearcher(ramDir, true); - - Query query = new TermRangeQuery("content", firstBeg, firstEnd, true, true); - ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs; - assertEquals("The index Term should not be included.", 0, hits.length); - - query = new TermRangeQuery("content", secondBeg, secondEnd, true, true); - hits = searcher.search(query, null, 1000).scoreDocs; - assertEquals("The index Term should be included.", 1, hits.length); - searcher.close(); - } - - public void testFarsiTermRangeQuery(Analyzer analyzer, BytesRef firstBeg, - BytesRef firstEnd, BytesRef secondBeg, BytesRef secondEnd) throws Exception { - - RAMDirectory farsiIndex = new RAMDirectory(); - IndexWriter writer = new IndexWriter(farsiIndex, new IndexWriterConfig( - TEST_VERSION_CURRENT, analyzer)); - Document doc = new Document(); - doc.add(new Field("content", "\u0633\u0627\u0628", TextField.TYPE_STORED)); - doc.add(new Field("body", "body", StringField.TYPE_STORED)); - writer.addDocument(doc); - writer.close(); - - IndexReader reader = IndexReader.open(farsiIndex, true); - IndexSearcher search = newSearcher(reader); - - // Unicode order would include U+0633 in [ U+062F - U+0698 ], but Farsi - // orders the U+0698 character before the U+0633 character, so the single - // index Term below should NOT be returned by a TermRangeQuery - // with a Farsi Collator (or an Arabic one for the case when Farsi is - // not supported). - Query csrq - = new TermRangeQuery("content", firstBeg, firstEnd, true, true); - ScoreDoc[] result = search.search(csrq, null, 1000).scoreDocs; - assertEquals("The index Term should not be included.", 0, result.length); - - csrq = new TermRangeQuery - ("content", secondBeg, secondEnd, true, true); - result = search.search(csrq, null, 1000).scoreDocs; - assertEquals("The index Term should be included.", 1, result.length); - search.close(); - } - - // Test using various international locales with accented characters (which - // sort differently depending on locale) - // - // Copied (and slightly modified) from - // org.apache.lucene.search.TestSort.testInternationalSort() - // - // TODO: this test is really fragile. there are already 3 different cases, - // depending upon unicode version. - public void testCollationKeySort(Analyzer usAnalyzer, - Analyzer franceAnalyzer, - Analyzer swedenAnalyzer, - Analyzer denmarkAnalyzer, - String usResult, - String frResult, - String svResult, - String dkResult) throws Exception { - RAMDirectory indexStore = new RAMDirectory(); - IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false))); - - // document data: - // the tracer field is used to determine which document was hit - String[][] sortData = new String[][] { - // tracer contents US France Sweden (sv_SE) Denmark (da_DK) - { "A", "x", "p\u00EAche", "p\u00EAche", "p\u00EAche", "p\u00EAche" }, - { "B", "y", "HAT", "HAT", "HAT", "HAT" }, - { "C", "x", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9", "p\u00E9ch\u00E9" }, - { "D", "y", "HUT", "HUT", "HUT", "HUT" }, - { "E", "x", "peach", "peach", "peach", "peach" }, - { "F", "y", "H\u00C5T", "H\u00C5T", "H\u00C5T", "H\u00C5T" }, - { "G", "x", "sin", "sin", "sin", "sin" }, - { "H", "y", "H\u00D8T", "H\u00D8T", "H\u00D8T", "H\u00D8T" }, - { "I", "x", "s\u00EDn", "s\u00EDn", "s\u00EDn", "s\u00EDn" }, - { "J", "y", "HOT", "HOT", "HOT", "HOT" }, - }; - - FieldType customType = new FieldType(); - customType.setStored(true); - - for (int i = 0 ; i < sortData.length ; ++i) { - Document doc = new Document(); - doc.add(new Field("tracer", sortData[i][0], customType)); - doc.add(new TextField("contents", sortData[i][1])); - if (sortData[i][2] != null) - doc.add(new TextField("US", usAnalyzer.tokenStream("US", new StringReader(sortData[i][2])))); - if (sortData[i][3] != null) - doc.add(new TextField("France", franceAnalyzer.tokenStream("France", new StringReader(sortData[i][3])))); - if (sortData[i][4] != null) - doc.add(new TextField("Sweden", swedenAnalyzer.tokenStream("Sweden", new StringReader(sortData[i][4])))); - if (sortData[i][5] != null) - doc.add(new TextField("Denmark", denmarkAnalyzer.tokenStream("Denmark", new StringReader(sortData[i][5])))); - writer.addDocument(doc); - } - writer.optimize(); - writer.close(); - IndexSearcher searcher = new IndexSearcher(indexStore, true); - - Sort sort = new Sort(); - Query queryX = new TermQuery(new Term ("contents", "x")); - Query queryY = new TermQuery(new Term ("contents", "y")); - - sort.setSort(new SortField("US", SortField.Type.STRING)); - assertMatches(searcher, queryY, sort, usResult); - - sort.setSort(new SortField("France", SortField.Type.STRING)); - assertMatches(searcher, queryX, sort, frResult); - - sort.setSort(new SortField("Sweden", SortField.Type.STRING)); - assertMatches(searcher, queryY, sort, svResult); - - sort.setSort(new SortField("Denmark", SortField.Type.STRING)); - assertMatches(searcher, queryY, sort, dkResult); - } - - // Make sure the documents returned by the search match the expected list - // Copied from TestSort.java - private void assertMatches(IndexSearcher searcher, Query query, Sort sort, - String expectedResult) throws IOException { - ScoreDoc[] result = searcher.search(query, null, 1000, sort).scoreDocs; - StringBuilder buff = new StringBuilder(10); - int n = result.length; - for (int i = 0 ; i < n ; ++i) { - Document doc = searcher.doc(result[i].doc); - IndexableField[] v = doc.getFields("tracer"); - for (int j = 0 ; j < v.length ; ++j) { - buff.append(v[j].stringValue()); - } - } - assertEquals(expectedResult, buff.toString()); - } - - public void assertThreadSafe(final Analyzer analyzer) throws Exception { - int numTestPoints = 100; - int numThreads = _TestUtil.nextInt(random, 3, 5); - final HashMap map = new HashMap(); - - // create a map up front. - // then with multiple threads, generate sort keys for all the keys in the map - // and ensure they are the same as the ones we produced in serial fashion. - - for (int i = 0; i < numTestPoints; i++) { - String term = _TestUtil.randomSimpleString(random); - TokenStream ts = analyzer.tokenStream("fake", new StringReader(term)); - TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); - BytesRef bytes = termAtt.getBytesRef(); - ts.reset(); - assertTrue(ts.incrementToken()); - termAtt.fillBytesRef(); - // ensure we make a copy of the actual bytes too - map.put(term, new BytesRef(bytes)); - } - - Thread threads[] = new Thread[numThreads]; - for (int i = 0; i < numThreads; i++) { - threads[i] = new Thread() { - @Override - public void run() { - try { - for (Map.Entry mapping : map.entrySet()) { - String term = mapping.getKey(); - BytesRef expected = mapping.getValue(); - TokenStream ts = analyzer.tokenStream("fake", new StringReader(term)); - TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); - BytesRef bytes = termAtt.getBytesRef(); - ts.reset(); - assertTrue(ts.incrementToken()); - termAtt.fillBytesRef(); - assertEquals(expected, bytes); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }; - } - for (int i = 0; i < numThreads; i++) { - threads[i].start(); - } - for (int i = 0; i < numThreads; i++) { - threads[i].join(); - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java 2011-11-01 18:41:12.544614524 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,140 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.io.Reader; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.automaton.CharacterRunAutomaton; - -/** - * Analyzer for testing - *

    - * This analyzer is a replacement for Whitespace/Simple/KeywordAnalyzers - * for unit tests. If you are testing a custom component such as a queryparser - * or analyzer-wrapper that consumes analysis streams, its a great idea to test - * it with this analyzer instead. MockAnalyzer has the following behavior: - *

      - *
    • By default, the assertions in {@link MockTokenizer} are turned on for extra - * checks that the consumer is consuming properly. These checks can be disabled - * with {@link #setEnableChecks(boolean)}. - *
    • Payload data is randomly injected into the stream for more thorough testing - * of payloads. - *
    - * @see MockTokenizer - */ -public final class MockAnalyzer extends Analyzer { - private final CharacterRunAutomaton runAutomaton; - private final boolean lowerCase; - private final CharacterRunAutomaton filter; - private final boolean enablePositionIncrements; - private int positionIncrementGap; - private final Random random; - private Map previousMappings = new HashMap(); - private boolean enableChecks = true; - - /** - * Creates a new MockAnalyzer. - * - * @param random Random for payloads behavior - * @param runAutomaton DFA describing how tokenization should happen (e.g. [a-zA-Z]+) - * @param lowerCase true if the tokenizer should lowercase terms - * @param filter DFA describing how terms should be filtered (set of stopwords, etc) - * @param enablePositionIncrements true if position increments should reflect filtered terms. - */ - public MockAnalyzer(Random random, CharacterRunAutomaton runAutomaton, boolean lowerCase, CharacterRunAutomaton filter, boolean enablePositionIncrements) { - super(new PerFieldReuseStrategy()); - this.random = random; - this.runAutomaton = runAutomaton; - this.lowerCase = lowerCase; - this.filter = filter; - this.enablePositionIncrements = enablePositionIncrements; - } - - /** - * Calls {@link #MockAnalyzer(Random, CharacterRunAutomaton, boolean, CharacterRunAutomaton, boolean) - * MockAnalyzer(random, runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false}). - */ - public MockAnalyzer(Random random, CharacterRunAutomaton runAutomaton, boolean lowerCase) { - this(random, runAutomaton, lowerCase, MockTokenFilter.EMPTY_STOPSET, false); - } - - /** - * Create a Whitespace-lowercasing analyzer with no stopwords removal. - *

    - * Calls {@link #MockAnalyzer(Random, CharacterRunAutomaton, boolean, CharacterRunAutomaton, boolean) - * MockAnalyzer(random, MockTokenizer.WHITESPACE, true, MockTokenFilter.EMPTY_STOPSET, false}). - */ - public MockAnalyzer(Random random) { - this(random, MockTokenizer.WHITESPACE, true); - } - - @Override - public TokenStreamComponents createComponents(String fieldName, Reader reader) { - MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase); - tokenizer.setEnableChecks(enableChecks); - TokenFilter filt = new MockTokenFilter(tokenizer, filter, enablePositionIncrements); - return new TokenStreamComponents(tokenizer, maybePayload(filt, fieldName)); - } - - private synchronized TokenFilter maybePayload(TokenFilter stream, String fieldName) { - Integer val = previousMappings.get(fieldName); - if (val == null) { - val = -1; // no payloads - if (LuceneTestCase.rarely(random)) { - switch(random.nextInt(3)) { - case 0: val = -1; // no payloads - break; - case 1: val = Integer.MAX_VALUE; // variable length payload - break; - case 2: val = random.nextInt(12); // fixed length payload - break; - } - } - previousMappings.put(fieldName, val); // save it so we are consistent for this field - } - - if (val == -1) - return stream; - else if (val == Integer.MAX_VALUE) - return new MockVariableLengthPayloadFilter(random, stream); - else - return new MockFixedLengthPayloadFilter(random, stream, val); - } - - public void setPositionIncrementGap(int positionIncrementGap){ - this.positionIncrementGap = positionIncrementGap; - } - - @Override - public int getPositionIncrementGap(String fieldName){ - return positionIncrementGap; - } - - /** - * Toggle consumer workflow checking: if your test consumes tokenstreams normally you - * should leave this enabled. - */ - public void setEnableChecks(boolean enableChecks) { - this.enableChecks = enableChecks; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java 2011-11-01 18:41:12.544614524 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,49 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Random; - -import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.index.Payload; - -public final class MockFixedLengthPayloadFilter extends TokenFilter { - private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class); - private final Random random; - private final byte[] bytes; - private final Payload payload; - - public MockFixedLengthPayloadFilter(Random random, TokenStream in, int length) { - super(in); - this.random = random; - this.bytes = new byte[length]; - this.payload = new Payload(bytes); - } - - @Override - public boolean incrementToken() throws IOException { - if (input.incrementToken()) { - random.nextBytes(bytes); - payloadAtt.setPayload(payload); - return true; - } else { - return false; - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java 2011-11-01 18:41:12.544614524 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,93 +0,0 @@ -package org.apache.lucene.analysis; -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.index.Payload; - -import java.io.IOException; -import java.io.Reader; - - -/** - * - * - **/ -public final class MockPayloadAnalyzer extends Analyzer { - - @Override - public TokenStreamComponents createComponents(String fieldName, Reader reader) { - Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true); - return new TokenStreamComponents(result, new MockPayloadFilter(result, fieldName)); - } -} - -/** - * - * - **/ -final class MockPayloadFilter extends TokenFilter { - String fieldName; - - int pos; - - int i; - - final PositionIncrementAttribute posIncrAttr; - final PayloadAttribute payloadAttr; - final CharTermAttribute termAttr; - - public MockPayloadFilter(TokenStream input, String fieldName) { - super(input); - this.fieldName = fieldName; - pos = 0; - i = 0; - posIncrAttr = input.addAttribute(PositionIncrementAttribute.class); - payloadAttr = input.addAttribute(PayloadAttribute.class); - termAttr = input.addAttribute(CharTermAttribute.class); - } - - @Override - public boolean incrementToken() throws IOException { - if (input.incrementToken()) { - payloadAttr.setPayload(new Payload(("pos: " + pos).getBytes())); - int posIncr; - if (i % 2 == 1) { - posIncr = 1; - } else { - posIncr = 0; - } - posIncrAttr.setPositionIncrement(posIncr); - pos += posIncr; - i++; - return true; - } else { - return false; - } - } - - @Override - public void reset() throws IOException { - super.reset(); - i = 0; - pos = 0; - } -} - diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenFilter.java lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenFilter.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenFilter.java 2011-11-01 18:41:12.544614524 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenFilter.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,108 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import static org.apache.lucene.util.automaton.BasicAutomata.makeEmpty; -import static org.apache.lucene.util.automaton.BasicAutomata.makeString; - -import java.io.IOException; -import java.util.Arrays; - -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.util.automaton.BasicOperations; -import org.apache.lucene.util.automaton.CharacterRunAutomaton; - -/** - * A tokenfilter for testing that removes terms accepted by a DFA. - *

      - *
    • Union a list of singletons to act like a stopfilter. - *
    • Use the complement to act like a keepwordfilter - *
    • Use a regex like .{12,} to act like a lengthfilter - *
    - */ -public final class MockTokenFilter extends TokenFilter { - /** Empty set of stopwords */ - public static final CharacterRunAutomaton EMPTY_STOPSET = - new CharacterRunAutomaton(makeEmpty()); - - /** Set of common english stopwords */ - public static final CharacterRunAutomaton ENGLISH_STOPSET = - new CharacterRunAutomaton(BasicOperations.union(Arrays.asList( - makeString("a"), makeString("an"), makeString("and"), makeString("are"), - makeString("as"), makeString("at"), makeString("be"), makeString("but"), - makeString("by"), makeString("for"), makeString("if"), makeString("in"), - makeString("into"), makeString("is"), makeString("it"), makeString("no"), - makeString("not"), makeString("of"), makeString("on"), makeString("or"), - makeString("such"), makeString("that"), makeString("the"), makeString("their"), - makeString("then"), makeString("there"), makeString("these"), makeString("they"), - makeString("this"), makeString("to"), makeString("was"), makeString("will"), - makeString("with")))); - - private final CharacterRunAutomaton filter; - private boolean enablePositionIncrements = false; - - private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); - private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class); - - /** - * Create a new MockTokenFilter. - * - * @param input TokenStream to filter - * @param filter DFA representing the terms that should be removed. - * @param enablePositionIncrements true if the removal should accumulate position increments. - */ - public MockTokenFilter(TokenStream input, CharacterRunAutomaton filter, boolean enablePositionIncrements) { - super(input); - this.filter = filter; - this.enablePositionIncrements = enablePositionIncrements; - } - - @Override - public boolean incrementToken() throws IOException { - // return the first non-stop word found - int skippedPositions = 0; - while (input.incrementToken()) { - if (!filter.run(termAtt.buffer(), 0, termAtt.length())) { - if (enablePositionIncrements) { - posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions); - } - return true; - } - skippedPositions += posIncrAtt.getPositionIncrement(); - } - // reached EOS -- return false - return false; - } - - /** - * @see #setEnablePositionIncrements(boolean) - */ - public boolean getEnablePositionIncrements() { - return enablePositionIncrements; - } - - /** - * If true, this Filter will preserve - * positions of the incoming tokens (ie, accumulate and - * set position increments of the removed stop tokens). - */ - public void setEnablePositionIncrements(boolean enable) { - this.enablePositionIncrements = enable; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java 2011-11-02 10:34:14.684585067 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,204 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.io.Reader; - -import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; -import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.util.AttributeSource.AttributeFactory; -import org.apache.lucene.util.automaton.CharacterRunAutomaton; -import org.apache.lucene.util.automaton.RegExp; - -/** - * Tokenizer for testing. - *

    - * This tokenizer is a replacement for {@link #WHITESPACE}, {@link #SIMPLE}, and {@link #KEYWORD} - * tokenizers. If you are writing a component such as a TokenFilter, its a great idea to test - * it wrapping this tokenizer instead for extra checks. This tokenizer has the following behavior: - *

      - *
    • An internal state-machine is used for checking consumer consistency. These checks can - * be disabled with {@link #setEnableChecks(boolean)}. - *
    • For convenience, optionally lowercases terms that it outputs. - *
    - */ -public class MockTokenizer extends Tokenizer { - /** Acts Similar to WhitespaceTokenizer */ - public static final CharacterRunAutomaton WHITESPACE = - new CharacterRunAutomaton(new RegExp("[^ \t\r\n]+").toAutomaton()); - /** Acts Similar to KeywordTokenizer. - * TODO: Keyword returns an "empty" token for an empty reader... - */ - public static final CharacterRunAutomaton KEYWORD = - new CharacterRunAutomaton(new RegExp(".*").toAutomaton()); - /** Acts like LetterTokenizer. */ - // the ugly regex below is Unicode 5.2 [:Letter:] - public static final CharacterRunAutomaton SIMPLE = - new CharacterRunAutomaton(new RegExp("[A-Za-zªµºÀ-ÖØ-öø-ˁˆ-ˑˠ-ˤˬˮͰ-ʹͶͷͺ-ͽΆΈ-ΊΌΎ-ΡΣ-ϵϷ-ҁҊ-ԥԱ-Ֆՙա-ևא-תװ-ײء-يٮٯٱ-ۓەۥۦۮۯۺ-ۼۿܐܒ-ܯݍ-ޥޱߊ-ߪߴߵߺࠀ-ࠕࠚࠤࠨऄ-हऽॐक़-ॡॱॲॹ-ॿঅ-ঌএঐও-নপ-রলশ-হঽৎড়ঢ়য়-ৡৰৱਅ-ਊਏਐਓ-ਨਪ-ਰਲਲ਼ਵਸ਼ਸਹਖ਼-ੜਫ਼ੲ-ੴઅ-ઍએ-ઑઓ-નપ-રલળવ-હઽૐૠૡଅ-ଌଏଐଓ-ନପ-ରଲଳଵ-ହଽଡ଼ଢ଼ୟ-ୡୱஃஅ-ஊஎ-ஐஒ-கஙசஜஞடணதந-பம-ஹௐఅ-ఌఎ-ఐఒ-నప-ళవ-హఽౘౙౠౡಅ-ಌಎ-ಐಒ-ನಪ-ಳವ-ಹಽೞೠೡഅ-ഌഎ-ഐഒ-നപ-ഹഽൠൡൺ-ൿඅ-ඖක-නඳ-රලව-ෆก-ะาำเ-ๆກຂຄງຈຊຍດ-ທນ-ຟມ-ຣລວສຫອ-ະາຳຽເ-ໄໆໜໝༀཀ-ཇཉ-ཬྈ-ྋက-ဪဿၐ-ၕၚ-ၝၡၥၦၮ-ၰၵ-ႁႎႠ-Ⴥა-ჺჼᄀ-ቈቊ-ቍቐ-ቖቘቚ-ቝበ-ኈኊ-ኍነ-ኰኲ-ኵኸ-ኾዀዂ-ዅወ-ዖዘ-ጐጒ-ጕጘ-ፚᎀ-ᎏᎠ-Ᏼᐁ-ᙬᙯ-ᙿᚁ-ᚚᚠ-ᛪᜀ-ᜌᜎ-ᜑᜠ-ᜱᝀ-ᝑᝠ-ᝬᝮ-ᝰក-ឳៗៜᠠ-ᡷᢀ-ᢨᢪᢰ-ᣵᤀ-ᤜᥐ-ᥭᥰ-ᥴᦀ-ᦫᧁ-ᧇᨀ-ᨖᨠ-ᩔᪧᬅ-ᬳᭅ-ᭋᮃ-ᮠᮮᮯᰀ-ᰣᱍ-ᱏᱚ-ᱽᳩ-ᳬᳮ-ᳱᴀ-ᶿḀ-ἕἘ-Ἕἠ-ὅὈ-Ὅὐ-ὗὙὛὝὟ-ώᾀ-ᾴᾶ-ᾼιῂ-ῄῆ-ῌῐ-ΐῖ-Ίῠ-Ῥῲ-ῴῶ-ῼⁱⁿₐ-ₔℂℇℊ-ℓℕℙ-ℝℤΩℨK-ℭℯ-ℹℼ-ℿⅅ-ⅉⅎↃↄⰀ-Ⱞⰰ-ⱞⱠ-ⳤⳫ-ⳮⴀ-ⴥⴰ-ⵥⵯⶀ-ⶖⶠ-ⶦⶨ-ⶮⶰ-ⶶⶸ-ⶾⷀ-ⷆⷈ-ⷎⷐ-ⷖⷘ-ⷞⸯ々〆〱-〵〻〼ぁ-ゖゝ-ゟァ-ヺー-ヿㄅ-ㄭㄱ-ㆎㆠ-ㆷㇰ-ㇿ㐀-䶵一-鿋ꀀ-ꒌꓐ-ꓽꔀ-ꘌꘐ-ꘟꘪꘫꙀ-ꙟꙢ-ꙮꙿ-ꚗꚠ-ꛥꜗ-ꜟꜢ-ꞈꞋꞌꟻ-ꠁꠃ-ꠅꠇ-ꠊꠌ-ꠢꡀ-ꡳꢂ-ꢳꣲ-ꣷꣻꤊ-ꤥꤰ-ꥆꥠ-ꥼꦄ-ꦲꧏꨀ-ꨨꩀ-ꩂꩄ-ꩋꩠ-ꩶꩺꪀ-ꪯꪱꪵꪶꪹ-ꪽꫀꫂꫛ-ꫝꯀ-ꯢ가-힣ힰ-ퟆퟋ-ퟻ豈-鶴侮-舘並-龎ff-stﬓ-ﬗיִײַ-ﬨשׁ-זּטּ-לּמּנּסּףּפּצּ-ﮱﯓ-ﴽﵐ-ﶏﶒ-ﷇﷰ-ﷻﹰ-ﹴﹶ-ﻼＡ-Ｚａ-ｚｦ-ﾾￂ-ￇￊ-ￏￒ-ￗￚ-ￜ𐀀-𐀋𐀍-𐀦𐀨-𐀺𐀼𐀽𐀿-𐁍𐁐-𐁝𐂀-𐃺𐊀-𐊜𐊠-𐋐𐌀-𐌞𐌰-𐍀𐍂-𐍉𐎀-𐎝𐎠-𐏃𐏈-𐏏𐐀-𐒝𐠀-𐠅𐠈𐠊-𐠵𐠷𐠸𐠼𐠿-𐡕𐤀-𐤕𐤠-𐤹𐨀𐨐-𐨓𐨕-𐨗𐨙-𐨳𐩠-𐩼𐬀-𐬵𐭀-𐭕𐭠-𐭲𐰀-𐱈𑂃-𑂯𒀀-𒍮𓀀-𓐮𝐀-𝑔𝑖-𝒜𝒞𝒟𝒢𝒥𝒦𝒩-𝒬𝒮-𝒹𝒻𝒽-𝓃𝓅-𝔅𝔇-𝔊𝔍-𝔔𝔖-𝔜𝔞-𝔹𝔻-𝔾𝕀-𝕄𝕆𝕊-𝕐𝕒-𝚥𝚨-𝛀𝛂-𝛚𝛜-𝛺𝛼-𝜔𝜖-𝜴𝜶-𝝎𝝐-𝝮𝝰-𝞈𝞊-𝞨𝞪-𝟂𝟄-𝟋𠀀-𪛖𪜀-𫜴丽-𪘀]+").toAutomaton()); - - private final CharacterRunAutomaton runAutomaton; - private final boolean lowerCase; - private final int maxTokenLength; - public static final int DEFAULT_MAX_TOKEN_LENGTH = Integer.MAX_VALUE; - private int state; - - private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); - private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); - int off = 0; - - // TODO: "register" with LuceneTestCase to ensure all streams are closed() ? - // currently, we can only check that the lifecycle is correct if someone is reusing, - // but not for "one-offs". - private static enum State { - SETREADER, // consumer set a reader input either via ctor or via reset(Reader) - RESET, // consumer has called reset() - INCREMENT, // consumer is consuming, has called incrementToken() == true - INCREMENT_FALSE, // consumer has called incrementToken() which returned false - END, // consumer has called end() to perform end of stream operations - CLOSE // consumer has called close() to release any resources - }; - - private State streamState = State.CLOSE; - private boolean enableChecks = true; - - public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) { - super(factory, input); - this.runAutomaton = runAutomaton; - this.lowerCase = lowerCase; - this.state = runAutomaton.getInitialState(); - this.streamState = State.SETREADER; - this.maxTokenLength = maxTokenLength; - } - - public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase, int maxTokenLength) { - this(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY, input, runAutomaton, lowerCase, maxTokenLength); - } - - public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) { - this(input, runAutomaton, lowerCase, DEFAULT_MAX_TOKEN_LENGTH); - } - - @Override - public final boolean incrementToken() throws IOException { - assert !enableChecks || (streamState == State.RESET || streamState == State.INCREMENT) - : "incrementToken() called while in wrong state: " + streamState; - clearAttributes(); - for (;;) { - int startOffset = off; - int cp = readCodePoint(); - if (cp < 0) { - break; - } else if (isTokenChar(cp)) { - int endOffset; - do { - char chars[] = Character.toChars(normalize(cp)); - for (int i = 0; i < chars.length; i++) - termAtt.append(chars[i]); - endOffset = off; - if (termAtt.length() >= maxTokenLength) { - break; - } - cp = readCodePoint(); - } while (cp >= 0 && isTokenChar(cp)); - offsetAtt.setOffset(correctOffset(startOffset), correctOffset(endOffset)); - streamState = State.INCREMENT; - return true; - } - } - streamState = State.INCREMENT_FALSE; - return false; - } - - protected int readCodePoint() throws IOException { - int ch = input.read(); - if (ch < 0) { - return ch; - } else { - assert !Character.isLowSurrogate((char) ch); - off++; - if (Character.isHighSurrogate((char) ch)) { - int ch2 = input.read(); - if (ch2 >= 0) { - off++; - assert Character.isLowSurrogate((char) ch2); - return Character.toCodePoint((char) ch, (char) ch2); - } - } - return ch; - } - } - - protected boolean isTokenChar(int c) { - state = runAutomaton.step(state, c); - if (state < 0) { - state = runAutomaton.getInitialState(); - return false; - } else { - return true; - } - } - - protected int normalize(int c) { - return lowerCase ? Character.toLowerCase(c) : c; - } - - @Override - public void reset() throws IOException { - super.reset(); - state = runAutomaton.getInitialState(); - off = 0; - assert !enableChecks || streamState != State.RESET : "double reset()"; - streamState = State.RESET; - } - - @Override - public void close() throws IOException { - super.close(); - // in some exceptional cases (e.g. TestIndexWriterExceptions) a test can prematurely close() - // these tests should disable this check, by default we check the normal workflow. - // TODO: investigate the CachingTokenFilter "double-close"... for now we ignore this - assert !enableChecks || streamState == State.END || streamState == State.CLOSE : "close() called in wrong state: " + streamState; - streamState = State.CLOSE; - } - - @Override - public void reset(Reader input) throws IOException { - super.reset(input); - assert !enableChecks || streamState == State.CLOSE : "setReader() called in wrong state: " + streamState; - streamState = State.SETREADER; - } - - @Override - public void end() throws IOException { - int finalOffset = correctOffset(off); - offsetAtt.setOffset(finalOffset, finalOffset); - // some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns false. - // these tests should disable this check (in general you should consume the entire stream) - assert !enableChecks || streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!"; - streamState = State.END; - } - - /** - * Toggle consumer workflow checking: if your test consumes tokenstreams normally you - * should leave this enabled. - */ - public void setEnableChecks(boolean enableChecks) { - this.enableChecks = enableChecks; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java 2011-11-01 18:41:12.544614524 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,51 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Random; - -import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; -import org.apache.lucene.index.Payload; - -public final class MockVariableLengthPayloadFilter extends TokenFilter { - private static final int MAXLENGTH = 129; - - private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class); - private final Random random; - private final byte[] bytes = new byte[MAXLENGTH]; - private final Payload payload; - - public MockVariableLengthPayloadFilter(Random random, TokenStream in) { - super(in); - this.random = random; - this.payload = new Payload(bytes); - } - - @Override - public boolean incrementToken() throws IOException { - if (input.incrementToken()) { - random.nextBytes(bytes); - payload.setData(bytes, 0, random.nextInt(MAXLENGTH)); - payloadAtt.setPayload(payload); - return true; - } else { - return false; - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/VocabularyAssert.java lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/VocabularyAssert.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/analysis/VocabularyAssert.java 2011-11-01 18:41:12.544614524 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/analysis/VocabularyAssert.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,83 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.BufferedReader; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.zip.ZipFile; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import org.junit.Assert; - -/** Utility class for doing vocabulary-based stemming tests */ -public class VocabularyAssert { - /** Run a vocabulary test against two data files. */ - public static void assertVocabulary(Analyzer a, InputStream voc, InputStream out) - throws IOException { - BufferedReader vocReader = new BufferedReader( - new InputStreamReader(voc, "UTF-8")); - BufferedReader outputReader = new BufferedReader( - new InputStreamReader(out, "UTF-8")); - String inputWord = null; - while ((inputWord = vocReader.readLine()) != null) { - String expectedWord = outputReader.readLine(); - Assert.assertNotNull(expectedWord); - BaseTokenStreamTestCase.checkOneTermReuse(a, inputWord, expectedWord); - } - } - - /** Run a vocabulary test against one file: tab separated. */ - public static void assertVocabulary(Analyzer a, InputStream vocOut) - throws IOException { - BufferedReader vocReader = new BufferedReader( - new InputStreamReader(vocOut, "UTF-8")); - String inputLine = null; - while ((inputLine = vocReader.readLine()) != null) { - if (inputLine.startsWith("#") || inputLine.trim().length() == 0) - continue; /* comment */ - String words[] = inputLine.split("\t"); - BaseTokenStreamTestCase.checkOneTermReuse(a, words[0], words[1]); - } - } - - /** Run a vocabulary test against two data files inside a zip file */ - public static void assertVocabulary(Analyzer a, File zipFile, String voc, String out) - throws IOException { - ZipFile zip = new ZipFile(zipFile); - InputStream v = zip.getInputStream(zip.getEntry(voc)); - InputStream o = zip.getInputStream(zip.getEntry(out)); - assertVocabulary(a, v, o); - v.close(); - o.close(); - zip.close(); - } - - /** Run a vocabulary test against a tab-separated data file inside a zip file */ - public static void assertVocabulary(Analyzer a, File zipFile, String vocOut) - throws IOException { - ZipFile zip = new ZipFile(zipFile); - InputStream vo = zip.getInputStream(zip.getEntry(vocOut)); - assertVocabulary(a, vo); - vo.close(); - zip.close(); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java 2011-11-01 18:41:12.548614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,230 +0,0 @@ -package org.apache.lucene.index.codecs.mockintblock; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.sep.IntStreamFactory; -import org.apache.lucene.index.codecs.sep.IntIndexInput; -import org.apache.lucene.index.codecs.sep.IntIndexOutput; -import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; -import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; -import org.apache.lucene.index.codecs.sep.SepPostingsReader; -import org.apache.lucene.index.codecs.sep.SepPostingsWriter; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput; -import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput; -import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; -import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.BlockTermsReader; -import org.apache.lucene.index.codecs.BlockTermsWriter; -import org.apache.lucene.index.codecs.TermsIndexReaderBase; -import org.apache.lucene.index.codecs.TermsIndexWriterBase; -import org.apache.lucene.store.*; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; - -/** - * A silly test codec to verify core support for fixed - * sized int block encoders is working. The int encoder - * used here just writes each block as a series of vInt. - */ - -public class MockFixedIntBlockCodec extends Codec { - - private final int blockSize; - - public MockFixedIntBlockCodec(int blockSize) { - super("MockFixedIntBlock"); - this.blockSize = blockSize; - } - - @Override - public String toString() { - return name + "(blockSize=" + blockSize + ")"; - } - - // only for testing - public IntStreamFactory getIntFactory() { - return new MockIntFactory(blockSize); - } - - public static class MockIntFactory extends IntStreamFactory { - private final int blockSize; - - public MockIntFactory(int blockSize) { - this.blockSize = blockSize; - } - - @Override - public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { - return new FixedIntBlockIndexInput(dir.openInput(fileName, context)) { - - @Override - protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException { - return new BlockReader() { - public void seek(long pos) {} - public void readBlock() throws IOException { - for(int i=0;i= 0; - out.writeVInt(buffer[i]); - } - } - }; - success = true; - return ret; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(out); - } - } - } - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new MockIntFactory(blockSize)); - - boolean success = false; - TermsIndexWriterBase indexWriter; - try { - indexWriter = new FixedGapTermsIndexWriter(state); - success = true; - } finally { - if (!success) { - postingsWriter.close(); - } - } - - success = false; - try { - FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, postingsWriter); - success = true; - return ret; - } finally { - if (!success) { - try { - postingsWriter.close(); - } finally { - indexWriter.close(); - } - } - } - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, - state.segmentInfo, - state.context, - new MockIntFactory(blockSize), state.codecId); - - TermsIndexReaderBase indexReader; - boolean success = false; - try { - indexReader = new FixedGapTermsIndexReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator(), state.codecId, - IOContext.DEFAULT); - success = true; - } finally { - if (!success) { - postingsReader.close(); - } - } - - success = false; - try { - FieldsProducer ret = new BlockTermsReader(indexReader, - state.dir, - state.fieldInfos, - state.segmentInfo.name, - postingsReader, - state.context, - StandardCodec.TERMS_CACHE_SIZE, - state.codecId); - success = true; - return ret; - } finally { - if (!success) { - try { - postingsReader.close(); - } finally { - indexReader.close(); - } - } - } - } - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { - SepPostingsReader.files(segmentInfo, codecId, files); - BlockTermsReader.files(dir, segmentInfo, codecId, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); - } - - @Override - public void getExtensions(Set extensions) { - SepPostingsWriter.getExtensions(extensions); - BlockTermsReader.getExtensions(extensions); - FixedGapTermsIndexReader.getIndexExtensions(extensions); - SepDocValuesConsumer.getExtensions(extensions); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new SepDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new SepDocValuesProducer(state); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java 2011-11-01 18:41:12.548614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,253 +0,0 @@ -package org.apache.lucene.index.codecs.mockintblock; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.sep.IntStreamFactory; -import org.apache.lucene.index.codecs.sep.IntIndexInput; -import org.apache.lucene.index.codecs.sep.IntIndexOutput; -import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; -import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; -import org.apache.lucene.index.codecs.sep.SepPostingsReader; -import org.apache.lucene.index.codecs.sep.SepPostingsWriter; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput; -import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput; -import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; -import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.BlockTermsReader; -import org.apache.lucene.index.codecs.BlockTermsWriter; -import org.apache.lucene.index.codecs.TermsIndexReaderBase; -import org.apache.lucene.index.codecs.TermsIndexWriterBase; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; - -/** - * A silly test codec to verify core support for variable - * sized int block encoders is working. The int encoder - * used here writes baseBlockSize ints at once, if the first - * int is <= 3, else 2*baseBlockSize. - */ - -public class MockVariableIntBlockCodec extends Codec { - private final int baseBlockSize; - - public MockVariableIntBlockCodec(int baseBlockSize) { - super("MockVariableIntBlock"); - this.baseBlockSize = baseBlockSize; - } - - @Override - public String toString() { - return name + "(baseBlockSize="+ baseBlockSize + ")"; - } - - public static class MockIntFactory extends IntStreamFactory { - - private final int baseBlockSize; - - public MockIntFactory(int baseBlockSize) { - this.baseBlockSize = baseBlockSize; - } - - @Override - public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { - final IndexInput in = dir.openInput(fileName, context); - final int baseBlockSize = in.readInt(); - return new VariableIntBlockIndexInput(in) { - - @Override - protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException { - return new BlockReader() { - public void seek(long pos) {} - public int readBlock() throws IOException { - buffer[0] = in.readVInt(); - final int count = buffer[0] <= 3 ? baseBlockSize-1 : 2*baseBlockSize-1; - assert buffer.length >= count: "buffer.length=" + buffer.length + " count=" + count; - for(int i=0;i= 0; - buffer[pendingCount++] = value; - // silly variable block length int encoder: if - // first value <= 3, we write N vints at once; - // else, 2*N - final int flushAt = buffer[0] <= 3 ? baseBlockSize : 2*baseBlockSize; - - // intentionally be non-causal here: - if (pendingCount == flushAt+1) { - for(int i=0;i files) throws IOException { - SepPostingsReader.files(segmentInfo, codecId, files); - BlockTermsReader.files(dir, segmentInfo, codecId, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); - } - - @Override - public void getExtensions(Set extensions) { - SepPostingsWriter.getExtensions(extensions); - BlockTermsReader.getExtensions(extensions); - FixedGapTermsIndexReader.getIndexExtensions(extensions); - SepDocValuesConsumer.getExtensions(extensions); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new SepDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new SepDocValuesProducer(state); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java 2011-11-01 18:41:12.552614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,479 +0,0 @@ -package org.apache.lucene.index.codecs.mockrandom; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Random; -import java.util.Set; - -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.codecs.BlockTreeTermsReader; -import org.apache.lucene.index.codecs.BlockTreeTermsWriter; -import org.apache.lucene.index.codecs.BlockTermsReader; -import org.apache.lucene.index.codecs.BlockTermsWriter; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; -import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.TermStats; -import org.apache.lucene.index.codecs.TermsIndexReaderBase; -import org.apache.lucene.index.codecs.TermsIndexWriterBase; -import org.apache.lucene.index.codecs.VariableGapTermsIndexReader; -import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter; -import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec; -import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec; -import org.apache.lucene.index.codecs.mocksep.MockSingleIntFactory; -import org.apache.lucene.index.codecs.pulsing.PulsingPostingsReader; -import org.apache.lucene.index.codecs.pulsing.PulsingPostingsWriter; -import org.apache.lucene.index.codecs.sep.IntIndexInput; -import org.apache.lucene.index.codecs.sep.IntIndexOutput; -import org.apache.lucene.index.codecs.sep.IntStreamFactory; -import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; -import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; -import org.apache.lucene.index.codecs.sep.SepPostingsReader; -import org.apache.lucene.index.codecs.sep.SepPostingsWriter; -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; -import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util._TestUtil; - -/** - * Randomly combines terms index impl w/ postings impls. - */ - -public class MockRandomCodec extends Codec { - private final boolean useSepDocValues; - private final Random seedRandom; - private final String SEED_EXT = "sd"; - - public MockRandomCodec(Random random) { - super("MockRandom"); - this.useSepDocValues = random.nextBoolean(); - this.seedRandom = new Random(random.nextLong()); - } - - // Chooses random IntStreamFactory depending on file's extension - private static class MockIntStreamFactory extends IntStreamFactory { - private final int salt; - private final List delegates = new ArrayList(); - - public MockIntStreamFactory(Random random) { - salt = random.nextInt(); - delegates.add(new MockSingleIntFactory()); - final int blockSize = _TestUtil.nextInt(random, 1, 2000); - delegates.add(new MockFixedIntBlockCodec.MockIntFactory(blockSize)); - final int baseBlockSize = _TestUtil.nextInt(random, 1, 127); - delegates.add(new MockVariableIntBlockCodec.MockIntFactory(baseBlockSize)); - // TODO: others - } - - private static String getExtension(String fileName) { - final int idx = fileName.indexOf('.'); - assert idx != -1; - return fileName.substring(idx); - } - - @Override - public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { - // Must only use extension, because IW.addIndexes can - // rename segment! - final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size()); - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: read using int factory " + f + " from fileName=" + fileName); - } - return f.openInput(dir, fileName, context); - } - - @Override - public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context) throws IOException { - final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size()); - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: write using int factory " + f + " to fileName=" + fileName); - } - return f.createOutput(dir, fileName, context); - } - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - // we pull this before the seed intentionally: because its not consumed at runtime - // (the skipInterval is written into postings header) - int skipInterval = _TestUtil.nextInt(seedRandom, 2, 10); - - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: skipInterval=" + skipInterval); - } - - final long seed = seedRandom.nextLong(); - - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: writing to seg=" + state.segmentName + " codecID=" + state.codecId + " seed=" + seed); - } - - final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SEED_EXT); - final IndexOutput out = state.directory.createOutput(seedFileName, state.context); - try { - out.writeLong(seed); - } finally { - out.close(); - } - - final Random random = new Random(seed); - - random.nextInt(); // consume a random for buffersize - - PostingsWriterBase postingsWriter; - if (random.nextBoolean()) { - postingsWriter = new SepPostingsWriter(state, new MockIntStreamFactory(random), skipInterval); - } else { - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: writing Standard postings"); - } - postingsWriter = new StandardPostingsWriter(state, skipInterval); - } - - if (random.nextBoolean()) { - final int totTFCutoff = _TestUtil.nextInt(random, 1, 20); - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: writing pulsing postings with totTFCutoff=" + totTFCutoff); - } - postingsWriter = new PulsingPostingsWriter(totTFCutoff, postingsWriter); - } - - final FieldsConsumer fields; - - if (random.nextBoolean()) { - // Use BlockTree terms dict - - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: writing BlockTree terms dict"); - } - - // TODO: would be nice to allow 1 but this is very - // slow to write - final int minTermsInBlock = _TestUtil.nextInt(random, 2, 100); - final int maxTermsInBlock = Math.max(2, (minTermsInBlock-1)*2 + random.nextInt(100)); - - boolean success = false; - try { - fields = new BlockTreeTermsWriter(state, postingsWriter, minTermsInBlock, maxTermsInBlock); - success = true; - } finally { - if (!success) { - postingsWriter.close(); - } - } - } else { - - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: writing Block terms dict"); - } - - boolean success = false; - - final TermsIndexWriterBase indexWriter; - try { - if (random.nextBoolean()) { - state.termIndexInterval = _TestUtil.nextInt(random, 1, 100); - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: fixed-gap terms index (tii=" + state.termIndexInterval + ")"); - } - indexWriter = new FixedGapTermsIndexWriter(state); - } else { - final VariableGapTermsIndexWriter.IndexTermSelector selector; - final int n2 = random.nextInt(3); - if (n2 == 0) { - final int tii = _TestUtil.nextInt(random, 1, 100); - selector = new VariableGapTermsIndexWriter.EveryNTermSelector(tii); - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: variable-gap terms index (tii=" + tii + ")"); - } - } else if (n2 == 1) { - final int docFreqThresh = _TestUtil.nextInt(random, 2, 100); - final int tii = _TestUtil.nextInt(random, 1, 100); - selector = new VariableGapTermsIndexWriter.EveryNOrDocFreqTermSelector(docFreqThresh, tii); - } else { - final long seed2 = random.nextLong(); - final int gap = _TestUtil.nextInt(random, 2, 40); - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: random-gap terms index (max gap=" + gap + ")"); - } - selector = new VariableGapTermsIndexWriter.IndexTermSelector() { - final Random rand = new Random(seed2); - - @Override - public boolean isIndexTerm(BytesRef term, TermStats stats) { - return rand.nextInt(gap) == gap/2; - } - - @Override - public void newField(FieldInfo fieldInfo) { - } - }; - } - indexWriter = new VariableGapTermsIndexWriter(state, selector); - } - success = true; - } finally { - if (!success) { - postingsWriter.close(); - } - } - - success = false; - try { - fields = new BlockTermsWriter(indexWriter, state, postingsWriter); - success = true; - } finally { - if (!success) { - try { - postingsWriter.close(); - } finally { - indexWriter.close(); - } - } - } - } - - return fields; - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - - final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, SEED_EXT); - final IndexInput in = state.dir.openInput(seedFileName, state.context); - final long seed = in.readLong(); - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " codecID=" + state.codecId + " seed=" + seed); - } - in.close(); - - final Random random = new Random(seed); - - int readBufferSize = _TestUtil.nextInt(random, 1, 4096); - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: readBufferSize=" + readBufferSize); - } - - PostingsReaderBase postingsReader; - - if (random.nextBoolean()) { - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: reading Sep postings"); - } - postingsReader = new SepPostingsReader(state.dir, state.segmentInfo, - state.context, new MockIntStreamFactory(random), state.codecId); - } else { - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: reading Standard postings"); - } - postingsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); - } - - if (random.nextBoolean()) { - final int totTFCutoff = _TestUtil.nextInt(random, 1, 20); - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: reading pulsing postings with totTFCutoff=" + totTFCutoff); - } - postingsReader = new PulsingPostingsReader(postingsReader); - } - - final FieldsProducer fields; - - if (random.nextBoolean()) { - // Use BlockTree terms dict - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: reading BlockTree terms dict"); - } - - boolean success = false; - try { - fields = new BlockTreeTermsReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - postingsReader, - state.context, - state.codecId, - state.termsIndexDivisor); - success = true; - } finally { - if (!success) { - postingsReader.close(); - } - } - } else { - - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: reading Block terms dict"); - } - final TermsIndexReaderBase indexReader; - boolean success = false; - try { - final boolean doFixedGap = random.nextBoolean(); - - // randomness diverges from writer, here: - if (state.termsIndexDivisor != -1) { - state.termsIndexDivisor = _TestUtil.nextInt(random, 1, 10); - } - - if (doFixedGap) { - // if termsIndexDivisor is set to -1, we should not touch it. It means a - // test explicitly instructed not to load the terms index. - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: fixed-gap terms index (divisor=" + state.termsIndexDivisor + ")"); - } - indexReader = new FixedGapTermsIndexReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId, state.context); - } else { - final int n2 = random.nextInt(3); - if (n2 == 1) { - random.nextInt(); - } else if (n2 == 2) { - random.nextLong(); - } - if (LuceneTestCase.VERBOSE) { - System.out.println("MockRandomCodec: variable-gap terms index (divisor=" + state.termsIndexDivisor + ")"); - } - indexReader = new VariableGapTermsIndexReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - state.termsIndexDivisor, - state.codecId, state.context); - - } - - success = true; - } finally { - if (!success) { - postingsReader.close(); - } - } - - final int termsCacheSize = _TestUtil.nextInt(random, 1, 1024); - - success = false; - try { - fields = new BlockTermsReader(indexReader, - state.dir, - state.fieldInfos, - state.segmentInfo.name, - postingsReader, - state.context, - termsCacheSize, - state.codecId); - success = true; - } finally { - if (!success) { - try { - postingsReader.close(); - } finally { - indexReader.close(); - } - } - } - } - - return fields; - } - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { - final String seedFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SEED_EXT); - files.add(seedFileName); - SepPostingsReader.files(segmentInfo, codecId, files); - StandardPostingsReader.files(dir, segmentInfo, codecId, files); - BlockTermsReader.files(dir, segmentInfo, codecId, files); - BlockTreeTermsReader.files(dir, segmentInfo, codecId, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - if (useSepDocValues) { - SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); - } else { - DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files); - } - // hackish! - Iterator it = files.iterator(); - while(it.hasNext()) { - final String file = it.next(); - if (!dir.fileExists(file)) { - it.remove(); - } - } - //System.out.println("MockRandom.files return " + files); - } - - @Override - public void getExtensions(Set extensions) { - SepPostingsWriter.getExtensions(extensions); - BlockTermsReader.getExtensions(extensions); - BlockTreeTermsReader.getExtensions(extensions); - FixedGapTermsIndexReader.getIndexExtensions(extensions); - VariableGapTermsIndexReader.getIndexExtensions(extensions); - if (useSepDocValues) { - SepDocValuesConsumer.getExtensions(extensions); - } else { - DefaultDocValuesConsumer.getExtensions(extensions); - } - extensions.add(SEED_EXT); - //System.out.println("MockRandom.getExtensions return " + extensions); - } - - // can we make this more evil? - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - if (useSepDocValues) { - return new SepDocValuesConsumer(state); - } else { - return new DefaultDocValuesConsumer(state); - } - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - if (useSepDocValues) { - return new SepDocValuesProducer(state); - } else { - return new DefaultDocValuesProducer(state); - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java 2011-11-01 18:41:12.548614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,166 +0,0 @@ -package org.apache.lucene.index.codecs.mocksep; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Set; - -import org.apache.lucene.index.PerDocWriteState; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; -import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.BlockTermsReader; -import org.apache.lucene.index.codecs.BlockTermsWriter; -import org.apache.lucene.index.codecs.TermsIndexReaderBase; -import org.apache.lucene.index.codecs.TermsIndexWriterBase; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer; -import org.apache.lucene.index.codecs.sep.SepDocValuesProducer; -import org.apache.lucene.index.codecs.sep.SepPostingsWriter; -import org.apache.lucene.index.codecs.sep.SepPostingsReader; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; - -/** - * A silly codec that simply writes each file separately as - * single vInts. Don't use this (performance will be poor)! - * This is here just to test the core sep codec - * classes. - */ -public class MockSepCodec extends Codec { - - public MockSepCodec() { - super("MockSep"); - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - - PostingsWriterBase postingsWriter = new SepPostingsWriter(state, new MockSingleIntFactory()); - - boolean success = false; - TermsIndexWriterBase indexWriter; - try { - indexWriter = new FixedGapTermsIndexWriter(state); - success = true; - } finally { - if (!success) { - postingsWriter.close(); - } - } - - success = false; - try { - FieldsConsumer ret = new BlockTermsWriter(indexWriter, state, postingsWriter); - success = true; - return ret; - } finally { - if (!success) { - try { - postingsWriter.close(); - } finally { - indexWriter.close(); - } - } - } - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - - PostingsReaderBase postingsReader = new SepPostingsReader(state.dir, state.segmentInfo, - state.context, new MockSingleIntFactory(), state.codecId); - - TermsIndexReaderBase indexReader; - boolean success = false; - try { - indexReader = new FixedGapTermsIndexReader(state.dir, - state.fieldInfos, - state.segmentInfo.name, - state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId, state.context); - success = true; - } finally { - if (!success) { - postingsReader.close(); - } - } - - success = false; - try { - FieldsProducer ret = new BlockTermsReader(indexReader, - state.dir, - state.fieldInfos, - state.segmentInfo.name, - postingsReader, - state.context, - StandardCodec.TERMS_CACHE_SIZE, - state.codecId); - success = true; - return ret; - } finally { - if (!success) { - try { - postingsReader.close(); - } finally { - indexReader.close(); - } - } - } - } - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { - SepPostingsReader.files(segmentInfo, codecId, files); - BlockTermsReader.files(dir, segmentInfo, codecId, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files); - SepDocValuesConsumer.files(dir, segmentInfo, codecId, files); - } - - @Override - public void getExtensions(Set extensions) { - getSepExtensions(extensions); - } - - public static void getSepExtensions(Set extensions) { - SepPostingsWriter.getExtensions(extensions); - BlockTermsReader.getExtensions(extensions); - FixedGapTermsIndexReader.getIndexExtensions(extensions); - SepDocValuesConsumer.getExtensions(extensions); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new SepDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new SepDocValuesProducer(state); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java 2011-11-01 18:41:12.548614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,38 +0,0 @@ -package org.apache.lucene.index.codecs.mocksep; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.index.codecs.sep.IntStreamFactory; -import org.apache.lucene.index.codecs.sep.IntIndexInput; -import org.apache.lucene.index.codecs.sep.IntIndexOutput; - -import java.io.IOException; - -/** @lucene.experimental */ -public class MockSingleIntFactory extends IntStreamFactory { - @Override - public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { - return new MockSingleIntIndexInput(dir, fileName, context); - } - @Override - public IntIndexOutput createOutput(Directory dir, String fileName, IOContext context) throws IOException { - return new MockSingleIntIndexOutput(dir, fileName, context); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java 2011-11-01 18:41:12.548614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,114 +0,0 @@ -package org.apache.lucene.index.codecs.mocksep; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.codecs.sep.IntIndexInput; -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; -import org.apache.lucene.util.CodecUtil; - -/** Reads IndexInputs written with {@link - * MockSingleIntIndexOutput}. NOTE: this class is just for - * demonstration puprposes (it is a very slow way to read a - * block of ints). - * - * @lucene.experimental - */ -public class MockSingleIntIndexInput extends IntIndexInput { - private final IndexInput in; - - public MockSingleIntIndexInput(Directory dir, String fileName, IOContext context) - throws IOException { - in = dir.openInput(fileName, context); - CodecUtil.checkHeader(in, MockSingleIntIndexOutput.CODEC, - MockSingleIntIndexOutput.VERSION_START, - MockSingleIntIndexOutput.VERSION_START); - } - - @Override - public Reader reader() throws IOException { - return new Reader((IndexInput) in.clone()); - } - - @Override - public void close() throws IOException { - in.close(); - } - - public static class Reader extends IntIndexInput.Reader { - // clone: - private final IndexInput in; - - public Reader(IndexInput in) { - this.in = in; - } - - /** Reads next single int */ - @Override - public int next() throws IOException { - //System.out.println("msii.next() fp=" + in.getFilePointer() + " vs " + in.length()); - return in.readVInt(); - } - } - - class Index extends IntIndexInput.Index { - private long fp; - - @Override - public void read(DataInput indexIn, boolean absolute) - throws IOException { - if (absolute) { - fp = indexIn.readVLong(); - } else { - fp += indexIn.readVLong(); - } - } - - @Override - public void set(IntIndexInput.Index other) { - fp = ((Index) other).fp; - } - - @Override - public void seek(IntIndexInput.Reader other) throws IOException { - ((Reader) other).in.seek(fp); - } - - @Override - public String toString() { - return Long.toString(fp); - } - - @Override - public Object clone() { - Index other = new Index(); - other.fp = fp; - return other; - } - } - - @Override - public Index index() { - return new Index(); - } -} - diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java 2011-11-01 18:41:12.548614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,104 +0,0 @@ -package org.apache.lucene.index.codecs.mocksep; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.CodecUtil; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.index.codecs.sep.IntIndexOutput; -import java.io.IOException; - -/** Writes ints directly to the file (not in blocks) as - * vInt. - * - * @lucene.experimental -*/ -public class MockSingleIntIndexOutput extends IntIndexOutput { - private final IndexOutput out; - final static String CODEC = "SINGLE_INTS"; - final static int VERSION_START = 0; - final static int VERSION_CURRENT = VERSION_START; - - public MockSingleIntIndexOutput(Directory dir, String fileName, IOContext context) throws IOException { - out = dir.createOutput(fileName, context); - boolean success = false; - try { - CodecUtil.writeHeader(out, CODEC, VERSION_CURRENT); - success = true; - } finally { - if (!success) { - IOUtils.closeWhileHandlingException(out); - } - } - } - - /** Write an int to the primary file */ - @Override - public void write(int v) throws IOException { - assert v >= 0; - out.writeVInt(v); - } - - @Override - public Index index() { - return new Index(); - } - - @Override - public void close() throws IOException { - out.close(); - } - - @Override - public String toString() { - return "MockSingleIntIndexOutput fp=" + out.getFilePointer(); - } - - private class Index extends IntIndexOutput.Index { - long fp; - long lastFP; - @Override - public void mark() { - fp = out.getFilePointer(); - } - @Override - public void copyFrom(IntIndexOutput.Index other, boolean copyLast) { - fp = ((Index) other).fp; - if (copyLast) { - lastFP = ((Index) other).fp; - } - } - @Override - public void write(IndexOutput indexOut, boolean absolute) - throws IOException { - if (absolute) { - indexOut.writeVLong(fp); - } else { - indexOut.writeVLong(fp - lastFP); - } - lastFP = fp; - } - - @Override - public String toString() { - return Long.toString(fp); - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java 2011-11-01 18:41:12.552614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,207 +0,0 @@ -package org.apache.lucene.index.codecs.preflexrw; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Comparator; - -import org.apache.lucene.index.CorruptIndexException; -import org.apache.lucene.index.FieldInfo; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.PostingsConsumer; -import org.apache.lucene.index.codecs.TermStats; -import org.apache.lucene.index.codecs.TermsConsumer; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; -import org.apache.lucene.index.codecs.preflex.TermInfo; -import org.apache.lucene.index.codecs.standard.DefaultSkipListWriter; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.IOUtils; - -class PreFlexFieldsWriter extends FieldsConsumer { - - private final TermInfosWriter termsOut; - private final IndexOutput freqOut; - private final IndexOutput proxOut; - private final DefaultSkipListWriter skipListWriter; - private final int totalNumDocs; - - public PreFlexFieldsWriter(SegmentWriteState state) throws IOException { - termsOut = new TermInfosWriter(state.directory, - state.segmentName, - state.fieldInfos, - state.termIndexInterval); - - final String freqFile = IndexFileNames.segmentFileName(state.segmentName, "", PreFlexCodec.FREQ_EXTENSION); - freqOut = state.directory.createOutput(freqFile, state.context); - totalNumDocs = state.numDocs; - - if (state.fieldInfos.hasProx()) { - final String proxFile = IndexFileNames.segmentFileName(state.segmentName, "", PreFlexCodec.PROX_EXTENSION); - proxOut = state.directory.createOutput(proxFile, state.context); - } else { - proxOut = null; - } - - skipListWriter = new DefaultSkipListWriter(termsOut.skipInterval, - termsOut.maxSkipLevels, - totalNumDocs, - freqOut, - proxOut); - //System.out.println("\nw start seg=" + segment); - } - - @Override - public TermsConsumer addField(FieldInfo field) throws IOException { - assert field.number != -1; - //System.out.println("w field=" + field.name + " storePayload=" + field.storePayloads + " number=" + field.number); - return new PreFlexTermsWriter(field); - } - - @Override - public void close() throws IOException { - IOUtils.close(termsOut, freqOut, proxOut); - } - - private class PreFlexTermsWriter extends TermsConsumer { - private final FieldInfo fieldInfo; - private final boolean omitTF; - private final boolean storePayloads; - - private final TermInfo termInfo = new TermInfo(); - private final PostingsWriter postingsWriter = new PostingsWriter(); - - public PreFlexTermsWriter(FieldInfo fieldInfo) { - this.fieldInfo = fieldInfo; - omitTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY; - storePayloads = fieldInfo.storePayloads; - } - - private class PostingsWriter extends PostingsConsumer { - private int lastDocID; - private int lastPayloadLength = -1; - private int lastPosition; - private int df; - - public PostingsWriter reset() { - df = 0; - lastDocID = 0; - lastPayloadLength = -1; - return this; - } - - @Override - public void startDoc(int docID, int termDocFreq) throws IOException { - //System.out.println(" w doc=" + docID); - - final int delta = docID - lastDocID; - if (docID < 0 || (df > 0 && delta <= 0)) { - throw new CorruptIndexException("docs out of order (" + docID + " <= " + lastDocID + " )"); - } - - if ((++df % termsOut.skipInterval) == 0) { - skipListWriter.setSkipData(lastDocID, storePayloads, lastPayloadLength); - skipListWriter.bufferSkip(df); - } - - lastDocID = docID; - - assert docID < totalNumDocs: "docID=" + docID + " totalNumDocs=" + totalNumDocs; - - if (omitTF) { - freqOut.writeVInt(delta); - } else { - final int code = delta << 1; - if (termDocFreq == 1) { - freqOut.writeVInt(code|1); - } else { - freqOut.writeVInt(code); - freqOut.writeVInt(termDocFreq); - } - } - lastPosition = 0; - } - - @Override - public void addPosition(int position, BytesRef payload) throws IOException { - assert proxOut != null; - - //System.out.println(" w pos=" + position + " payl=" + payload); - final int delta = position - lastPosition; - lastPosition = position; - - if (storePayloads) { - final int payloadLength = payload == null ? 0 : payload.length; - if (payloadLength != lastPayloadLength) { - //System.out.println(" write payload len=" + payloadLength); - lastPayloadLength = payloadLength; - proxOut.writeVInt((delta<<1)|1); - proxOut.writeVInt(payloadLength); - } else { - proxOut.writeVInt(delta << 1); - } - if (payloadLength > 0) { - proxOut.writeBytes(payload.bytes, payload.offset, payload.length); - } - } else { - proxOut.writeVInt(delta); - } - } - - @Override - public void finishDoc() throws IOException { - } - } - - @Override - public PostingsConsumer startTerm(BytesRef text) throws IOException { - //System.out.println(" w term=" + text.utf8ToString()); - skipListWriter.resetSkip(); - termInfo.freqPointer = freqOut.getFilePointer(); - if (proxOut != null) { - termInfo.proxPointer = proxOut.getFilePointer(); - } - return postingsWriter.reset(); - } - - @Override - public void finishTerm(BytesRef text, TermStats stats) throws IOException { - if (stats.docFreq > 0) { - long skipPointer = skipListWriter.writeSkip(freqOut); - termInfo.docFreq = stats.docFreq; - termInfo.skipOffset = (int) (skipPointer - termInfo.freqPointer); - //System.out.println(" w finish term=" + text.utf8ToString() + " fnum=" + fieldInfo.number); - termsOut.add(fieldInfo.number, - text, - termInfo); - } - } - - @Override - public void finish(long sumTotalTermCount, long sumDocFreq, int docCount) throws IOException { - } - - @Override - public Comparator getComparator() throws IOException { - return BytesRef.getUTF8SortedAsUTF16Comparator(); - } - } -} \ No newline at end of file diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java 2011-11-02 10:34:14.684585067 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,76 +0,0 @@ -package org.apache.lucene.index.codecs.preflexrw; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.index.SegmentWriteState; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; -import org.apache.lucene.index.codecs.preflex.PreFlexFields; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.util.LuceneTestCase; - -/** Codec, only for testing, that can write and read the - * pre-flex index format. - * - * @lucene.experimental - */ -public class PreFlexRWCodec extends PreFlexCodec { - - public PreFlexRWCodec() { - // NOTE: we impersonate the PreFlex codec so that it can - // read the segments we write! - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - return new PreFlexFieldsWriter(state); - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - - // Whenever IW opens readers, eg for merging, we have to - // keep terms order in UTF16: - - return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor) { - @Override - protected boolean sortTermsByUnicode() { - // We carefully peek into stack track above us: if - // we are part of a "merge", we must sort by UTF16: - boolean unicodeSortOrder = true; - - StackTraceElement[] trace = new Exception().getStackTrace(); - for (int i = 0; i < trace.length; i++) { - //System.out.println(trace[i].getClassName()); - if ("merge".equals(trace[i].getMethodName())) { - unicodeSortOrder = false; - if (LuceneTestCase.VERBOSE) { - System.out.println("NOTE: PreFlexRW codec: forcing legacy UTF16 term sort order"); - } - break; - } - } - - return unicodeSortOrder; - } - }; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java 2011-11-01 18:41:12.552614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,286 +0,0 @@ -package org.apache.lucene.index.codecs.preflexrw; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -import java.io.Closeable; -import java.io.IOException; - -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexFileNames; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; -import org.apache.lucene.index.codecs.preflex.TermInfo; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexOutput; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CharsRef; -import org.apache.lucene.util.IOUtils; -import org.apache.lucene.util.UnicodeUtil; - - -/** This stores a monotonically increasing set of pairs in a - Directory. A TermInfos can be written once, in order. */ - -final class TermInfosWriter implements Closeable { - /** The file format version, a negative number. */ - public static final int FORMAT = -3; - - // Changed strings to true utf8 with length-in-bytes not - // length-in-chars - public static final int FORMAT_VERSION_UTF8_LENGTH_IN_BYTES = -4; - - // NOTE: always change this if you switch to a new format! - public static final int FORMAT_CURRENT = FORMAT_VERSION_UTF8_LENGTH_IN_BYTES; - - private FieldInfos fieldInfos; - private IndexOutput output; - private TermInfo lastTi = new TermInfo(); - private long size; - - // TODO: the default values for these two parameters should be settable from - // IndexWriter. However, once that's done, folks will start setting them to - // ridiculous values and complaining that things don't work well, as with - // mergeFactor. So, let's wait until a number of folks find that alternate - // values work better. Note that both of these values are stored in the - // segment, so that it's safe to change these w/o rebuilding all indexes. - - /** Expert: The fraction of terms in the "dictionary" which should be stored - * in RAM. Smaller values use more memory, but make searching slightly - * faster, while larger values use less memory and make searching slightly - * slower. Searching is typically not dominated by dictionary lookup, so - * tweaking this is rarely useful.*/ - int indexInterval = 128; - - /** Expert: The fraction of {@link TermDocs} entries stored in skip tables, - * used to accelerate {@link TermDocs#skipTo(int)}. Larger values result in - * smaller indexes, greater acceleration, but fewer accelerable cases, while - * smaller values result in bigger indexes, less acceleration and more - * accelerable cases. More detailed experiments would be useful here. */ - int skipInterval = 16; - - /** Expert: The maximum number of skip levels. Smaller values result in - * slightly smaller indexes, but slower skipping in big posting lists. - */ - int maxSkipLevels = 10; - - private long lastIndexPointer; - private boolean isIndex; - private final BytesRef lastTerm = new BytesRef(); - private int lastFieldNumber = -1; - - private TermInfosWriter other; - - TermInfosWriter(Directory directory, String segment, FieldInfos fis, - int interval) - throws IOException { - initialize(directory, segment, fis, interval, false); - boolean success = false; - try { - other = new TermInfosWriter(directory, segment, fis, interval, true); - other.other = this; - success = true; - } finally { - if (!success) { - try { - IOUtils.closeWhileHandlingException(output); - } catch (IOException e) { - // cannot happen since we suppress exceptions - throw new RuntimeException(e); - } - - try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", - (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION - : PreFlexCodec.TERMS_EXTENSION))); - } catch (IOException ignored) { - } - } - } - } - - private TermInfosWriter(Directory directory, String segment, FieldInfos fis, - int interval, boolean isIndex) throws IOException { - initialize(directory, segment, fis, interval, isIndex); - } - - private void initialize(Directory directory, String segment, FieldInfos fis, - int interval, boolean isi) throws IOException { - indexInterval = interval; - fieldInfos = fis; - isIndex = isi; - output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", - (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION - : PreFlexCodec.TERMS_EXTENSION)), IOContext.DEFAULT); - boolean success = false; - try { - output.writeInt(FORMAT_CURRENT); // write format - output.writeLong(0); // leave space for size - output.writeInt(indexInterval); // write indexInterval - output.writeInt(skipInterval); // write skipInterval - output.writeInt(maxSkipLevels); // write maxSkipLevels - assert initUTF16Results(); - success = true; - } finally { - if (!success) { - try { - IOUtils.closeWhileHandlingException(output); - } catch (IOException e) { - // cannot happen since we suppress exceptions - throw new RuntimeException(e); - } - - try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", - (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION - : PreFlexCodec.TERMS_EXTENSION))); - } catch (IOException ignored) { - } - } - } - } - - // Currently used only by assert statements - CharsRef utf16Result1; - CharsRef utf16Result2; - private final BytesRef scratchBytes = new BytesRef(); - - // Currently used only by assert statements - private boolean initUTF16Results() { - utf16Result1 = new CharsRef(10); - utf16Result2 = new CharsRef(10); - return true; - } - - // Currently used only by assert statement - private int compareToLastTerm(int fieldNumber, BytesRef term) { - - if (lastFieldNumber != fieldNumber) { - final int cmp = fieldInfos.fieldName(lastFieldNumber).compareTo(fieldInfos.fieldName(fieldNumber)); - // If there is a field named "" (empty string) then we - // will get 0 on this comparison, yet, it's "OK". But - // it's not OK if two different field numbers map to - // the same name. - if (cmp != 0 || lastFieldNumber != -1) - return cmp; - } - - scratchBytes.copy(term); - assert lastTerm.offset == 0; - UnicodeUtil.UTF8toUTF16(lastTerm.bytes, 0, lastTerm.length, utf16Result1); - - assert scratchBytes.offset == 0; - UnicodeUtil.UTF8toUTF16(scratchBytes.bytes, 0, scratchBytes.length, utf16Result2); - - final int len; - if (utf16Result1.length < utf16Result2.length) - len = utf16Result1.length; - else - len = utf16Result2.length; - - for(int i=0;i, TermInfo> pair to the set. - Term must be lexicographically greater than all previous Terms added. - TermInfo pointers must be positive and greater than all previous.*/ - public void add(int fieldNumber, BytesRef term, TermInfo ti) - throws IOException { - - assert compareToLastTerm(fieldNumber, term) < 0 || - (isIndex && term.length == 0 && lastTerm.length == 0) : - "Terms are out of order: field=" + fieldInfos.fieldName(fieldNumber) + " (number " + fieldNumber + ")" + - " lastField=" + fieldInfos.fieldName(lastFieldNumber) + " (number " + lastFieldNumber + ")" + - " text=" + term.utf8ToString() + " lastText=" + lastTerm.utf8ToString(); - - assert ti.freqPointer >= lastTi.freqPointer: "freqPointer out of order (" + ti.freqPointer + " < " + lastTi.freqPointer + ")"; - assert ti.proxPointer >= lastTi.proxPointer: "proxPointer out of order (" + ti.proxPointer + " < " + lastTi.proxPointer + ")"; - - if (!isIndex && size % indexInterval == 0) - other.add(lastFieldNumber, lastTerm, lastTi); // add an index term - - writeTerm(fieldNumber, term); // write term - - output.writeVInt(ti.docFreq); // write doc freq - output.writeVLong(ti.freqPointer - lastTi.freqPointer); // write pointers - output.writeVLong(ti.proxPointer - lastTi.proxPointer); - - if (ti.docFreq >= skipInterval) { - output.writeVInt(ti.skipOffset); - } - - if (isIndex) { - output.writeVLong(other.output.getFilePointer() - lastIndexPointer); - lastIndexPointer = other.output.getFilePointer(); // write pointer - } - - lastFieldNumber = fieldNumber; - lastTi.set(ti); - size++; - } - - private void writeTerm(int fieldNumber, BytesRef term) - throws IOException { - - //System.out.println(" tiw.write field=" + fieldNumber + " term=" + term.utf8ToString()); - - // TODO: UTF16toUTF8 could tell us this prefix - // Compute prefix in common with last term: - int start = 0; - final int limit = term.length < lastTerm.length ? term.length : lastTerm.length; - while(start < limit) { - if (term.bytes[start+term.offset] != lastTerm.bytes[start+lastTerm.offset]) - break; - start++; - } - - final int length = term.length - start; - output.writeVInt(start); // write shared prefix length - output.writeVInt(length); // write delta length - output.writeBytes(term.bytes, start+term.offset, length); // write delta bytes - output.writeVInt(fieldNumber); // write field num - lastTerm.copy(term); - } - - /** Called to complete TermInfos creation. */ - public void close() throws IOException { - try { - output.seek(4); // write size after format - output.writeLong(size); - } finally { - try { - output.close(); - } finally { - if (!isIndex) { - other.close(); - } - } - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java 2011-11-01 18:41:12.552614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/DocHelper.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,319 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.analysis.MockTokenizer; -import org.apache.lucene.document.BinaryField; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.TextField; -import org.apache.lucene.search.similarities.SimilarityProvider; -import org.apache.lucene.store.Directory; - -import static org.apache.lucene.util.LuceneTestCase.TEST_VERSION_CURRENT; - -class DocHelper { - - public static final FieldType customType; - public static final String FIELD_1_TEXT = "field one text"; - public static final String TEXT_FIELD_1_KEY = "textField1"; - public static Field textField1; - static { - customType = new FieldType(TextField.TYPE_STORED); - textField1 = new Field(TEXT_FIELD_1_KEY, FIELD_1_TEXT, customType); - } - - public static final FieldType customType2; - public static final String FIELD_2_TEXT = "field field field two text"; - //Fields will be lexicographically sorted. So, the order is: field, text, two - public static final int [] FIELD_2_FREQS = {3, 1, 1}; - public static final String TEXT_FIELD_2_KEY = "textField2"; - public static Field textField2; - static { - customType2 = new FieldType(TextField.TYPE_STORED); - customType2.setStoreTermVectors(true); - customType2.setStoreTermVectorPositions(true); - customType2.setStoreTermVectorOffsets(true); - textField2 = new Field(TEXT_FIELD_2_KEY, FIELD_2_TEXT, customType2); - } - - public static final FieldType customType3; - public static final String FIELD_3_TEXT = "aaaNoNorms aaaNoNorms bbbNoNorms"; - public static final String TEXT_FIELD_3_KEY = "textField3"; - public static Field textField3; - - static { - customType3 = new FieldType(TextField.TYPE_STORED); - customType3.setOmitNorms(true); - textField3 = new Field(TEXT_FIELD_3_KEY, FIELD_3_TEXT, customType3); - } - - public static final String KEYWORD_TEXT = "Keyword"; - public static final String KEYWORD_FIELD_KEY = "keyField"; - public static Field keyField; - static { - keyField = new Field(KEYWORD_FIELD_KEY, KEYWORD_TEXT, StringField.TYPE_STORED); - } - - public static final FieldType customType5; - public static final String NO_NORMS_TEXT = "omitNormsText"; - public static final String NO_NORMS_KEY = "omitNorms"; - public static Field noNormsField; - static { - customType5 = new FieldType(TextField.TYPE_STORED); - customType5.setOmitNorms(true); - customType5.setTokenized(false); - noNormsField = new Field(NO_NORMS_KEY, NO_NORMS_TEXT, customType5); - } - - public static final FieldType customType6; - public static final String NO_TF_TEXT = "analyzed with no tf and positions"; - public static final String NO_TF_KEY = "omitTermFreqAndPositions"; - public static Field noTFField; - static { - customType6 = new FieldType(TextField.TYPE_STORED); - customType6.setIndexOptions(IndexOptions.DOCS_ONLY); - noTFField = new Field(NO_TF_KEY, NO_TF_TEXT, customType6); - } - - public static final FieldType customType7; - public static final String UNINDEXED_FIELD_TEXT = "unindexed field text"; - public static final String UNINDEXED_FIELD_KEY = "unIndField"; - public static Field unIndField; - static { - customType7 = new FieldType(); - customType7.setStored(true); - unIndField = new Field(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT, customType7); - } - - - public static final String UNSTORED_1_FIELD_TEXT = "unstored field text"; - public static final String UNSTORED_FIELD_1_KEY = "unStoredField1"; - public static Field unStoredField1 = new Field(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT, TextField.TYPE_UNSTORED); - - public static final FieldType customType8; - public static final String UNSTORED_2_FIELD_TEXT = "unstored field text"; - public static final String UNSTORED_FIELD_2_KEY = "unStoredField2"; - public static Field unStoredField2; - static { - customType8 = new FieldType(TextField.TYPE_UNSTORED); - customType8.setStoreTermVectors(true); - unStoredField2 = new Field(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT, customType8); - } - - public static final String LAZY_FIELD_BINARY_KEY = "lazyFieldBinary"; - public static byte [] LAZY_FIELD_BINARY_BYTES; - public static Field lazyFieldBinary; - - public static final String LAZY_FIELD_KEY = "lazyField"; - public static final String LAZY_FIELD_TEXT = "These are some field bytes"; - public static Field lazyField = new Field(LAZY_FIELD_KEY, LAZY_FIELD_TEXT, customType); - - public static final String LARGE_LAZY_FIELD_KEY = "largeLazyField"; - public static String LARGE_LAZY_FIELD_TEXT; - public static Field largeLazyField; - - //From Issue 509 - public static final String FIELD_UTF1_TEXT = "field one \u4e00text"; - public static final String TEXT_FIELD_UTF1_KEY = "textField1Utf8"; - public static Field textUtfField1 = new Field(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT, customType); - - public static final String FIELD_UTF2_TEXT = "field field field \u4e00two text"; - //Fields will be lexicographically sorted. So, the order is: field, text, two - public static final int [] FIELD_UTF2_FREQS = {3, 1, 1}; - public static final String TEXT_FIELD_UTF2_KEY = "textField2Utf8"; - public static Field textUtfField2 = new Field(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT, customType2); - - - - - public static Map nameValues = null; - - // ordered list of all the fields... - // could use LinkedHashMap for this purpose if Java1.4 is OK - public static Field[] fields = new Field[] { - textField1, - textField2, - textField3, - keyField, - noNormsField, - noTFField, - unIndField, - unStoredField1, - unStoredField2, - textUtfField1, - textUtfField2, - lazyField, - lazyFieldBinary,//placeholder for binary field, since this is null. It must be second to last. - largeLazyField//placeholder for large field, since this is null. It must always be last - }; - - public static Map all =new HashMap(); - public static Map indexed =new HashMap(); - public static Map stored =new HashMap(); - public static Map unstored=new HashMap(); - public static Map unindexed=new HashMap(); - public static Map termvector=new HashMap(); - public static Map notermvector=new HashMap(); - public static Map lazy= new HashMap(); - public static Map noNorms=new HashMap(); - public static Map noTf=new HashMap(); - - static { - //Initialize the large Lazy Field - StringBuilder buffer = new StringBuilder(); - for (int i = 0; i < 10000; i++) - { - buffer.append("Lazily loading lengths of language in lieu of laughing "); - } - - try { - LAZY_FIELD_BINARY_BYTES = "These are some binary field bytes".getBytes("UTF8"); - } catch (UnsupportedEncodingException e) { - } - lazyFieldBinary = new BinaryField(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES); - fields[fields.length - 2] = lazyFieldBinary; - LARGE_LAZY_FIELD_TEXT = buffer.toString(); - largeLazyField = new Field(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT, customType); - fields[fields.length - 1] = largeLazyField; - for (int i=0; i map, IndexableField field) { - map.put(field.name(), field); - } - - - static - { - nameValues = new HashMap(); - nameValues.put(TEXT_FIELD_1_KEY, FIELD_1_TEXT); - nameValues.put(TEXT_FIELD_2_KEY, FIELD_2_TEXT); - nameValues.put(TEXT_FIELD_3_KEY, FIELD_3_TEXT); - nameValues.put(KEYWORD_FIELD_KEY, KEYWORD_TEXT); - nameValues.put(NO_NORMS_KEY, NO_NORMS_TEXT); - nameValues.put(NO_TF_KEY, NO_TF_TEXT); - nameValues.put(UNINDEXED_FIELD_KEY, UNINDEXED_FIELD_TEXT); - nameValues.put(UNSTORED_FIELD_1_KEY, UNSTORED_1_FIELD_TEXT); - nameValues.put(UNSTORED_FIELD_2_KEY, UNSTORED_2_FIELD_TEXT); - nameValues.put(LAZY_FIELD_KEY, LAZY_FIELD_TEXT); - nameValues.put(LAZY_FIELD_BINARY_KEY, LAZY_FIELD_BINARY_BYTES); - nameValues.put(LARGE_LAZY_FIELD_KEY, LARGE_LAZY_FIELD_TEXT); - nameValues.put(TEXT_FIELD_UTF1_KEY, FIELD_UTF1_TEXT); - nameValues.put(TEXT_FIELD_UTF2_KEY, FIELD_UTF2_TEXT); - } - - /** - * Adds the fields above to a document - * @param doc The document to write - */ - public static void setupDoc(Document doc) { - for (int i=0; i= remainder ? remainder : bytesInBuffer; - System.arraycopy(buffer, bufferOffset, dest, destOffset, bytesToCopy); - destOffset += bytesToCopy; - start += bytesToCopy; - remainder -= bytesToCopy; - } - pointer += len; - } - - @Override - public void close() { - // ignore - } - - @Override - protected void seekInternal(long pos) { - pointer = (int) pos; - } - - @Override - public long length() { - return length; - } - -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java 2011-11-01 18:41:12.552614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,111 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Random; -import java.util.Map; - -import org.apache.lucene.util._TestUtil; - -public class MockRandomMergePolicy extends MergePolicy { - private final Random random; - - public MockRandomMergePolicy(Random random) { - // fork a private random, since we are called - // unpredictably from threads: - this.random = new Random(random.nextLong()); - } - - @Override - public MergeSpecification findMerges(SegmentInfos segmentInfos) { - MergeSpecification mergeSpec = null; - //System.out.println("MRMP: findMerges sis=" + segmentInfos); - - if (segmentInfos.size() > 1 && random.nextInt(5) == 3) { - - List segments = new ArrayList(segmentInfos.asList()); - Collections.shuffle(segments, random); - - // TODO: sometimes make more than 1 merge? - mergeSpec = new MergeSpecification(); - final int segsToMerge = _TestUtil.nextInt(random, 1, segmentInfos.size()); - mergeSpec.add(new OneMerge(segments.subList(0, segsToMerge))); - } - - return mergeSpec; - } - - @Override - public MergeSpecification findMergesForOptimize( - SegmentInfos segmentInfos, int maxSegmentCount, Map segmentsToOptimize) - throws CorruptIndexException, IOException { - - final List eligibleSegments = new ArrayList(); - for(SegmentInfo info : segmentInfos) { - if (segmentsToOptimize.containsKey(info)) { - eligibleSegments.add(info); - } - } - - //System.out.println("MRMP: findMergesForOptimize sis=" + segmentInfos + " eligible=" + eligibleSegments); - MergeSpecification mergeSpec = null; - if (eligibleSegments.size() > 1 || (eligibleSegments.size() == 1 && eligibleSegments.get(0).hasDeletions())) { - mergeSpec = new MergeSpecification(); - // Already shuffled having come out of a set but - // shuffle again for good measure: - Collections.shuffle(eligibleSegments, random); - int upto = 0; - while(upto < eligibleSegments.size()) { - int max = Math.min(10, eligibleSegments.size()-upto); - int inc = max <= 2 ? max : _TestUtil.nextInt(random, 2, max); - mergeSpec.add(new OneMerge(eligibleSegments.subList(upto, upto+inc))); - upto += inc; - } - } - - if (mergeSpec != null) { - for(OneMerge merge : mergeSpec.merges) { - for(SegmentInfo info : merge.segments) { - assert segmentsToOptimize.containsKey(info); - } - } - } - return mergeSpec; - } - - @Override - public MergeSpecification findMergesToExpungeDeletes( - SegmentInfos segmentInfos) - throws CorruptIndexException, IOException { - return findMerges(segmentInfos); - } - - @Override - public void close() { - } - - @Override - public boolean useCompoundFile(SegmentInfos infos, SegmentInfo mergedInfo) throws IOException { - // 80% of the time we create CFS: - return random.nextInt(5) != 1; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java 2011-11-01 18:41:12.552614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,107 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Random; - -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.memory.MemoryCodec; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; -import org.apache.lucene.index.codecs.pulsing.PulsingCodec; -import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.lucene.util._TestUtil; - -/** - * CodecProvider that assigns per-field random codecs. - *

    - * The same field/codec assignment will happen regardless of order, - * a hash is computed up front that determines the mapping. - * This means fields can be put into things like HashSets and added to - * documents in different orders and the test will still be deterministic - * and reproducable. - */ -public class RandomCodecProvider extends CodecProvider { - private List knownCodecs = new ArrayList(); - private Map previousMappings = new HashMap(); - private final int perFieldSeed; - - public RandomCodecProvider(Random random, boolean useNoMemoryExpensiveCodec) { - this.perFieldSeed = random.nextInt(); - // TODO: make it possible to specify min/max iterms per - // block via CL: - int minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); - int maxItemsPerBlock = 2*(Math.max(2, minItemsPerBlock-1)) + random.nextInt(100); - register(new StandardCodec(minItemsPerBlock, maxItemsPerBlock)); - register(new PreFlexCodec()); - // TODO: make it possible to specify min/max iterms per - // block via CL: - minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); - maxItemsPerBlock = 2*(Math.max(1, minItemsPerBlock-1)) + random.nextInt(100); - register(new PulsingCodec( 1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock)); - if (!useNoMemoryExpensiveCodec) { - register(new SimpleTextCodec()); - register(new MemoryCodec()); - } - Collections.shuffle(knownCodecs, random); - } - - @Override - public synchronized void register(Codec codec) { - if (!codec.name.equals("PreFlex")) - knownCodecs.add(codec); - super.register(codec); - } - - @Override - public synchronized void unregister(Codec codec) { - knownCodecs.remove(codec); - super.unregister(codec); - } - - @Override - public synchronized String getFieldCodec(String name) { - Codec codec = previousMappings.get(name); - if (codec == null) { - codec = knownCodecs.get(Math.abs(perFieldSeed ^ name.hashCode()) % knownCodecs.size()); - if (codec instanceof SimpleTextCodec && perFieldSeed % 5 != 0) { - // make simpletext rarer, choose again - codec = knownCodecs.get(Math.abs(perFieldSeed ^ name.toUpperCase(Locale.ENGLISH).hashCode()) % knownCodecs.size()); - } - previousMappings.put(name, codec); - } - return codec.name; - } - - @Override - public synchronized boolean hasFieldCodec(String name) { - return true; // we have a codec for every field - } - - @Override - public synchronized String toString() { - return "RandomCodecProvider: " + previousMappings.toString(); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java 2011-11-01 18:41:12.552614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,415 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.Closeable; -import java.io.IOException; -import java.util.Iterator; -import java.util.Random; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.IndexDocValuesField; -import org.apache.lucene.document.Document; -import org.apache.lucene.index.IndexWriter; // javadoc -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.values.ValueType; -import org.apache.lucene.search.Query; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.Version; -import org.apache.lucene.util._TestUtil; - -/** Silly class that randomizes the indexing experience. EG - * it may swap in a different merge policy/scheduler; may - * commit periodically; may or may not optimize in the end, - * may flush by doc count instead of RAM, etc. - */ - -public class RandomIndexWriter implements Closeable { - - public IndexWriter w; - private final Random r; - int docCount; - int flushAt; - private double flushAtFactor = 1.0; - private boolean getReaderCalled; - private final int fixedBytesLength; - private final long docValuesFieldPrefix; - private volatile boolean doDocValues; - private CodecProvider codecProvider; - - // Randomly calls Thread.yield so we mixup thread scheduling - private static final class MockIndexWriter extends IndexWriter { - - private final Random r; - - public MockIndexWriter(Random r, Directory dir, IndexWriterConfig conf) throws IOException { - super(dir, conf); - // must make a private random since our methods are - // called from different threads; else test failures may - // not be reproducible from the original seed - this.r = new Random(r.nextInt()); - } - - @Override - boolean testPoint(String name) { - if (r.nextInt(4) == 2) - Thread.yield(); - return true; - } - } - - /** create a RandomIndexWriter with a random config: Uses TEST_VERSION_CURRENT and MockAnalyzer */ - public RandomIndexWriter(Random r, Directory dir) throws IOException { - this(r, dir, LuceneTestCase.newIndexWriterConfig(r, LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(r))); - } - - /** create a RandomIndexWriter with a random config: Uses TEST_VERSION_CURRENT */ - public RandomIndexWriter(Random r, Directory dir, Analyzer a) throws IOException { - this(r, dir, LuceneTestCase.newIndexWriterConfig(r, LuceneTestCase.TEST_VERSION_CURRENT, a)); - } - - /** create a RandomIndexWriter with a random config */ - public RandomIndexWriter(Random r, Directory dir, Version v, Analyzer a) throws IOException { - this(r, dir, LuceneTestCase.newIndexWriterConfig(r, v, a)); - } - - /** create a RandomIndexWriter with the provided config */ - public RandomIndexWriter(Random r, Directory dir, IndexWriterConfig c) throws IOException { - this.r = r; - w = new MockIndexWriter(r, dir, c); - flushAt = _TestUtil.nextInt(r, 10, 1000); - if (LuceneTestCase.VERBOSE) { - System.out.println("RIW config=" + w.getConfig()); - System.out.println("codec default=" + w.getConfig().getCodecProvider().getDefaultFieldCodec()); - w.setInfoStream(System.out); - } - /* TODO: find some what to make that random... - * This must be fixed across all fixed bytes - * fields in one index. so if you open another writer - * this might change if I use r.nextInt(x) - * maybe we can peek at the existing files here? - */ - fixedBytesLength = 37; - docValuesFieldPrefix = r.nextLong(); - codecProvider = w.getConfig().getCodecProvider(); - switchDoDocValues(); - } - - private void switchDoDocValues() { - // randomly enable / disable docValues - doDocValues = LuceneTestCase.rarely(r); - } - - /** - * Adds a Document. - * @see IndexWriter#addDocument(Iterable) - */ - public void addDocument(final Iterable doc) throws IOException { - if (doDocValues && doc instanceof Document) { - randomPerDocFieldValues(r, (Document) doc); - } - if (r.nextInt(5) == 3) { - // TODO: maybe, we should simply buffer up added docs - // (but we need to clone them), and only when - // getReader, commit, etc. are called, we do an - // addDocuments? Would be better testing. - w.addDocuments(new Iterable>() { - - @Override - public Iterator> iterator() { - return new Iterator>() { - boolean done; - - @Override - public boolean hasNext() { - return !done; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - @Override - public Iterable next() { - if (done) { - throw new IllegalStateException(); - } - done = true; - return doc; - } - }; - } - }); - } else { - w.addDocument(doc); - } - - maybeCommit(); - } - - private void randomPerDocFieldValues(Random random, Document doc) { - - ValueType[] values = ValueType.values(); - ValueType type = values[random.nextInt(values.length)]; - String name = "random_" + type.name() + "" + docValuesFieldPrefix; - if ("PreFlex".equals(codecProvider.getFieldCodec(name)) || doc.getField(name) != null) - return; - IndexDocValuesField docValuesField = new IndexDocValuesField(name); - switch (type) { - case BYTES_FIXED_DEREF: - case BYTES_FIXED_STRAIGHT: - case BYTES_FIXED_SORTED: - //make sure we use a valid unicode string with a fixed size byte length - final String randomUnicodeString = _TestUtil.randomFixedByteLengthUnicodeString(random, fixedBytesLength); - BytesRef fixedRef = new BytesRef(randomUnicodeString); - if (fixedRef.length > fixedBytesLength) { - fixedRef = new BytesRef(fixedRef.bytes, 0, fixedBytesLength); - } else { - fixedRef.grow(fixedBytesLength); - fixedRef.length = fixedBytesLength; - } - docValuesField.setBytes(fixedRef, type); - break; - case BYTES_VAR_DEREF: - case BYTES_VAR_STRAIGHT: - case BYTES_VAR_SORTED: - BytesRef ref = new BytesRef(_TestUtil.randomUnicodeString(random, 200)); - docValuesField.setBytes(ref, type); - break; - case FLOAT_32: - docValuesField.setFloat(random.nextFloat()); - break; - case FLOAT_64: - docValuesField.setFloat(random.nextDouble()); - break; - case VAR_INTS: - docValuesField.setInt(random.nextLong()); - break; - case FIXED_INTS_16: - docValuesField.setInt(random.nextInt(Short.MAX_VALUE)); - break; - case FIXED_INTS_32: - docValuesField.setInt(random.nextInt()); - break; - case FIXED_INTS_64: - docValuesField.setInt(random.nextLong()); - break; - case FIXED_INTS_8: - docValuesField.setInt(random.nextInt(128)); - break; - default: - throw new IllegalArgumentException("no such type: " + type); - } - - doc.add(docValuesField); - } - - private void maybeCommit() throws IOException { - if (docCount++ == flushAt) { - if (LuceneTestCase.VERBOSE) { - System.out.println("RIW.add/updateDocument: now doing a commit at docCount=" + docCount); - } - w.commit(); - flushAt += _TestUtil.nextInt(r, (int) (flushAtFactor * 10), (int) (flushAtFactor * 1000)); - if (flushAtFactor < 2e6) { - // gradually but exponentially increase time b/w flushes - flushAtFactor *= 1.05; - } - switchDoDocValues(); - } - } - - public void addDocuments(Iterable> docs) throws IOException { - w.addDocuments(docs); - maybeCommit(); - } - - public void updateDocuments(Term delTerm, Iterable> docs) throws IOException { - w.updateDocuments(delTerm, docs); - maybeCommit(); - } - - /** - * Updates a document. - * @see IndexWriter#updateDocument(Term, Iterable) - */ - public void updateDocument(Term t, final Iterable doc) throws IOException { - if (doDocValues) { - randomPerDocFieldValues(r, (Document) doc); - } - if (r.nextInt(5) == 3) { - w.updateDocuments(t, new Iterable>() { - - @Override - public Iterator> iterator() { - return new Iterator>() { - boolean done; - - @Override - public boolean hasNext() { - return !done; - } - - @Override - public void remove() { - throw new UnsupportedOperationException(); - } - - @Override - public Iterable next() { - if (done) { - throw new IllegalStateException(); - } - done = true; - return doc; - } - }; - } - }); - } else { - w.updateDocument(t, doc); - } - maybeCommit(); - } - - public void addIndexes(Directory... dirs) throws CorruptIndexException, IOException { - w.addIndexes(dirs); - } - - public void deleteDocuments(Term term) throws CorruptIndexException, IOException { - w.deleteDocuments(term); - } - - public void deleteDocuments(Query q) throws CorruptIndexException, IOException { - w.deleteDocuments(q); - } - - public void commit() throws CorruptIndexException, IOException { - w.commit(); - switchDoDocValues(); - } - - public int numDocs() throws IOException { - return w.numDocs(); - } - - public int maxDoc() { - return w.maxDoc(); - } - - public void deleteAll() throws IOException { - w.deleteAll(); - } - - public IndexReader getReader() throws IOException { - return getReader(true); - } - - private boolean doRandomOptimize = true; - private boolean doRandomOptimizeAssert = true; - - public void expungeDeletes(boolean doWait) throws IOException { - w.expungeDeletes(doWait); - } - - public void expungeDeletes() throws IOException { - w.expungeDeletes(); - } - - public void setDoRandomOptimize(boolean v) { - doRandomOptimize = v; - } - - public void setDoRandomOptimizeAssert(boolean v) { - doRandomOptimizeAssert = v; - } - - private void doRandomOptimize() throws IOException { - if (doRandomOptimize) { - final int segCount = w.getSegmentCount(); - if (r.nextBoolean() || segCount == 0) { - // full optimize - w.optimize(); - } else { - // partial optimize - final int limit = _TestUtil.nextInt(r, 1, segCount); - w.optimize(limit); - assert !doRandomOptimizeAssert || w.getSegmentCount() <= limit: "limit=" + limit + " actual=" + w.getSegmentCount(); - } - } - switchDoDocValues(); - } - - public IndexReader getReader(boolean applyDeletions) throws IOException { - getReaderCalled = true; - if (r.nextInt(4) == 2) { - doRandomOptimize(); - } - // If we are writing with PreFlexRW, force a full - // IndexReader.open so terms are sorted in codepoint - // order during searching: - if (!applyDeletions || !w.codecs.getDefaultFieldCodec().equals("PreFlex") && r.nextBoolean()) { - if (LuceneTestCase.VERBOSE) { - System.out.println("RIW.getReader: use NRT reader"); - } - if (r.nextInt(5) == 1) { - w.commit(); - } - return w.getReader(applyDeletions); - } else { - if (LuceneTestCase.VERBOSE) { - System.out.println("RIW.getReader: open new reader"); - } - w.commit(); - switchDoDocValues(); - if (r.nextBoolean()) { - return IndexReader.open(w.getDirectory(), new KeepOnlyLastCommitDeletionPolicy(), r.nextBoolean(), _TestUtil.nextInt(r, 1, 10), w.getConfig().getCodecProvider()); - } else { - return w.getReader(applyDeletions); - } - } - } - - /** - * Close this writer. - * @see IndexWriter#close() - */ - public void close() throws IOException { - // if someone isn't using getReader() API, we want to be sure to - // maybeOptimize since presumably they might open a reader on the dir. - if (getReaderCalled == false && r.nextInt(8) == 2) { - doRandomOptimize(); - } - w.close(); - } - - /** - * Forces an optimize. - *

    - * NOTE: this should be avoided in tests unless absolutely necessary, - * as it will result in less test coverage. - * @see IndexWriter#optimize() - */ - public void optimize() throws IOException { - w.optimize(); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java lucene-2621/lucene/src/test-framework/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java 2011-11-01 18:41:12.552614523 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,657 +0,0 @@ -package org.apache.lucene.index; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.lucene.analysis.MockAnalyzer; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.PhraseQuery; -import org.apache.lucene.search.Query; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.Sort; -import org.apache.lucene.search.SortField; -import org.apache.lucene.search.TermQuery; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.MockDirectoryWrapper; -import org.apache.lucene.util.Bits; -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.LineFileDocs; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.NamedThreadFactory; -import org.apache.lucene.util._TestUtil; - -// TODO -// - mix in optimize, addIndexes -// - randomly mix in non-congruent docs - -/** Utility class that spawns multiple indexing and - * searching threads. */ -public abstract class ThreadedIndexingAndSearchingTestCase extends LuceneTestCase { - - protected final AtomicBoolean failed = new AtomicBoolean(); - protected final AtomicInteger addCount = new AtomicInteger(); - protected final AtomicInteger delCount = new AtomicInteger(); - protected final AtomicInteger packCount = new AtomicInteger(); - - protected Directory dir; - protected IndexWriter writer; - - private static class SubDocs { - public final String packID; - public final List subIDs; - public boolean deleted; - - public SubDocs(String packID, List subIDs) { - this.packID = packID; - this.subIDs = subIDs; - } - } - - // Called per-search - protected abstract IndexSearcher getCurrentSearcher() throws Exception; - - protected abstract IndexSearcher getFinalSearcher() throws Exception; - - protected void releaseSearcher(IndexSearcher s) throws Exception { - } - - // Called once to run searching - protected abstract void doSearching(ExecutorService es, long stopTime) throws Exception; - - protected Directory getDirectory(Directory in) { - return in; - } - - protected void updateDocuments(Term id, List> docs) throws Exception { - writer.updateDocuments(id, docs); - } - - protected void addDocuments(Term id, List> docs) throws Exception { - writer.addDocuments(docs); - } - - protected void addDocument(Term id, Iterable doc) throws Exception { - writer.addDocument(doc); - } - - protected void updateDocument(Term term, Iterable doc) throws Exception { - writer.updateDocument(term, doc); - } - - protected void deleteDocuments(Term term) throws Exception { - writer.deleteDocuments(term); - } - - protected void doAfterIndexingThreadDone() { - } - - private Thread[] launchIndexingThreads(final LineFileDocs docs, - int numThreads, - final long stopTime, - final Set delIDs, - final Set delPackIDs, - final List allSubDocs) - throws Exception { - final Thread[] threads = new Thread[numThreads]; - for(int thread=0;thread toDeleteIDs = new ArrayList(); - final List toDeleteSubDocs = new ArrayList(); - while(System.currentTimeMillis() < stopTime && !failed.get()) { - try { - - // Occasional longish pause if running - // nightly - if (LuceneTestCase.TEST_NIGHTLY && random.nextInt(6) == 3) { - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": now long sleep"); - } - Thread.sleep(_TestUtil.nextInt(random, 50, 500)); - } - - // Rate limit ingest rate: - if (random.nextInt(7) == 5) { - Thread.sleep(_TestUtil.nextInt(random, 1, 10)); - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": done sleep"); - } - } - - Document doc = docs.nextDoc(); - if (doc == null) { - break; - } - - // Maybe add randomly named field - final String addedField; - if (random.nextBoolean()) { - addedField = "extra" + random.nextInt(40); - doc.add(newField(addedField, "a random field", TextField.TYPE_STORED)); - } else { - addedField = null; - } - - if (random.nextBoolean()) { - - if (random.nextBoolean()) { - // Add/update doc block: - final String packID; - final SubDocs delSubDocs; - if (toDeleteSubDocs.size() > 0 && random.nextBoolean()) { - delSubDocs = toDeleteSubDocs.get(random.nextInt(toDeleteSubDocs.size())); - assert !delSubDocs.deleted; - toDeleteSubDocs.remove(delSubDocs); - // Update doc block, replacing prior packID - packID = delSubDocs.packID; - } else { - delSubDocs = null; - // Add doc block, using new packID - packID = packCount.getAndIncrement() + ""; - } - - final Field packIDField = newField("packID", packID, StringField.TYPE_STORED); - final List docIDs = new ArrayList(); - final SubDocs subDocs = new SubDocs(packID, docIDs); - final List docsList = new ArrayList(); - - allSubDocs.add(subDocs); - doc.add(packIDField); - docsList.add(_TestUtil.cloneDocument(doc)); - docIDs.add(doc.get("docid")); - - final int maxDocCount = _TestUtil.nextInt(random, 1, 10); - while(docsList.size() < maxDocCount) { - doc = docs.nextDoc(); - if (doc == null) { - break; - } - docsList.add(_TestUtil.cloneDocument(doc)); - docIDs.add(doc.get("docid")); - } - addCount.addAndGet(docsList.size()); - - final Term packIDTerm = new Term("packID", packID); - - if (delSubDocs != null) { - delSubDocs.deleted = true; - delIDs.addAll(delSubDocs.subIDs); - delCount.addAndGet(delSubDocs.subIDs.size()); - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": update pack packID=" + delSubDocs.packID + " count=" + docsList.size() + " docs=" + docIDs); - } - updateDocuments(packIDTerm, docsList); - } else { - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": add pack packID=" + packID + " count=" + docsList.size() + " docs=" + docIDs); - } - addDocuments(packIDTerm, docsList); - } - doc.removeField("packID"); - - if (random.nextInt(5) == 2) { - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": buffer del id:" + packID); - } - toDeleteSubDocs.add(subDocs); - } - - } else { - // Add single doc - final String docid = doc.get("docid"); - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": add doc docid:" + docid); - } - addDocument(new Term("docid", docid), doc); - addCount.getAndIncrement(); - - if (random.nextInt(5) == 3) { - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid")); - } - toDeleteIDs.add(docid); - } - } - } else { - - // Update single doc, but we never re-use - // and ID so the delete will never - // actually happen: - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": update doc id:" + doc.get("docid")); - } - final String docid = doc.get("docid"); - updateDocument(new Term("docid", docid), doc); - addCount.getAndIncrement(); - - if (random.nextInt(5) == 3) { - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": buffer del id:" + doc.get("docid")); - } - toDeleteIDs.add(docid); - } - } - - if (random.nextInt(30) == 17) { - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": apply " + toDeleteIDs.size() + " deletes"); - } - for(String id : toDeleteIDs) { - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": del term=id:" + id); - } - deleteDocuments(new Term("docid", id)); - } - final int count = delCount.addAndGet(toDeleteIDs.size()); - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": tot " + count + " deletes"); - } - delIDs.addAll(toDeleteIDs); - toDeleteIDs.clear(); - - for(SubDocs subDocs : toDeleteSubDocs) { - assert !subDocs.deleted; - delPackIDs.add(subDocs.packID); - deleteDocuments(new Term("packID", subDocs.packID)); - subDocs.deleted = true; - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": del subs: " + subDocs.subIDs + " packID=" + subDocs.packID); - } - delIDs.addAll(subDocs.subIDs); - delCount.addAndGet(subDocs.subIDs.size()); - } - toDeleteSubDocs.clear(); - } - if (addedField != null) { - doc.removeField(addedField); - } - } catch (Throwable t) { - System.out.println(Thread.currentThread().getName() + ": hit exc"); - t.printStackTrace(); - failed.set(true); - throw new RuntimeException(t); - } - } - if (VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": indexing done"); - } - - doAfterIndexingThreadDone(); - } - }; - threads[thread].setDaemon(true); - threads[thread].start(); - } - - return threads; - } - - protected void runSearchThreads(final long stopTimeMS) throws Exception { - final int numThreads = _TestUtil.nextInt(random, 1, 5); - final Thread[] searchThreads = new Thread[numThreads]; - final AtomicInteger totHits = new AtomicInteger(); - - // silly starting guess: - final AtomicInteger totTermCount = new AtomicInteger(100); - - // TODO: we should enrich this to do more interesting searches - for(int thread=0;thread 0) { - smokeTestSearcher(s); - Fields fields = MultiFields.getFields(s.getIndexReader()); - if (fields == null) { - continue; - } - Terms terms = fields.terms("body"); - if (terms == null) { - continue; - } - TermsEnum termsEnum = terms.iterator(); - int seenTermCount = 0; - int shift; - int trigger; - if (totTermCount.get() < 10) { - shift = 0; - trigger = 1; - } else { - trigger = totTermCount.get()/10; - shift = random.nextInt(trigger); - } - BytesRef term = termsEnum.next(); - if (term == null) { - if (seenTermCount == 0) { - break; - } - totTermCount.set(seenTermCount); - seenTermCount = 0; - if (totTermCount.get() < 10) { - shift = 0; - trigger = 1; - } else { - trigger = totTermCount.get()/10; - //System.out.println("trigger " + trigger); - shift = random.nextInt(trigger); - } - termsEnum.seekCeil(new BytesRef("")); - continue; - } - seenTermCount++; - // search 10 terms - if (trigger == 0) { - trigger = 1; - } - if ((seenTermCount + shift) % trigger == 0) { - //if (VERBOSE) { - //System.out.println(Thread.currentThread().getName() + " now search body:" + term.utf8ToString()); - //} - totHits.addAndGet(runQuery(s, new TermQuery(new Term("body", term)))); - } - //if (VERBOSE) { - //System.out.println(Thread.currentThread().getName() + ": search done"); - //} - } - } finally { - releaseSearcher(s); - } - } catch (Throwable t) { - System.out.println(Thread.currentThread().getName() + ": hit exc"); - failed.set(true); - t.printStackTrace(System.out); - throw new RuntimeException(t); - } - } - } - }; - searchThreads[thread].setDaemon(true); - searchThreads[thread].start(); - } - - for(int thread=0;thread delIDs = Collections.synchronizedSet(new HashSet()); - final Set delPackIDs = Collections.synchronizedSet(new HashSet()); - final List allSubDocs = Collections.synchronizedList(new ArrayList()); - - final long stopTime = System.currentTimeMillis() + RUN_TIME_SEC*1000; - - final Thread[] indexThreads = launchIndexingThreads(docs, NUM_INDEX_THREADS, stopTime, delIDs, delPackIDs, allSubDocs); - - if (VERBOSE) { - System.out.println("TEST: DONE start indexing threads [" + (System.currentTimeMillis()-t0) + " ms]"); - } - - // Let index build up a bit - Thread.sleep(100); - - doSearching(es, stopTime); - - if (VERBOSE) { - System.out.println("TEST: all searching done [" + (System.currentTimeMillis()-t0) + " ms]"); - } - - for(int thread=0;thread saveMissCount); - } - - return docIdSet; - } - - @Override - public String toString() { - return "CachingWrapperFilterHelper("+filter+")"; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof CachingWrapperFilterHelper)) return false; - return this.filter.equals(o); - } - - @Override - public int hashCode() { - return this.filter.hashCode() ^ 0x5525aacb; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java lucene-2621/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java 2011-11-03 14:37:00.688533051 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/search/CheckHits.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,525 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; -import java.util.Locale; -import java.util.Set; -import java.util.TreeSet; -import java.util.Random; - -import junit.framework.Assert; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexReader.AtomicReaderContext; -import org.apache.lucene.store.Directory; - -public class CheckHits { - - /** - * Some explains methods calculate their values though a slightly - * different order of operations from the actual scoring method ... - * this allows for a small amount of relative variation - */ - public static float EXPLAIN_SCORE_TOLERANCE_DELTA = 0.001f; - - /** - * In general we use a relative epsilon, but some tests do crazy things - * like boost documents with 0, creating tiny tiny scores where the - * relative difference is large but the absolute difference is tiny. - * we ensure the the epsilon is always at least this big. - */ - public static float EXPLAIN_SCORE_TOLERANCE_MINIMUM = 1e-6f; - - /** - * Tests that all documents up to maxDoc which are *not* in the - * expected result set, have an explanation which indicates that - * the document does not match - */ - public static void checkNoMatchExplanations(Query q, String defaultFieldName, - IndexSearcher searcher, int[] results) - throws IOException { - - String d = q.toString(defaultFieldName); - Set ignore = new TreeSet(); - for (int i = 0; i < results.length; i++) { - ignore.add(Integer.valueOf(results[i])); - } - - int maxDoc = searcher.maxDoc(); - for (int doc = 0; doc < maxDoc; doc++) { - if (ignore.contains(Integer.valueOf(doc))) continue; - - Explanation exp = searcher.explain(q, doc); - Assert.assertNotNull("Explanation of [["+d+"]] for #"+doc+" is null", - exp); - Assert.assertFalse("Explanation of [["+d+"]] for #"+doc+ - " doesn't indicate non-match: " + exp.toString(), - exp.isMatch()); - } - - } - - /** - * Tests that a query matches the an expected set of documents using a - * HitCollector. - * - *

    - * Note that when using the HitCollector API, documents will be collected - * if they "match" regardless of what their score is. - *

    - * @param query the query to test - * @param searcher the searcher to test the query against - * @param defaultFieldName used for displaying the query in assertion messages - * @param results a list of documentIds that must match the query - * @see Searcher#search(Query,Collector) - * @see #checkHits - */ - public static void checkHitCollector(Random random, Query query, String defaultFieldName, - IndexSearcher searcher, int[] results) - throws IOException { - - QueryUtils.check(random,query,searcher); - - Set correct = new TreeSet(); - for (int i = 0; i < results.length; i++) { - correct.add(Integer.valueOf(results[i])); - } - final Set actual = new TreeSet(); - final Collector c = new SetCollector(actual); - - searcher.search(query, c); - Assert.assertEquals("Simple: " + query.toString(defaultFieldName), - correct, actual); - - for (int i = -1; i < 2; i++) { - actual.clear(); - IndexSearcher s = QueryUtils.wrapUnderlyingReader - (random, searcher, i); - s.search(query, c); - Assert.assertEquals("Wrap Reader " + i + ": " + - query.toString(defaultFieldName), - correct, actual); - FieldCache.DEFAULT.purge(s.getIndexReader()); // our wrapping can create insanity otherwise - s.close(); - } - } - - public static class SetCollector extends Collector { - final Set bag; - public SetCollector(Set bag) { - this.bag = bag; - } - private int base = 0; - @Override - public void setScorer(Scorer scorer) throws IOException {} - @Override - public void collect(int doc) { - bag.add(Integer.valueOf(doc + base)); - } - @Override - public void setNextReader(AtomicReaderContext context) { - base = context.docBase; - } - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - } - - /** - * Tests that a query matches the an expected set of documents using Hits. - * - *

    - * Note that when using the Hits API, documents will only be returned - * if they have a positive normalized score. - *

    - * @param query the query to test - * @param searcher the searcher to test the query against - * @param defaultFieldName used for displaing the query in assertion messages - * @param results a list of documentIds that must match the query - * @see Searcher#search(Query, int) - * @see #checkHitCollector - */ - public static void checkHits( - Random random, - Query query, - String defaultFieldName, - IndexSearcher searcher, - int[] results) - throws IOException { - - ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs; - - Set correct = new TreeSet(); - for (int i = 0; i < results.length; i++) { - correct.add(Integer.valueOf(results[i])); - } - - Set actual = new TreeSet(); - for (int i = 0; i < hits.length; i++) { - actual.add(Integer.valueOf(hits[i].doc)); - } - - Assert.assertEquals(query.toString(defaultFieldName), correct, actual); - - QueryUtils.check(random, query,searcher); - } - - /** Tests that a Hits has an expected order of documents */ - public static void checkDocIds(String mes, int[] results, ScoreDoc[] hits) - throws IOException { - Assert.assertEquals(mes + " nr of hits", hits.length, results.length); - for (int i = 0; i < results.length; i++) { - Assert.assertEquals(mes + " doc nrs for hit " + i, results[i], hits[i].doc); - } - } - - /** Tests that two queries have an expected order of documents, - * and that the two queries have the same score values. - */ - public static void checkHitsQuery( - Query query, - ScoreDoc[] hits1, - ScoreDoc[] hits2, - int[] results) - throws IOException { - - checkDocIds("hits1", results, hits1); - checkDocIds("hits2", results, hits2); - checkEqual(query, hits1, hits2); - } - - public static void checkEqual(Query query, ScoreDoc[] hits1, ScoreDoc[] hits2) throws IOException { - final float scoreTolerance = 1.0e-6f; - if (hits1.length != hits2.length) { - Assert.fail("Unequal lengths: hits1="+hits1.length+",hits2="+hits2.length); - } - for (int i = 0; i < hits1.length; i++) { - if (hits1[i].doc != hits2[i].doc) { - Assert.fail("Hit " + i + " docnumbers don't match\n" - + hits2str(hits1, hits2,0,0) - + "for query:" + query.toString()); - } - - if ((hits1[i].doc != hits2[i].doc) - || Math.abs(hits1[i].score - hits2[i].score) > scoreTolerance) - { - Assert.fail("Hit " + i + ", doc nrs " + hits1[i].doc + " and " + hits2[i].doc - + "\nunequal : " + hits1[i].score - + "\n and: " + hits2[i].score - + "\nfor query:" + query.toString()); - } - } - } - - public static String hits2str(ScoreDoc[] hits1, ScoreDoc[] hits2, int start, int end) throws IOException { - StringBuilder sb = new StringBuilder(); - int len1=hits1==null ? 0 : hits1.length; - int len2=hits2==null ? 0 : hits2.length; - if (end<=0) { - end = Math.max(len1,len2); - } - - sb.append("Hits length1=").append(len1).append("\tlength2=").append(len2); - - sb.append('\n'); - for (int i=start; i times others" (where is float). - float x = 0; - String descr = expl.getDescription().toLowerCase(Locale.ENGLISH); - boolean productOf = descr.endsWith("product of:"); - boolean sumOf = descr.endsWith("sum of:"); - boolean maxOf = descr.endsWith("max of:"); - boolean maxTimesOthers = false; - if (!(productOf || sumOf || maxOf)) { - // maybe 'max plus x times others' - int k1 = descr.indexOf("max plus "); - if (k1>=0) { - k1 += "max plus ".length(); - int k2 = descr.indexOf(" ",k1); - try { - x = Float.parseFloat(descr.substring(k1,k2).trim()); - if (descr.substring(k2).trim().equals("times others of:")) { - maxTimesOthers = true; - } - } catch (NumberFormatException e) { - } - } - } - // TODO: this is a TERRIBLE assertion!!!! - Assert.assertTrue( - q+": multi valued explanation description=\""+descr - +"\" must be 'max of plus x times others' or end with 'product of'" - +" or 'sum of:' or 'max of:' - "+expl, - productOf || sumOf || maxOf || maxTimesOthers); - float sum = 0; - float product = 1; - float max = 0; - for (int i=0; i maxDiff - || scorerDiff > maxDiff) { - StringBuilder sbord = new StringBuilder(); - for (int i = 0; i < order.length; i++) - sbord.append(order[i] == skip_op ? " skip()" : " next()"); - throw new RuntimeException("ERROR matching docs:" + "\n\t" - + (doc != scorerDoc ? "--> " : "") + "doc=" + doc + ", scorerDoc=" + scorerDoc - + "\n\t" + (!more ? "--> " : "") + "tscorer.more=" + more - + "\n\t" + (scoreDiff > maxDiff ? "--> " : "") - + "scorerScore=" + scorerScore + " scoreDiff=" + scoreDiff - + " maxDiff=" + maxDiff + "\n\t" - + (scorerDiff > maxDiff ? "--> " : "") + "scorerScore2=" - + scorerScore2 + " scorerDiff=" + scorerDiff - + "\n\thitCollector.doc=" + doc + " score=" + score - + "\n\t Scorer=" + scorer + "\n\t Query=" + q + " " - + q.getClass().getName() + "\n\t Searcher=" + s - + "\n\t Order=" + sbord + "\n\t Op=" - + (op == skip_op ? " skip()" : " next()")); - } - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void setNextReader(AtomicReaderContext context) throws IOException { - // confirm that skipping beyond the last doc, on the - // previous reader, hits NO_MORE_DOCS - if (lastReader[0] != null) { - final IndexReader previousReader = lastReader[0]; - IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); - Weight w = indexSearcher.createNormalizedWeight(q); - AtomicReaderContext ctx = (AtomicReaderContext)indexSearcher.getTopReaderContext(); - Scorer scorer = w.scorer(ctx, true, false, ctx.reader.getLiveDocs()); - if (scorer != null) { - boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; - Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); - } - leafPtr++; - indexSearcher.close(); - } - lastReader[0] = context.reader; - assert readerContextArray[leafPtr].reader == context.reader; - this.scorer = null; - lastDoc[0] = -1; - } - - @Override - public boolean acceptsDocsOutOfOrder() { - return true; - } - }); - - if (lastReader[0] != null) { - // confirm that skipping beyond the last doc, on the - // previous reader, hits NO_MORE_DOCS - final IndexReader previousReader = lastReader[0]; - IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader, false); - Weight w = indexSearcher.createNormalizedWeight(q); - AtomicReaderContext ctx = (AtomicReaderContext)previousReader.getTopReaderContext(); - Scorer scorer = w.scorer(ctx, true, false, ctx.reader.getLiveDocs()); - if (scorer != null) { - boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; - Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); - } - indexSearcher.close(); - } - } - } - - // check that first skip on just created scorers always goes to the right doc - private static void checkFirstSkipTo(final Query q, final IndexSearcher s) throws IOException { - //System.out.println("checkFirstSkipTo: "+q); - final float maxDiff = 1e-3f; - final int lastDoc[] = {-1}; - final IndexReader lastReader[] = {null}; - final AtomicReaderContext[] context = ReaderUtil.leaves(s.getTopReaderContext()); - s.search(q,new Collector() { - private Scorer scorer; - private int leafPtr; - private Bits liveDocs; - @Override - public void setScorer(Scorer scorer) throws IOException { - this.scorer = scorer; - } - @Override - public void collect(int doc) throws IOException { - float score = scorer.score(); - try { - long startMS = System.currentTimeMillis(); - for (int i=lastDoc[0]+1; i<=doc; i++) { - Weight w = s.createNormalizedWeight(q); - Scorer scorer = w.scorer(context[leafPtr], true, false, liveDocs); - Assert.assertTrue("query collected "+doc+" but skipTo("+i+") says no more docs!",scorer.advance(i) != DocIdSetIterator.NO_MORE_DOCS); - Assert.assertEquals("query collected "+doc+" but skipTo("+i+") got to "+scorer.docID(),doc,scorer.docID()); - float skipToScore = scorer.score(); - Assert.assertEquals("unstable skipTo("+i+") score!",skipToScore,scorer.score(),maxDiff); - Assert.assertEquals("query assigned doc "+doc+" a score of <"+score+"> but skipTo("+i+") has <"+skipToScore+">!",score,skipToScore,maxDiff); - - // Hurry things along if they are going slow (eg - // if you got SimpleText codec this will kick in): - if (i < doc && System.currentTimeMillis() - startMS > 5) { - i = doc-1; - } - } - lastDoc[0] = doc; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - @Override - public void setNextReader(AtomicReaderContext context) throws IOException { - // confirm that skipping beyond the last doc, on the - // previous reader, hits NO_MORE_DOCS - if (lastReader[0] != null) { - final IndexReader previousReader = lastReader[0]; - IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); - Weight w = indexSearcher.createNormalizedWeight(q); - Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), true, false, previousReader.getLiveDocs()); - if (scorer != null) { - boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; - Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); - } - indexSearcher.close(); - leafPtr++; - } - - lastReader[0] = context.reader; - lastDoc[0] = -1; - liveDocs = context.reader.getLiveDocs(); - } - @Override - public boolean acceptsDocsOutOfOrder() { - return false; - } - }); - - if (lastReader[0] != null) { - // confirm that skipping beyond the last doc, on the - // previous reader, hits NO_MORE_DOCS - final IndexReader previousReader = lastReader[0]; - IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader); - Weight w = indexSearcher.createNormalizedWeight(q); - Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), true, false, previousReader.getLiveDocs()); - if (scorer != null) { - boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS; - Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more); - } - indexSearcher.close(); - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java lucene-2621/lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java 2011-11-01 18:41:12.544614524 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,158 +0,0 @@ -package org.apache.lucene.search; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; - -import org.apache.lucene.search.similarities.AfterEffect; -import org.apache.lucene.search.similarities.AfterEffectB; -import org.apache.lucene.search.similarities.AfterEffectL; -import org.apache.lucene.search.similarities.BM25Similarity; -import org.apache.lucene.search.similarities.BasicModel; -import org.apache.lucene.search.similarities.BasicModelBE; -import org.apache.lucene.search.similarities.BasicModelD; -import org.apache.lucene.search.similarities.BasicModelG; -import org.apache.lucene.search.similarities.BasicModelIF; -import org.apache.lucene.search.similarities.BasicModelIn; -import org.apache.lucene.search.similarities.BasicModelIne; -import org.apache.lucene.search.similarities.BasicModelP; -import org.apache.lucene.search.similarities.DFRSimilarity; -import org.apache.lucene.search.similarities.DefaultSimilarity; -import org.apache.lucene.search.similarities.DefaultSimilarityProvider; -import org.apache.lucene.search.similarities.Distribution; -import org.apache.lucene.search.similarities.DistributionLL; -import org.apache.lucene.search.similarities.DistributionSPL; -import org.apache.lucene.search.similarities.IBSimilarity; -import org.apache.lucene.search.similarities.LMDirichletSimilarity; -import org.apache.lucene.search.similarities.LMJelinekMercerSimilarity; -import org.apache.lucene.search.similarities.Lambda; -import org.apache.lucene.search.similarities.LambdaDF; -import org.apache.lucene.search.similarities.LambdaTTF; -import org.apache.lucene.search.similarities.Normalization; -import org.apache.lucene.search.similarities.NormalizationH1; -import org.apache.lucene.search.similarities.NormalizationH2; -import org.apache.lucene.search.similarities.NormalizationH3; -import org.apache.lucene.search.similarities.NormalizationZ; -import org.apache.lucene.search.similarities.Similarity; - -public class RandomSimilarityProvider extends DefaultSimilarityProvider { - final List knownSims; - Map previousMappings = new HashMap(); - final int perFieldSeed; - final boolean shouldCoord; - final boolean shouldQueryNorm; - - public RandomSimilarityProvider(Random random) { - perFieldSeed = random.nextInt(); - shouldCoord = random.nextBoolean(); - shouldQueryNorm = random.nextBoolean(); - knownSims = new ArrayList(allSims); - Collections.shuffle(knownSims, random); - } - - @Override - public float coord(int overlap, int maxOverlap) { - if (shouldCoord) { - return super.coord(overlap, maxOverlap); - } else { - return 1.0f; - } - } - - @Override - public float queryNorm(float sumOfSquaredWeights) { - if (shouldQueryNorm) { - return super.queryNorm(sumOfSquaredWeights); - } else { - return 1.0f; - } - } - - @Override - public synchronized Similarity get(String field) { - assert field != null; - Similarity sim = previousMappings.get(field); - if (sim == null) { - sim = knownSims.get(Math.abs(perFieldSeed ^ field.hashCode()) % knownSims.size()); - previousMappings.put(field, sim); - } - return sim; - } - - // all the similarities that we rotate through - /** The DFR basic models to test. */ - static BasicModel[] BASIC_MODELS = { - /* TODO: enable new BasicModelBE(), */ /* TODO: enable new BasicModelD(), */ new BasicModelG(), - new BasicModelIF(), new BasicModelIn(), new BasicModelIne(), - /* TODO: enable new BasicModelP() */ - }; - /** The DFR aftereffects to test. */ - static AfterEffect[] AFTER_EFFECTS = { - new AfterEffectB(), new AfterEffectL(), new AfterEffect.NoAfterEffect() - }; - /** The DFR normalizations to test. */ - static Normalization[] NORMALIZATIONS = { - new NormalizationH1(), new NormalizationH2(), - new NormalizationH3(), new NormalizationZ() - // TODO: if we enable NoNormalization, we have to deal with - // a couple tests (e.g. TestDocBoost, TestSort) that expect length normalization - // new Normalization.NoNormalization() - }; - /** The distributions for IB. */ - static Distribution[] DISTRIBUTIONS = { - new DistributionLL(), new DistributionSPL() - }; - /** Lambdas for IB. */ - static Lambda[] LAMBDAS = { - new LambdaDF(), new LambdaTTF() - }; - static List allSims; - static { - allSims = new ArrayList(); - allSims.add(new DefaultSimilarity()); - allSims.add(new BM25Similarity()); - for (BasicModel basicModel : BASIC_MODELS) { - for (AfterEffect afterEffect : AFTER_EFFECTS) { - for (Normalization normalization : NORMALIZATIONS) { - allSims.add(new DFRSimilarity(basicModel, afterEffect, normalization)); - } - } - } - for (Distribution distribution : DISTRIBUTIONS) { - for (Lambda lambda : LAMBDAS) { - for (Normalization normalization : NORMALIZATIONS) { - allSims.add(new IBSimilarity(distribution, lambda, normalization)); - } - } - } - /* TODO: enable Dirichlet - allSims.add(new LMDirichletSimilarity()); */ - allSims.add(new LMJelinekMercerSimilarity(0.1f)); - allSims.add(new LMJelinekMercerSimilarity(0.7f)); - } - - @Override - public synchronized String toString() { - return "RandomSimilarityProvider(queryNorm=" + shouldQueryNorm + ",coord=" + shouldCoord + "): " + previousMappings.toString(); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java lucene-2621/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java 2011-11-01 18:41:12.556614525 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,709 +0,0 @@ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.Closeable; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.IdentityHashMap; -import java.util.Iterator; -import java.util.Map; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; - -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.util.ThrottledIndexOutput; -import org.apache.lucene.util._TestUtil; - -/** - * This is a Directory Wrapper that adds methods - * intended to be used only by unit tests. - * It also adds a number of features useful for testing: - *
      - *
    • Instances created by {@link LuceneTestCase#newDirectory()} are tracked - * to ensure they are closed by the test. - *
    • When a MockDirectoryWrapper is closed, it will throw an exception if - * it has any open files against it (with a stacktrace indicating where - * they were opened from). - *
    • When a MockDirectoryWrapper is closed, it runs CheckIndex to test if - * the index was corrupted. - *
    • MockDirectoryWrapper simulates some "features" of Windows, such as - * refusing to write/delete to open files. - *
    - */ - -public class MockDirectoryWrapper extends Directory { - final Directory delegate; - long maxSize; - - // Max actual bytes used. This is set by MockRAMOutputStream: - long maxUsedSize; - double randomIOExceptionRate; - Random randomState; - boolean noDeleteOpenFile = true; - boolean preventDoubleWrite = true; - boolean checkIndexOnClose = true; - boolean trackDiskUsage = false; - private Set unSyncedFiles; - private Set createdFiles; - private Set openFilesForWrite = new HashSet(); - Set openLocks = Collections.synchronizedSet(new HashSet()); - volatile boolean crashed; - private ThrottledIndexOutput throttledOutput; - private Throttling throttling = Throttling.SOMETIMES; - - final AtomicInteger inputCloneCount = new AtomicInteger(); - - // use this for tracking files for crash. - // additionally: provides debugging information in case you leave one open - private Map openFileHandles = Collections.synchronizedMap(new IdentityHashMap()); - - // NOTE: we cannot initialize the Map here due to the - // order in which our constructor actually does this - // member initialization vs when it calls super. It seems - // like super is called, then our members are initialized: - private Map openFiles; - - // Only tracked if noDeleteOpenFile is true: if an attempt - // is made to delete an open file, we enroll it here. - private Set openFilesDeleted; - - private synchronized void init() { - if (openFiles == null) { - openFiles = new HashMap(); - openFilesDeleted = new HashSet(); - } - - if (createdFiles == null) - createdFiles = new HashSet(); - if (unSyncedFiles == null) - unSyncedFiles = new HashSet(); - } - - public MockDirectoryWrapper(Random random, Directory delegate) { - this.delegate = delegate; - // must make a private random since our methods are - // called from different threads; else test failures may - // not be reproducible from the original seed - this.randomState = new Random(random.nextInt()); - this.throttledOutput = new ThrottledIndexOutput(ThrottledIndexOutput - .mBitsToBytes(40 + randomState.nextInt(10)), 5 + randomState.nextInt(5), null); - // force wrapping of lockfactory - try { - setLockFactory(new MockLockFactoryWrapper(this, delegate.getLockFactory())); - } catch (IOException e) { - throw new RuntimeException(e); - } - init(); - } - - public int getInputCloneCount() { - return inputCloneCount.get(); - } - - public void setTrackDiskUsage(boolean v) { - trackDiskUsage = v; - } - - /** If set to true, we throw an IOException if the same - * file is opened by createOutput, ever. */ - public void setPreventDoubleWrite(boolean value) { - preventDoubleWrite = value; - } - - public static enum Throttling { - /** always emulate a slow hard disk. could be very slow! */ - ALWAYS, - /** sometimes (2% of the time) emulate a slow hard disk. */ - SOMETIMES, - /** never throttle output */ - NEVER - } - - public void setThrottling(Throttling throttling) { - this.throttling = throttling; - } - - @Override - public synchronized void sync(Collection names) throws IOException { - maybeYield(); - for (String name : names) - maybeThrowDeterministicException(); - if (crashed) - throw new IOException("cannot sync after crash"); - unSyncedFiles.removeAll(names); - delegate.sync(names); - } - - @Override - public String toString() { - // NOTE: do not maybeYield here, since it consumes - // randomness and can thus (unexpectedly during - // debugging) change the behavior of a seed - // maybeYield(); - return "MockDirWrapper(" + delegate + ")"; - } - - public synchronized final long sizeInBytes() throws IOException { - if (delegate instanceof RAMDirectory) - return ((RAMDirectory) delegate).sizeInBytes(); - else { - // hack - long size = 0; - for (String file : delegate.listAll()) - size += delegate.fileLength(file); - return size; - } - } - - /** Simulates a crash of OS or machine by overwriting - * unsynced files. */ - public synchronized void crash() throws IOException { - crashed = true; - openFiles = new HashMap(); - openFilesForWrite = new HashSet(); - openFilesDeleted = new HashSet(); - Iterator it = unSyncedFiles.iterator(); - unSyncedFiles = new HashSet(); - // first force-close all files, so we can corrupt on windows etc. - // clone the file map, as these guys want to remove themselves on close. - Map m = new IdentityHashMap(openFileHandles); - for (Closeable f : m.keySet()) - try { - f.close(); - } catch (Exception ignored) {} - - int count = 0; - while(it.hasNext()) { - String name = it.next(); - if (count % 3 == 0) { - deleteFile(name, true); - } else if (count % 3 == 1) { - // Zero out file entirely - long length = fileLength(name); - byte[] zeroes = new byte[256]; - long upto = 0; - IndexOutput out = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState)); - while(upto < length) { - final int limit = (int) Math.min(length-upto, zeroes.length); - out.writeBytes(zeroes, 0, limit); - upto += limit; - } - out.close(); - } else if (count % 3 == 2) { - // Truncate the file: - IndexOutput out = delegate.createOutput(name, LuceneTestCase.newIOContext(randomState)); - out.setLength(fileLength(name)/2); - out.close(); - } - count++; - } - } - - public synchronized void clearCrash() throws IOException { - crashed = false; - openLocks.clear(); - } - - public void setMaxSizeInBytes(long maxSize) { - this.maxSize = maxSize; - } - public long getMaxSizeInBytes() { - return this.maxSize; - } - - /** - * Returns the peek actual storage used (bytes) in this - * directory. - */ - public long getMaxUsedSizeInBytes() { - return this.maxUsedSize; - } - public void resetMaxUsedSizeInBytes() throws IOException { - this.maxUsedSize = getRecomputedActualSizeInBytes(); - } - - /** - * Emulate windows whereby deleting an open file is not - * allowed (raise IOException). - */ - public void setNoDeleteOpenFile(boolean value) { - this.noDeleteOpenFile = value; - } - public boolean getNoDeleteOpenFile() { - return noDeleteOpenFile; - } - - /** - * Set whether or not checkindex should be run - * on close - */ - public void setCheckIndexOnClose(boolean value) { - this.checkIndexOnClose = value; - } - - public boolean getCheckIndexOnClose() { - return checkIndexOnClose; - } - /** - * If 0.0, no exceptions will be thrown. Else this should - * be a double 0.0 - 1.0. We will randomly throw an - * IOException on the first write to an OutputStream based - * on this probability. - */ - public void setRandomIOExceptionRate(double rate) { - randomIOExceptionRate = rate; - } - public double getRandomIOExceptionRate() { - return randomIOExceptionRate; - } - - void maybeThrowIOException() throws IOException { - maybeThrowIOException(null); - } - - void maybeThrowIOException(String message) throws IOException { - if (randomIOExceptionRate > 0.0) { - int number = Math.abs(randomState.nextInt() % 1000); - if (number < randomIOExceptionRate*1000) { - if (LuceneTestCase.VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": MockDirectoryWrapper: now throw random exception" + (message == null ? "" : " (" + message + ")")); - new Throwable().printStackTrace(System.out); - } - throw new IOException("a random IOException" + (message == null ? "" : "(" + message + ")")); - } - } - } - - @Override - public synchronized void deleteFile(String name) throws IOException { - maybeYield(); - deleteFile(name, false); - } - - // sets the cause of the incoming ioe to be the stack - // trace when the offending file name was opened - private synchronized IOException fillOpenTrace(IOException ioe, String name, boolean input) { - for(Map.Entry ent : openFileHandles.entrySet()) { - if (input && ent.getKey() instanceof MockIndexInputWrapper && ((MockIndexInputWrapper) ent.getKey()).name.equals(name)) { - ioe.initCause(ent.getValue()); - break; - } else if (!input && ent.getKey() instanceof MockIndexOutputWrapper && ((MockIndexOutputWrapper) ent.getKey()).name.equals(name)) { - ioe.initCause(ent.getValue()); - break; - } - } - return ioe; - } - - private void maybeYield() { - if (randomState.nextBoolean()) { - Thread.yield(); - } - } - - private synchronized void deleteFile(String name, boolean forced) throws IOException { - maybeYield(); - - maybeThrowDeterministicException(); - - if (crashed && !forced) - throw new IOException("cannot delete after crash"); - - if (unSyncedFiles.contains(name)) - unSyncedFiles.remove(name); - if (!forced && noDeleteOpenFile) { - if (openFiles.containsKey(name)) { - openFilesDeleted.add(name); - throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot delete"), name, true); - } else { - openFilesDeleted.remove(name); - } - } - delegate.deleteFile(name); - } - - public synchronized Set getOpenDeletedFiles() { - return new HashSet(openFilesDeleted); - } - - @Override - public synchronized IndexOutput createOutput(String name, IOContext context) throws IOException { - maybeYield(); - if (crashed) - throw new IOException("cannot createOutput after crash"); - init(); - synchronized(this) { - if (preventDoubleWrite && createdFiles.contains(name) && !name.equals("segments.gen")) - throw new IOException("file \"" + name + "\" was already written to"); - } - if (noDeleteOpenFile && openFiles.containsKey(name)) - throw new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open: cannot overwrite"); - - if (crashed) - throw new IOException("cannot createOutput after crash"); - unSyncedFiles.add(name); - createdFiles.add(name); - - if (delegate instanceof RAMDirectory) { - RAMDirectory ramdir = (RAMDirectory) delegate; - RAMFile file = new RAMFile(ramdir); - RAMFile existing = ramdir.fileMap.get(name); - - // Enforce write once: - if (existing!=null && !name.equals("segments.gen") && preventDoubleWrite) - throw new IOException("file " + name + " already exists"); - else { - if (existing!=null) { - ramdir.sizeInBytes.getAndAdd(-existing.sizeInBytes); - existing.directory = null; - } - ramdir.fileMap.put(name, file); - } - } - - //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name); - IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name, LuceneTestCase.newIOContext(randomState)), name); - addFileHandle(io, name, Handle.Output); - openFilesForWrite.add(name); - - // throttling REALLY slows down tests, so don't do it very often for SOMETIMES. - if (throttling == Throttling.ALWAYS || - (throttling == Throttling.SOMETIMES && randomState.nextInt(50) == 0)) { - if (LuceneTestCase.VERBOSE) { - System.out.println("MockDirectoryWrapper: throttling indexOutput"); - } - return throttledOutput.newFromDelegate(io); - } else { - return io; - } - } - - private static enum Handle { - Input, Output, Slice - } - - synchronized void addFileHandle(Closeable c, String name, Handle handle) { - Integer v = openFiles.get(name); - if (v != null) { - v = Integer.valueOf(v.intValue()+1); - openFiles.put(name, v); - } else { - openFiles.put(name, Integer.valueOf(1)); - } - - openFileHandles.put(c, new RuntimeException("unclosed Index" + handle.name() + ": " + name)); - } - - @Override - public synchronized IndexInput openInput(String name, IOContext context) throws IOException { - maybeYield(); - if (!delegate.fileExists(name)) - throw new FileNotFoundException(name); - - // cannot open a file for input if it's still open for - // output, except for segments.gen and segments_N - if (openFilesForWrite.contains(name) && !name.startsWith("segments")) { - throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); - } - - IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name, LuceneTestCase.newIOContext(randomState))); - addFileHandle(ii, name, Handle.Input); - return ii; - } - - /** Provided for testing purposes. Use sizeInBytes() instead. */ - public synchronized final long getRecomputedSizeInBytes() throws IOException { - if (!(delegate instanceof RAMDirectory)) - return sizeInBytes(); - long size = 0; - for(final RAMFile file: ((RAMDirectory)delegate).fileMap.values()) { - size += file.getSizeInBytes(); - } - return size; - } - - /** Like getRecomputedSizeInBytes(), but, uses actual file - * lengths rather than buffer allocations (which are - * quantized up to nearest - * RAMOutputStream.BUFFER_SIZE (now 1024) bytes. - */ - - public final synchronized long getRecomputedActualSizeInBytes() throws IOException { - if (!(delegate instanceof RAMDirectory)) - return sizeInBytes(); - long size = 0; - for (final RAMFile file : ((RAMDirectory)delegate).fileMap.values()) - size += file.length; - return size; - } - - @Override - public synchronized void close() throws IOException { - maybeYield(); - if (openFiles == null) { - openFiles = new HashMap(); - openFilesDeleted = new HashSet(); - } - if (noDeleteOpenFile && openFiles.size() > 0) { - // print the first one as its very verbose otherwise - Exception cause = null; - Iterator stacktraces = openFileHandles.values().iterator(); - if (stacktraces.hasNext()) - cause = stacktraces.next(); - // RuntimeException instead of IOException because - // super() does not throw IOException currently: - throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open files: " + openFiles, cause); - } - if (noDeleteOpenFile && openLocks.size() > 0) { - throw new RuntimeException("MockDirectoryWrapper: cannot close: there are still open locks: " + openLocks); - } - open = false; - if (checkIndexOnClose) { - if (LuceneTestCase.VERBOSE) { - System.out.println("\nNOTE: MockDirectoryWrapper: now run CheckIndex"); - } - if (codecProvider != null) { - if (IndexReader.indexExists(this, codecProvider)) { - _TestUtil.checkIndex(this, codecProvider); - } - } else { - if (IndexReader.indexExists(this)) { - _TestUtil.checkIndex(this); - } - } - } - delegate.close(); - } - - synchronized void removeOpenFile(Closeable c, String name) { - Integer v = openFiles.get(name); - // Could be null when crash() was called - if (v != null) { - if (v.intValue() == 1) { - openFiles.remove(name); - openFilesDeleted.remove(name); - } else { - v = Integer.valueOf(v.intValue()-1); - openFiles.put(name, v); - } - } - - openFileHandles.remove(c); - } - - public synchronized void removeIndexOutput(IndexOutput out, String name) { - openFilesForWrite.remove(name); - removeOpenFile(out, name); - } - - public synchronized void removeIndexInput(IndexInput in, String name) { - removeOpenFile(in, name); - } - - private CodecProvider codecProvider; - - // We pass this CodecProvider to checkIndex when dir is closed... - public void setCodecProvider(CodecProvider cp) { - codecProvider = cp; - } - - boolean open = true; - - public synchronized boolean isOpen() { - return open; - } - - /** - * Objects that represent fail-able conditions. Objects of a derived - * class are created and registered with the mock directory. After - * register, each object will be invoked once for each first write - * of a file, giving the object a chance to throw an IOException. - */ - public static class Failure { - /** - * eval is called on the first write of every new file. - */ - public void eval(MockDirectoryWrapper dir) throws IOException { } - - /** - * reset should set the state of the failure to its default - * (freshly constructed) state. Reset is convenient for tests - * that want to create one failure object and then reuse it in - * multiple cases. This, combined with the fact that Failure - * subclasses are often anonymous classes makes reset difficult to - * do otherwise. - * - * A typical example of use is - * Failure failure = new Failure() { ... }; - * ... - * mock.failOn(failure.reset()) - */ - public Failure reset() { return this; } - - protected boolean doFail; - - public void setDoFail() { - doFail = true; - } - - public void clearDoFail() { - doFail = false; - } - } - - ArrayList failures; - - /** - * add a Failure object to the list of objects to be evaluated - * at every potential failure point - */ - synchronized public void failOn(Failure fail) { - if (failures == null) { - failures = new ArrayList(); - } - failures.add(fail); - } - - /** - * Iterate through the failures list, giving each object a - * chance to throw an IOE - */ - synchronized void maybeThrowDeterministicException() throws IOException { - if (failures != null) { - for(int i = 0; i < failures.size(); i++) { - failures.get(i).eval(this); - } - } - } - - @Override - public synchronized String[] listAll() throws IOException { - maybeYield(); - return delegate.listAll(); - } - - @Override - public synchronized boolean fileExists(String name) throws IOException { - maybeYield(); - return delegate.fileExists(name); - } - - @Override - public synchronized long fileModified(String name) throws IOException { - maybeYield(); - return delegate.fileModified(name); - } - - @Override - public synchronized long fileLength(String name) throws IOException { - maybeYield(); - return delegate.fileLength(name); - } - - @Override - public synchronized Lock makeLock(String name) { - maybeYield(); - return delegate.makeLock(name); - } - - @Override - public synchronized void clearLock(String name) throws IOException { - maybeYield(); - delegate.clearLock(name); - } - - @Override - public synchronized void setLockFactory(LockFactory lockFactory) throws IOException { - maybeYield(); - delegate.setLockFactory(lockFactory); - } - - @Override - public synchronized LockFactory getLockFactory() { - maybeYield(); - return delegate.getLockFactory(); - } - - @Override - public synchronized String getLockID() { - maybeYield(); - return delegate.getLockID(); - } - - @Override - public synchronized void copy(Directory to, String src, String dest, IOContext context) throws IOException { - maybeYield(); - // randomize the IOContext here? - delegate.copy(to, src, dest, context); - } - - @Override - public IndexInputSlicer createSlicer(final String name, IOContext context) - throws IOException { - maybeYield(); - if (!delegate.fileExists(name)) - throw new FileNotFoundException(name); - // cannot open a file for input if it's still open for - // output, except for segments.gen and segments_N - if (openFilesForWrite.contains(name) && !name.startsWith("segments")) { - throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); - } - - final IndexInputSlicer delegateHandle = delegate.createSlicer(name, context); - final IndexInputSlicer handle = new IndexInputSlicer() { - - private boolean isClosed; - @Override - public void close() throws IOException { - if (!isClosed) { - delegateHandle.close(); - MockDirectoryWrapper.this.removeOpenFile(this, name); - isClosed = true; - } - } - - @Override - public IndexInput openSlice(long offset, long length) throws IOException { - maybeYield(); - IndexInput ii = new MockIndexInputWrapper(MockDirectoryWrapper.this, name, delegateHandle.openSlice(offset, length)); - addFileHandle(ii, name, Handle.Input); - return ii; - } - - @Override - public IndexInput openFullSlice() throws IOException { - maybeYield(); - IndexInput ii = new MockIndexInputWrapper(MockDirectoryWrapper.this, name, delegateHandle.openFullSlice()); - addFileHandle(ii, name, Handle.Input); - return ii; - } - - }; - addFileHandle(handle, name, Handle.Slice); - return handle; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java lucene-2621/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java 2011-11-01 18:41:12.556614525 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,180 +0,0 @@ -package org.apache.lucene.store; - -import java.io.IOException; -import java.util.Map; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * Used by MockDirectoryWrapper to create an input stream that - * keeps track of when it's been closed. - */ - -public class MockIndexInputWrapper extends IndexInput { - private MockDirectoryWrapper dir; - final String name; - private IndexInput delegate; - private boolean isClone; - private boolean closed; - - /** Construct an empty output buffer. */ - public MockIndexInputWrapper(MockDirectoryWrapper dir, String name, IndexInput delegate) { - this.name = name; - this.dir = dir; - this.delegate = delegate; - } - - @Override - public void close() throws IOException { - try { - // turn on the following to look for leaks closing inputs, - // after fixing TestTransactions - // dir.maybeThrowDeterministicException(); - } finally { - closed = true; - delegate.close(); - // Pending resolution on LUCENE-686 we may want to - // remove the conditional check so we also track that - // all clones get closed: - if (!isClone) { - dir.removeIndexInput(this, name); - } - } - } - - private void ensureOpen() { - if (closed) { - throw new RuntimeException("Abusing closed IndexInput!"); - } - } - - @Override - public Object clone() { - ensureOpen(); - dir.inputCloneCount.incrementAndGet(); - IndexInput iiclone = (IndexInput) delegate.clone(); - MockIndexInputWrapper clone = new MockIndexInputWrapper(dir, name, iiclone); - clone.isClone = true; - // Pending resolution on LUCENE-686 we may want to - // uncomment this code so that we also track that all - // clones get closed: - /* - synchronized(dir.openFiles) { - if (dir.openFiles.containsKey(name)) { - Integer v = (Integer) dir.openFiles.get(name); - v = Integer.valueOf(v.intValue()+1); - dir.openFiles.put(name, v); - } else { - throw new RuntimeException("BUG: cloned file was not open?"); - } - } - */ - return clone; - } - - @Override - public long getFilePointer() { - ensureOpen(); - return delegate.getFilePointer(); - } - - @Override - public void seek(long pos) throws IOException { - ensureOpen(); - delegate.seek(pos); - } - - @Override - public long length() { - ensureOpen(); - return delegate.length(); - } - - @Override - public byte readByte() throws IOException { - ensureOpen(); - return delegate.readByte(); - } - - @Override - public void readBytes(byte[] b, int offset, int len) throws IOException { - ensureOpen(); - delegate.readBytes(b, offset, len); - } - - @Override - public void copyBytes(IndexOutput out, long numBytes) throws IOException { - ensureOpen(); - delegate.copyBytes(out, numBytes); - } - - @Override - public void readBytes(byte[] b, int offset, int len, boolean useBuffer) - throws IOException { - ensureOpen(); - delegate.readBytes(b, offset, len, useBuffer); - } - - @Override - public short readShort() throws IOException { - ensureOpen(); - return delegate.readShort(); - } - - @Override - public int readInt() throws IOException { - ensureOpen(); - return delegate.readInt(); - } - - @Override - public long readLong() throws IOException { - ensureOpen(); - return delegate.readLong(); - } - - @Override - public String readString() throws IOException { - ensureOpen(); - return delegate.readString(); - } - - @Override - public Map readStringStringMap() throws IOException { - ensureOpen(); - return delegate.readStringStringMap(); - } - - @Override - public int readVInt() throws IOException { - ensureOpen(); - return delegate.readVInt(); - } - - @Override - public long readVLong() throws IOException { - ensureOpen(); - return delegate.readVLong(); - } - - @Override - public String toString() { - return "MockIndexInputWrapper(" + delegate + ")"; - } -} - diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java lucene-2621/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java 2011-11-01 18:41:12.556614525 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,164 +0,0 @@ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.util.LuceneTestCase; - -/** - * Used by MockRAMDirectory to create an output stream that - * will throw an IOException on fake disk full, track max - * disk space actually used, and maybe throw random - * IOExceptions. - */ - -public class MockIndexOutputWrapper extends IndexOutput { - private MockDirectoryWrapper dir; - private final IndexOutput delegate; - private boolean first=true; - final String name; - - byte[] singleByte = new byte[1]; - - /** Construct an empty output buffer. */ - public MockIndexOutputWrapper(MockDirectoryWrapper dir, IndexOutput delegate, String name) { - this.dir = dir; - this.name = name; - this.delegate = delegate; - } - - @Override - public void close() throws IOException { - try { - dir.maybeThrowDeterministicException(); - } finally { - delegate.close(); - if (dir.trackDiskUsage) { - // Now compute actual disk usage & track the maxUsedSize - // in the MockDirectoryWrapper: - long size = dir.getRecomputedActualSizeInBytes(); - if (size > dir.maxUsedSize) { - dir.maxUsedSize = size; - } - } - dir.removeIndexOutput(this, name); - } - } - - @Override - public void flush() throws IOException { - dir.maybeThrowDeterministicException(); - delegate.flush(); - } - - @Override - public void writeByte(byte b) throws IOException { - singleByte[0] = b; - writeBytes(singleByte, 0, 1); - } - - @Override - public void writeBytes(byte[] b, int offset, int len) throws IOException { - long freeSpace = dir.maxSize == 0 ? 0 : dir.maxSize - dir.sizeInBytes(); - long realUsage = 0; - - // If MockRAMDir crashed since we were opened, then - // don't write anything: - if (dir.crashed) - throw new IOException("MockRAMDirectory was crashed; cannot write to " + name); - - // Enforce disk full: - if (dir.maxSize != 0 && freeSpace <= len) { - // Compute the real disk free. This will greatly slow - // down our test but makes it more accurate: - realUsage = dir.getRecomputedActualSizeInBytes(); - freeSpace = dir.maxSize - realUsage; - } - - if (dir.maxSize != 0 && freeSpace <= len) { - if (freeSpace > 0) { - realUsage += freeSpace; - delegate.writeBytes(b, offset, (int) freeSpace); - } - if (realUsage > dir.maxUsedSize) { - dir.maxUsedSize = realUsage; - } - String message = "fake disk full at " + dir.getRecomputedActualSizeInBytes() + " bytes when writing " + name + " (file length=" + delegate.length(); - if (freeSpace > 0) { - message += "; wrote " + freeSpace + " of " + len + " bytes"; - } - message += ")"; - if (LuceneTestCase.VERBOSE) { - System.out.println(Thread.currentThread().getName() + ": MDW: now throw fake disk full"); - new Throwable().printStackTrace(System.out); - } - throw new IOException(message); - } else { - if (dir.randomState.nextInt(200) == 0) { - final int half = len/2; - delegate.writeBytes(b, offset, half); - Thread.yield(); - delegate.writeBytes(b, offset+half, len-half); - } else { - delegate.writeBytes(b, offset, len); - } - } - - dir.maybeThrowDeterministicException(); - - if (first) { - // Maybe throw random exception; only do this on first - // write to a new file: - first = false; - dir.maybeThrowIOException(name); - } - } - - @Override - public long getFilePointer() { - return delegate.getFilePointer(); - } - - @Override - public void seek(long pos) throws IOException { - delegate.seek(pos); - } - - @Override - public long length() throws IOException { - return delegate.length(); - } - - @Override - public void setLength(long length) throws IOException { - delegate.setLength(length); - } - - @Override - public void copyBytes(DataInput input, long numBytes) throws IOException { - delegate.copyBytes(input, numBytes); - // TODO: we may need to check disk full here as well - dir.maybeThrowDeterministicException(); - } - - @Override - public String toString() { - return "MockIndexOutputWrapper(" + delegate + ")"; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/store/MockLockFactoryWrapper.java lucene-2621/lucene/src/test-framework/org/apache/lucene/store/MockLockFactoryWrapper.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/store/MockLockFactoryWrapper.java 2011-11-01 18:41:12.556614525 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/store/MockLockFactoryWrapper.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,87 +0,0 @@ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -public class MockLockFactoryWrapper extends LockFactory { - MockDirectoryWrapper dir; - LockFactory delegate; - - public MockLockFactoryWrapper(MockDirectoryWrapper dir, LockFactory delegate) { - this.dir = dir; - this.delegate = delegate; - } - - @Override - public void setLockPrefix(String lockPrefix) { - delegate.setLockPrefix(lockPrefix); - } - - @Override - public String getLockPrefix() { - return delegate.getLockPrefix(); - } - - @Override - public Lock makeLock(String lockName) { - return new MockLock(delegate.makeLock(lockName), lockName); - } - - @Override - public void clearLock(String lockName) throws IOException { - delegate.clearLock(lockName); - dir.openLocks.remove(lockName); - } - - @Override - public String toString() { - return "MockLockFactoryWrapper(" + delegate.toString() + ")"; - } - - private class MockLock extends Lock { - private Lock delegateLock; - private String name; - - MockLock(Lock delegate, String name) { - this.delegateLock = delegate; - this.name = name; - } - - @Override - public boolean obtain() throws IOException { - if (delegateLock.obtain()) { - dir.openLocks.add(name); - return true; - } else { - return false; - } - } - - @Override - public void release() throws IOException { - delegateLock.release(); - dir.openLocks.remove(name); - } - - @Override - public boolean isLocked() throws IOException { - return delegateLock.isLocked(); - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/store/_TestHelper.java lucene-2621/lucene/src/test-framework/org/apache/lucene/store/_TestHelper.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/store/_TestHelper.java 2011-11-01 18:41:12.556614525 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/store/_TestHelper.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,65 +0,0 @@ -package org.apache.lucene.store; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.IOException; - -import org.apache.lucene.store.SimpleFSDirectory.SimpleFSIndexInput; - -/** This class provides access to package-level features defined in the - * store package. It is used for testing only. - */ -public class _TestHelper { - - /** Returns true if the instance of the provided input stream is actually - * an SimpleFSIndexInput. - */ - public static boolean isSimpleFSIndexInput(IndexInput is) { - return is instanceof SimpleFSIndexInput; - } - - /** Returns true if the provided input stream is an SimpleFSIndexInput and - * is a clone, that is it does not own its underlying file descriptor. - */ - public static boolean isSimpleFSIndexInputClone(IndexInput is) { - if (isSimpleFSIndexInput(is)) { - return ((SimpleFSIndexInput) is).isClone; - } else { - return false; - } - } - - /** Given an instance of SimpleFSDirectory.SimpleFSIndexInput, this method returns - * true if the underlying file descriptor is valid, and false otherwise. - * This can be used to determine if the OS file has been closed. - * The descriptor becomes invalid when the non-clone instance of the - * SimpleFSIndexInput that owns this descriptor is closed. However, the - * descriptor may possibly become invalid in other ways as well. - */ - public static boolean isSimpleFSIndexInputOpen(IndexInput is) - throws IOException - { - if (isSimpleFSIndexInput(is)) { - SimpleFSIndexInput fis = (SimpleFSIndexInput) is; - return fis.isFDValid(); - } else { - return false; - } - } - -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java lucene-2621/lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java 2011-11-01 18:41:12.564614526 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,411 +0,0 @@ -package org.apache.lucene.util.automaton; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.IdentityHashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.Set; - -import org.apache.lucene.util.ArrayUtil; -import org.apache.lucene.util.UnicodeUtil; -import org.apache.lucene.util._TestUtil; - -public class AutomatonTestUtil { - /** Returns random string, including full unicode range. */ - public static String randomRegexp(Random r) { - while (true) { - String regexp = randomRegexpString(r); - // we will also generate some undefined unicode queries - if (!UnicodeUtil.validUTF16String(regexp)) - continue; - try { - new RegExp(regexp, RegExp.NONE); - return regexp; - } catch (Exception e) {} - } - } - - private static String randomRegexpString(Random r) { - final int end = r.nextInt(20); - if (end == 0) { - // allow 0 length - return ""; - } - final char[] buffer = new char[end]; - for (int i = 0; i < end; i++) { - int t = r.nextInt(15); - if (0 == t && i < end - 1) { - // Make a surrogate pair - // High surrogate - buffer[i++] = (char) _TestUtil.nextInt(r, 0xd800, 0xdbff); - // Low surrogate - buffer[i] = (char) _TestUtil.nextInt(r, 0xdc00, 0xdfff); - } - else if (t <= 1) buffer[i] = (char) r.nextInt(0x80); - else if (2 == t) buffer[i] = (char) _TestUtil.nextInt(r, 0x80, 0x800); - else if (3 == t) buffer[i] = (char) _TestUtil.nextInt(r, 0x800, 0xd7ff); - else if (4 == t) buffer[i] = (char) _TestUtil.nextInt(r, 0xe000, 0xffff); - else if (5 == t) buffer[i] = '.'; - else if (6 == t) buffer[i] = '?'; - else if (7 == t) buffer[i] = '*'; - else if (8 == t) buffer[i] = '+'; - else if (9 == t) buffer[i] = '('; - else if (10 == t) buffer[i] = ')'; - else if (11 == t) buffer[i] = '-'; - else if (12 == t) buffer[i] = '['; - else if (13 == t) buffer[i] = ']'; - else if (14 == t) buffer[i] = '|'; - } - return new String(buffer, 0, end); - } - - /** picks a random int code point, avoiding surrogates; - * throws IllegalArgumentException if this transition only - * accepts surrogates */ - private static int getRandomCodePoint(final Random r, final Transition t) { - final int code; - if (t.max < UnicodeUtil.UNI_SUR_HIGH_START || - t.min > UnicodeUtil.UNI_SUR_HIGH_END) { - // easy: entire range is before or after surrogates - code = t.min+r.nextInt(t.max-t.min+1); - } else if (t.min >= UnicodeUtil.UNI_SUR_HIGH_START) { - if (t.max > UnicodeUtil.UNI_SUR_LOW_END) { - // after surrogates - code = 1+UnicodeUtil.UNI_SUR_LOW_END+r.nextInt(t.max-UnicodeUtil.UNI_SUR_LOW_END); - } else { - throw new IllegalArgumentException("transition accepts only surrogates: " + t); - } - } else if (t.max <= UnicodeUtil.UNI_SUR_LOW_END) { - if (t.min < UnicodeUtil.UNI_SUR_HIGH_START) { - // before surrogates - code = t.min + r.nextInt(UnicodeUtil.UNI_SUR_HIGH_START - t.min); - } else { - throw new IllegalArgumentException("transition accepts only surrogates: " + t); - } - } else { - // range includes all surrogates - int gap1 = UnicodeUtil.UNI_SUR_HIGH_START - t.min; - int gap2 = t.max - UnicodeUtil.UNI_SUR_LOW_END; - int c = r.nextInt(gap1+gap2); - if (c < gap1) { - code = t.min + c; - } else { - code = UnicodeUtil.UNI_SUR_LOW_END + c - gap1 + 1; - } - } - - assert code >= t.min && code <= t.max && (code < UnicodeUtil.UNI_SUR_HIGH_START || code > UnicodeUtil.UNI_SUR_LOW_END): - "code=" + code + " min=" + t.min + " max=" + t.max; - return code; - } - - public static class RandomAcceptedStrings { - - private final Map leadsToAccept; - private final Automaton a; - - private static class ArrivingTransition { - final State from; - final Transition t; - public ArrivingTransition(State from, Transition t) { - this.from = from; - this.t = t; - } - } - - public RandomAcceptedStrings(Automaton a) { - this.a = a; - if (a.isSingleton()) { - leadsToAccept = null; - return; - } - - // must use IdentityHashmap because two Transitions w/ - // different start nodes can be considered the same - leadsToAccept = new IdentityHashMap(); - final Map> allArriving = new HashMap>(); - - final LinkedList q = new LinkedList(); - final Set seen = new HashSet(); - - // reverse map the transitions, so we can quickly look - // up all arriving transitions to a given state - for(State s: a.getNumberedStates()) { - for(int i=0;i tl = allArriving.get(t.to); - if (tl == null) { - tl = new ArrayList(); - allArriving.put(t.to, tl); - } - tl.add(new ArrivingTransition(s, t)); - } - if (s.accept) { - q.add(s); - seen.add(s); - } - } - - // Breadth-first search, from accept states, - // backwards: - while(!q.isEmpty()) { - final State s = q.removeFirst(); - List arriving = allArriving.get(s); - if (arriving != null) { - for(ArrivingTransition at : arriving) { - final State from = at.from; - if (!seen.contains(from)) { - q.add(from); - seen.add(from); - leadsToAccept.put(at.t, Boolean.TRUE); - } - } - } - } - } - - public int[] getRandomAcceptedString(Random r) { - - final List soFar = new ArrayList(); - if (a.isSingleton()) { - // accepts only one - final String s = a.singleton; - - int charUpto = 0; - while(charUpto < s.length()) { - final int cp = s.codePointAt(charUpto); - charUpto += Character.charCount(cp); - soFar.add(cp); - } - } else { - - State s = a.initial; - - while(true) { - - if (s.accept) { - if (s.numTransitions == 0) { - // stop now - break; - } else { - if (r.nextBoolean()) { - break; - } - } - } - - if (s.numTransitions == 0) { - throw new RuntimeException("this automaton has dead states"); - } - - boolean cheat = r.nextBoolean(); - - final Transition t; - if (cheat) { - // pick a transition that we know is the fastest - // path to an accept state - List toAccept = new ArrayList(); - for(int i=0;i initialset = new HashSet(); - initialset.add(a.initial); - determinizeSimple(a, initialset); - } - - /** - * Simple, original brics implementation of determinize() - * Determinizes the given automaton using the given set of initial states. - */ - public static void determinizeSimple(Automaton a, Set initialset) { - int[] points = a.getStartPoints(); - // subset construction - Map, Set> sets = new HashMap, Set>(); - LinkedList> worklist = new LinkedList>(); - Map, State> newstate = new HashMap, State>(); - sets.put(initialset, initialset); - worklist.add(initialset); - a.initial = new State(); - newstate.put(initialset, a.initial); - while (worklist.size() > 0) { - Set s = worklist.removeFirst(); - State r = newstate.get(s); - for (State q : s) - if (q.accept) { - r.accept = true; - break; - } - for (int n = 0; n < points.length; n++) { - Set p = new HashSet(); - for (State q : s) - for (Transition t : q.getTransitions()) - if (t.min <= points[n] && points[n] <= t.max) - p.add(t.to); - if (!sets.containsKey(p)) { - sets.put(p, p); - worklist.add(p); - newstate.put(p, new State()); - } - State q = newstate.get(p); - int min = points[n]; - int max; - if (n + 1 < points.length) - max = points[n + 1] - 1; - else - max = Character.MAX_CODE_POINT; - r.addTransition(new Transition(min, max, q)); - } - } - a.deterministic = true; - a.clearNumberedStates(); - a.removeDeadTransitions(); - } - - /** - * Returns true if the language of this automaton is finite. - *

    - * WARNING: this method is slow, it will blow up if the automaton is large. - * this is only used to test the correctness of our faster implementation. - */ - public static boolean isFiniteSlow(Automaton a) { - if (a.isSingleton()) return true; - return isFiniteSlow(a.initial, new HashSet()); - } - - /** - * Checks whether there is a loop containing s. (This is sufficient since - * there are never transitions to dead states.) - */ - // TODO: not great that this is recursive... in theory a - // large automata could exceed java's stack - private static boolean isFiniteSlow(State s, HashSet path) { - path.add(s); - for (Transition t : s.getTransitions()) - if (path.contains(t.to) || !isFiniteSlow(t.to, path)) return false; - path.remove(s); - return true; - } - - - /** - * Checks that an automaton has no detached states that are unreachable - * from the initial state. - */ - public static void assertNoDetachedStates(Automaton a) { - int numStates = a.getNumberOfStates(); - a.clearNumberedStates(); // force recomputation of cached numbered states - assert numStates == a.getNumberOfStates() : "automaton has " + (numStates - a.getNumberOfStates()) + " detached states"; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java lucene-2621/lucene/src/test-framework/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java 2011-11-01 18:41:12.564614526 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,361 +0,0 @@ -package org.apache.lucene.util.automaton; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.*; - -import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.CharsRef; -import org.apache.lucene.util.UnicodeUtil; - -/** - * Builds a minimal deterministic automaton that accepts a set of strings. The - * algorithm requires sorted input data, but is very fast (nearly linear with - * the input size). - */ -public final class DaciukMihovAutomatonBuilder { - /** - * DFSA state with char labels on transitions. - */ - final static class State { - - /** An empty set of labels. */ - private final static int[] NO_LABELS = new int[0]; - - /** An empty set of states. */ - private final static State[] NO_STATES = new State[0]; - - /** - * Labels of outgoing transitions. Indexed identically to {@link #states}. - * Labels must be sorted lexicographically. - */ - int[] labels = NO_LABELS; - - /** - * States reachable from outgoing transitions. Indexed identically to - * {@link #labels}. - */ - State[] states = NO_STATES; - - /** - * true if this state corresponds to the end of at least one - * input sequence. - */ - boolean is_final; - - /** - * Returns the target state of a transition leaving this state and labeled - * with label. If no such transition exists, returns - * null. - */ - public State getState(int label) { - final int index = Arrays.binarySearch(labels, label); - return index >= 0 ? states[index] : null; - } - - /** - * Returns an array of outgoing transition labels. The array is sorted in - * lexicographic order and indexes correspond to states returned from - * {@link #getStates()}. - */ - public int[] getTransitionLabels() { - return this.labels; - } - - /** - * Returns an array of outgoing transitions from this state. The returned - * array must not be changed. - */ - public State[] getStates() { - return this.states; - } - - /** - * Two states are equal if: - *

      - *
    • they have an identical number of outgoing transitions, labeled with - * the same labels
      • - *
    • corresponding outgoing transitions lead to the same states (to states - * with an identical right-language). - *
    - */ - @Override - public boolean equals(Object obj) { - final State other = (State) obj; - return is_final == other.is_final - && Arrays.equals(this.labels, other.labels) - && referenceEquals(this.states, other.states); - } - - /** - * Return true if this state has any children (outgoing - * transitions). - */ - public boolean hasChildren() { - return labels.length > 0; - } - - /** - * Is this state a final state in the automaton? - */ - public boolean isFinal() { - return is_final; - } - - /** - * Compute the hash code of the current status of this state. - */ - @Override - public int hashCode() { - int hash = is_final ? 1 : 0; - - hash ^= hash * 31 + this.labels.length; - for (int c : this.labels) - hash ^= hash * 31 + c; - - /* - * Compare the right-language of this state using reference-identity of - * outgoing states. This is possible because states are interned (stored - * in registry) and traversed in post-order, so any outgoing transitions - * are already interned. - */ - for (State s : this.states) { - hash ^= System.identityHashCode(s); - } - - return hash; - } - - /** - * Create a new outgoing transition labeled label and return - * the newly created target state for this transition. - */ - State newState(int label) { - assert Arrays.binarySearch(labels, label) < 0 : "State already has transition labeled: " - + label; - - labels = copyOf(labels, labels.length + 1); - states = copyOf(states, states.length + 1); - - labels[labels.length - 1] = label; - return states[states.length - 1] = new State(); - } - - /** - * Return the most recent transitions's target state. - */ - State lastChild() { - assert hasChildren() : "No outgoing transitions."; - return states[states.length - 1]; - } - - /** - * Return the associated state if the most recent transition is labeled with - * label. - */ - State lastChild(int label) { - final int index = labels.length - 1; - State s = null; - if (index >= 0 && labels[index] == label) { - s = states[index]; - } - assert s == getState(label); - return s; - } - - /** - * Replace the last added outgoing transition's target state with the given - * state. - */ - void replaceLastChild(State state) { - assert hasChildren() : "No outgoing transitions."; - states[states.length - 1] = state; - } - - /** - * JDK1.5-replacement of {@link Arrays#copyOf(int[], int)} - */ - private static int[] copyOf(int[] original, int newLength) { - int[] copy = new int[newLength]; - System.arraycopy(original, 0, copy, 0, - Math.min(original.length, newLength)); - return copy; - } - - /** - * JDK1.5-replacement of {@link Arrays#copyOf(char[], int)} - */ - public static State[] copyOf(State[] original, int newLength) { - State[] copy = new State[newLength]; - System.arraycopy(original, 0, copy, 0, - Math.min(original.length, newLength)); - return copy; - } - - /** - * Compare two lists of objects for reference-equality. - */ - private static boolean referenceEquals(Object[] a1, Object[] a2) { - if (a1.length != a2.length) return false; - - for (int i = 0; i < a1.length; i++) - if (a1[i] != a2[i]) return false; - - return true; - } - } - - /** - * "register" for state interning. - */ - private HashMap register = new HashMap(); - - /** - * Root automaton state. - */ - private State root = new State(); - - /** - * Previous sequence added to the automaton in {@link #add(CharSequence)}. - */ - private CharsRef previous; - - private static final Comparator comparator = CharsRef.getUTF16SortedAsUTF8Comparator(); - - /** - * Add another character sequence to this automaton. The sequence must be - * lexicographically larger or equal compared to any previous sequences added - * to this automaton (the input must be sorted). - */ - public void add(CharsRef current) { - assert register != null : "Automaton already built."; - assert previous == null - || comparator.compare(previous, current) <= 0 : "Input must be sorted: " - + previous + " >= " + current; - assert setPrevious(current); - - // Descend in the automaton (find matching prefix). - int pos = 0, max = current.length(); - State next, state = root; - while (pos < max && (next = state.lastChild(Character.codePointAt(current, pos))) != null) { - state = next; - // todo, optimize me - pos += Character.charCount(Character.codePointAt(current, pos)); - } - - if (state.hasChildren()) replaceOrRegister(state); - - addSuffix(state, current, pos); - } - - /** - * Finalize the automaton and return the root state. No more strings can be - * added to the builder after this call. - * - * @return Root automaton state. - */ - public State complete() { - if (this.register == null) throw new IllegalStateException(); - - if (root.hasChildren()) replaceOrRegister(root); - - register = null; - return root; - } - - /** - * Internal recursive traversal for conversion. - */ - private static org.apache.lucene.util.automaton.State convert(State s, - IdentityHashMap visited) { - org.apache.lucene.util.automaton.State converted = visited.get(s); - if (converted != null) return converted; - - converted = new org.apache.lucene.util.automaton.State(); - converted.setAccept(s.is_final); - - visited.put(s, converted); - int i = 0; - int[] labels = s.labels; - for (DaciukMihovAutomatonBuilder.State target : s.states) { - converted.addTransition(new Transition(labels[i++], convert(target, - visited))); - } - - return converted; - } - - /** - * Build a minimal, deterministic automaton from a sorted list of strings. - */ - public static Automaton build(Collection input) { - final DaciukMihovAutomatonBuilder builder = new DaciukMihovAutomatonBuilder(); - - CharsRef scratch = new CharsRef(); - for (BytesRef b : input) { - UnicodeUtil.UTF8toUTF16(b, scratch); - builder.add(scratch); - } - - Automaton a = new Automaton(); - a.initial = convert(builder.complete(), new IdentityHashMap()); - a.deterministic = true; - return a; - } - - /** - * Copy current into an internal buffer. - */ - private boolean setPrevious(CharsRef current) { - // don't need to copy, once we fix https://issues.apache.org/jira/browse/LUCENE-3277 - // still, called only from assert - previous = new CharsRef(current); - return true; - } - - /** - * Replace last child of state with an already registered state - * or register the last child state. - */ - private void replaceOrRegister(State state) { - final State child = state.lastChild(); - - if (child.hasChildren()) replaceOrRegister(child); - - final State registered = register.get(child); - if (registered != null) { - state.replaceLastChild(registered); - } else { - register.put(child, child); - } - } - - /** - * Add a suffix of current starting at fromIndex - * (inclusive) to state state. - */ - private void addSuffix(State state, CharSequence current, int fromIndex) { - final int len = current.length(); - while (fromIndex < len) { - int cp = Character.codePointAt(current, fromIndex); - state = state.newState(cp); - fromIndex += Character.charCount(cp); - } - state.is_final = true; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java lucene-2621/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java 2011-11-01 18:41:12.560614526 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,186 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.Closeable; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.io.InputStream; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.zip.GZIPInputStream; -import java.util.Random; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.document.StringField; -import org.apache.lucene.document.TextField; - -/** Minimal port of contrib/benchmark's LneDocSource + - * DocMaker, so tests can enum docs from a line file created - * by contrib/benchmark's WriteLineDoc task */ -public class LineFileDocs implements Closeable { - - private BufferedReader reader; - private final static int BUFFER_SIZE = 1 << 16; // 64K - private final AtomicInteger id = new AtomicInteger(); - private final String path; - - /** If forever is true, we rewind the file at EOF (repeat - * the docs over and over) */ - public LineFileDocs(Random random, String path) throws IOException { - this.path = path; - open(random); - } - - public LineFileDocs(Random random) throws IOException { - this(random, LuceneTestCase.TEST_LINE_DOCS_FILE); - } - - public synchronized void close() throws IOException { - if (reader != null) { - reader.close(); - reader = null; - } - } - - private synchronized void open(Random random) throws IOException { - InputStream is = getClass().getResourceAsStream(path); - if (is == null) { - // if its not in classpath, we load it as absolute filesystem path (e.g. Hudson's home dir) - is = new FileInputStream(path); - } - File file = new File(path); - long size; - if (file.exists()) { - size = file.length(); - } else { - size = is.available(); - } - if (path.endsWith(".gz")) { - is = new GZIPInputStream(is); - // guestimate: - size *= 2.8; - } - - reader = new BufferedReader(new InputStreamReader(is, "UTF-8"), BUFFER_SIZE); - - // Override sizes for currently "known" line files: - if (path.equals("europarl.lines.txt.gz")) { - size = 15129506L; - } else if (path.equals("/home/hudson/lucene-data/enwiki.random.lines.txt.gz")) { - size = 3038178822L; - } - - // Randomly seek to starting point: - if (random != null && size > 3) { - final long seekTo = (random.nextLong()&Long.MAX_VALUE) % (size/3); - if (LuceneTestCase.VERBOSE) { - System.out.println("TEST: LineFileDocs: seek to fp=" + seekTo + " on open"); - } - reader.skip(seekTo); - reader.readLine(); - } - } - - public synchronized void reset(Random random) throws IOException { - close(); - open(random); - id.set(0); - } - - private final static char SEP = '\t'; - - private static final class DocState { - final Document doc; - final Field titleTokenized; - final Field title; - final Field body; - final Field id; - final Field date; - - public DocState() { - doc = new Document(); - - title = new StringField("title", ""); - doc.add(title); - - FieldType ft = new FieldType(TextField.TYPE_STORED); - ft.setStoreTermVectors(true); - ft.setStoreTermVectorOffsets(true); - ft.setStoreTermVectorPositions(true); - - titleTokenized = new Field("titleTokenized", "", ft); - doc.add(titleTokenized); - - body = new Field("body", "", ft); - doc.add(body); - - id = new Field("docid", "", StringField.TYPE_STORED); - doc.add(id); - - date = new Field("date", "", StringField.TYPE_STORED); - doc.add(date); - } - } - - private final ThreadLocal threadDocs = new ThreadLocal(); - - /** Note: Document instance is re-used per-thread */ - public Document nextDoc() throws IOException { - String line; - synchronized(this) { - line = reader.readLine(); - if (line == null) { - // Always rewind at end: - if (LuceneTestCase.VERBOSE) { - System.out.println("TEST: LineFileDocs: now rewind file..."); - } - close(); - open(null); - line = reader.readLine(); - } - } - - DocState docState = threadDocs.get(); - if (docState == null) { - docState = new DocState(); - threadDocs.set(docState); - } - - int spot = line.indexOf(SEP); - if (spot == -1) { - throw new RuntimeException("line: [" + line + "] is in an invalid format !"); - } - int spot2 = line.indexOf(SEP, 1 + spot); - if (spot2 == -1) { - throw new RuntimeException("line: [" + line + "] is in an invalid format !"); - } - - docState.body.setValue(line.substring(1+spot2, line.length())); - final String title = line.substring(0, spot); - docState.title.setValue(title); - docState.titleTokenized.setValue(title); - docState.date.setValue(line.substring(1+spot, spot2)); - docState.id.setValue(Integer.toString(id.getAndIncrement())); - return docState.doc; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitDividingSelector.java lucene-2621/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitDividingSelector.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitDividingSelector.java 2011-11-01 18:41:12.560614526 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitDividingSelector.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -package org.apache.lucene.util; -import java.io.File; - -import org.apache.tools.ant.BuildException; -import org.apache.tools.ant.types.Parameter; -import org.apache.tools.ant.types.selectors.BaseExtendSelector; - -/** Divides filesets into equal groups */ -public class LuceneJUnitDividingSelector extends BaseExtendSelector { - private int counter; - /** Number of total parts to split. */ - private int divisor; - /** Current part to accept. */ - private int part; - - @Override - public void setParameters(Parameter[] pParameters) { - super.setParameters(pParameters); - for (int j = 0; j < pParameters.length; j++) { - Parameter p = pParameters[j]; - if ("divisor".equalsIgnoreCase(p.getName())) { - divisor = Integer.parseInt(p.getValue()); - } - else if ("part".equalsIgnoreCase(p.getName())) { - part = Integer.parseInt(p.getValue()); - } - else { - throw new BuildException("unknown " + p.getName()); - } - } - } - - @Override - public void verifySettings() { - super.verifySettings(); - if (divisor <= 0 || part <= 0) { - throw new BuildException("part or divisor not set"); - } - if (part > divisor) { - throw new BuildException("part must be <= divisor"); - } - } - - @Override - public boolean isSelected(File dir, String name, File path) { - counter = counter % divisor + 1; - return counter == part; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitResultFormatter.java lucene-2621/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitResultFormatter.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitResultFormatter.java 2011-11-01 18:41:12.564614526 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitResultFormatter.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,294 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -package org.apache.lucene.util; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.io.OutputStream; -import java.text.NumberFormat; -import java.util.logging.LogManager; - -import junit.framework.AssertionFailedError; -import junit.framework.Test; - -import org.apache.lucene.store.LockReleaseFailedException; -import org.apache.lucene.store.NativeFSLockFactory; -import org.apache.tools.ant.taskdefs.optional.junit.JUnitResultFormatter; -import org.apache.tools.ant.taskdefs.optional.junit.JUnitTest; -import org.apache.tools.ant.taskdefs.optional.junit.JUnitTestRunner; -import org.apache.tools.ant.util.FileUtils; -import org.apache.tools.ant.util.StringUtils; -import org.junit.Ignore; - -/** - * Just like BriefJUnitResultFormatter "brief" bundled with ant, - * except all formatted text is buffered until the test suite is finished. - * At this point, the output is written at once in synchronized fashion. - * This way tests can run in parallel without interleaving output. - */ -public class LuceneJUnitResultFormatter implements JUnitResultFormatter { - private static final double ONE_SECOND = 1000.0; - - private static final NativeFSLockFactory lockFactory; - - /** Where to write the log to. */ - private OutputStream out; - - /** Formatter for timings. */ - private NumberFormat numberFormat = NumberFormat.getInstance(); - - /** Output suite has written to System.out */ - private String systemOutput = null; - - /** Output suite has written to System.err */ - private String systemError = null; - - /** Buffer output until the end of the test */ - private ByteArrayOutputStream sb; // use a BOS for our mostly ascii-output - - private static final org.apache.lucene.store.Lock lock; - - static { - File lockDir = new File( - System.getProperty("tests.lockdir", System.getProperty("java.io.tmpdir")), - "lucene_junit_lock"); - lockDir.mkdirs(); - if (!lockDir.exists()) { - throw new RuntimeException("Could not make Lock directory:" + lockDir); - } - try { - lockFactory = new NativeFSLockFactory(lockDir); - lock = lockFactory.makeLock("junit_lock"); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** Constructor for LuceneJUnitResultFormatter. */ - public LuceneJUnitResultFormatter() { - } - - /** - * Sets the stream the formatter is supposed to write its results to. - * @param out the output stream to write to - */ - public void setOutput(OutputStream out) { - this.out = out; - } - - /** - * @see JUnitResultFormatter#setSystemOutput(String) - */ - /** {@inheritDoc}. */ - public void setSystemOutput(String out) { - systemOutput = out; - } - - /** - * @see JUnitResultFormatter#setSystemError(String) - */ - /** {@inheritDoc}. */ - public void setSystemError(String err) { - systemError = err; - } - - - /** - * The whole testsuite started. - * @param suite the test suite - */ - public synchronized void startTestSuite(JUnitTest suite) { - if (out == null) { - return; // Quick return - no output do nothing. - } - sb = new ByteArrayOutputStream(); // don't reuse, so its gc'ed - try { - LogManager.getLogManager().readConfiguration(); - } catch (Exception e) {} - append("Testsuite: "); - append(suite.getName()); - append(StringUtils.LINE_SEP); - } - - /** - * The whole testsuite ended. - * @param suite the test suite - */ - public synchronized void endTestSuite(JUnitTest suite) { - append("Tests run: "); - append(suite.runCount()); - append(", Failures: "); - append(suite.failureCount()); - append(", Errors: "); - append(suite.errorCount()); - append(", Time elapsed: "); - append(numberFormat.format(suite.getRunTime() / ONE_SECOND)); - append(" sec"); - append(StringUtils.LINE_SEP); - append(StringUtils.LINE_SEP); - - // append the err and output streams to the log - if (systemOutput != null && systemOutput.length() > 0) { - append("------------- Standard Output ---------------") - .append(StringUtils.LINE_SEP) - .append(systemOutput) - .append("------------- ---------------- ---------------") - .append(StringUtils.LINE_SEP); - } - - // HACK: junit gives us no way to do this in LuceneTestCase - try { - Class clazz = Class.forName(suite.getName()); - Ignore ignore = clazz.getAnnotation(Ignore.class); - if (ignore != null) { - if (systemError == null) systemError = ""; - systemError += "NOTE: Ignoring test class '" + clazz.getSimpleName() + "': " - + ignore.value() + StringUtils.LINE_SEP; - } - } catch (ClassNotFoundException e) { /* no problem */ } - // END HACK - - if (systemError != null && systemError.length() > 0) { - append("------------- Standard Error -----------------") - .append(StringUtils.LINE_SEP) - .append(systemError) - .append("------------- ---------------- ---------------") - .append(StringUtils.LINE_SEP); - } - - if (out != null) { - try { - lock.obtain(5000); - try { - sb.writeTo(out); - out.flush(); - } finally { - try { - lock.release(); - } catch(LockReleaseFailedException e) { - // well lets pretend its released anyway - } - } - } catch (IOException e) { - throw new RuntimeException("unable to write results", e); - } finally { - if (out != System.out && out != System.err) { - FileUtils.close(out); - } - } - } - } - - /** - * A test started. - * @param test a test - */ - public void startTest(Test test) { - } - - /** - * A test ended. - * @param test a test - */ - public void endTest(Test test) { - } - - /** - * Interface TestListener for JUnit <= 3.4. - * - *

    A Test failed. - * @param test a test - * @param t the exception thrown by the test - */ - public void addFailure(Test test, Throwable t) { - formatError("\tFAILED", test, t); - } - - /** - * Interface TestListener for JUnit > 3.4. - * - *

    A Test failed. - * @param test a test - * @param t the assertion failed by the test - */ - public void addFailure(Test test, AssertionFailedError t) { - addFailure(test, (Throwable) t); - } - - /** - * A test caused an error. - * @param test a test - * @param error the error thrown by the test - */ - public void addError(Test test, Throwable error) { - formatError("\tCaused an ERROR", test, error); - } - - /** - * Format the test for printing.. - * @param test a test - * @return the formatted testname - */ - protected String formatTest(Test test) { - if (test == null) { - return "Null Test: "; - } else { - return "Testcase: " + test.toString() + ":"; - } - } - - /** - * Format an error and print it. - * @param type the type of error - * @param test the test that failed - * @param error the exception that the test threw - */ - protected synchronized void formatError(String type, Test test, - Throwable error) { - if (test != null) { - endTest(test); - } - - append(formatTest(test) + type); - append(StringUtils.LINE_SEP); - append(error.getMessage()); - append(StringUtils.LINE_SEP); - String strace = JUnitTestRunner.getFilteredTrace(error); - append(strace); - append(StringUtils.LINE_SEP); - append(StringUtils.LINE_SEP); - } - - public LuceneJUnitResultFormatter append(String s) { - if (s == null) - s = "(null)"; - try { - sb.write(s.getBytes()); // intentionally use default charset, its a console. - } catch (IOException e) { - throw new RuntimeException(e); - } - return this; - } - - public LuceneJUnitResultFormatter append(long l) { - return append(Long.toString(l)); - } -} - diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java lucene-2621/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java 2011-11-01 18:41:12.564614526 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,1461 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.File; -import java.io.IOException; -import java.io.PrintStream; -import java.lang.annotation.Documented; -import java.lang.annotation.ElementType; -import java.lang.annotation.Inherited; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; -import java.lang.reflect.Constructor; -import java.util.*; -import java.util.Map.Entry; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.document.Field; -import org.apache.lucene.document.FieldType; -import org.apache.lucene.index.*; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec; -import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec; -import org.apache.lucene.index.codecs.mocksep.MockSepCodec; -import org.apache.lucene.index.codecs.mockrandom.MockRandomCodec; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; -import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec; -import org.apache.lucene.index.codecs.pulsing.PulsingCodec; -import org.apache.lucene.search.BooleanQuery; -import org.apache.lucene.search.FieldCache; -import org.apache.lucene.search.FieldCache.CacheEntry; -import org.apache.lucene.search.AssertingIndexSearcher; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.RandomSimilarityProvider; -import org.apache.lucene.search.similarities.SimilarityProvider; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.FSDirectory; -import org.apache.lucene.store.FlushInfo; -import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.LockFactory; -import org.apache.lucene.store.MergeInfo; -import org.apache.lucene.store.MockDirectoryWrapper; -import org.apache.lucene.store.MockDirectoryWrapper.Throttling; -import org.apache.lucene.util.FieldCacheSanityChecker.Insanity; -import org.junit.*; -import org.junit.rules.MethodRule; -import org.junit.rules.TestWatchman; -import org.junit.runner.RunWith; -import org.junit.runners.model.FrameworkMethod; -import org.junit.runners.model.Statement; - -/** - * Base class for all Lucene unit tests, Junit3 or Junit4 variant. - *

    - *

    - *

    - * If you - * override either setUp() or - * tearDown() in your unit test, make sure you - * call super.setUp() and - * super.tearDown() - *

    - * - * @After - replaces setup - * @Before - replaces teardown - * @Test - any public method with this annotation is a test case, regardless - * of its name - *

    - *

    - * See Junit4 documentation for a complete list of features. - *

    - * Import from org.junit rather than junit.framework. - *

    - * You should be able to use this class anywhere you used LuceneTestCase - * if you annotate your derived class correctly with the annotations above - * @see #assertSaneFieldCaches(String) - */ - -@RunWith(LuceneTestCaseRunner.class) -public abstract class LuceneTestCase extends Assert { - - /** - * true iff tests are run in verbose mode. Note: if it is false, tests are not - * expected to print any messages. - */ - public static final boolean VERBOSE = Boolean.getBoolean("tests.verbose"); - - /** Use this constant when creating Analyzers and any other version-dependent stuff. - *

    NOTE: Change this when development starts for new Lucene version: - */ - public static final Version TEST_VERSION_CURRENT = Version.LUCENE_40; - - /** - * If this is set, it is the only method that should run. - */ - static final String TEST_METHOD; - - /** Create indexes in this directory, optimally use a subdir, named after the test */ - public static final File TEMP_DIR; - static { - String method = System.getProperty("testmethod", "").trim(); - TEST_METHOD = method.length() == 0 ? null : method; - String s = System.getProperty("tempDir", System.getProperty("java.io.tmpdir")); - if (s == null) - throw new RuntimeException("To run tests, you need to define system property 'tempDir' or 'java.io.tmpdir'."); - TEMP_DIR = new File(s); - TEMP_DIR.mkdirs(); - } - - /** set of directories we created, in afterclass we try to clean these up */ - private static final Map tempDirs = Collections.synchronizedMap(new HashMap()); - - // by default we randomly pick a different codec for - // each test case (non-J4 tests) and each test class (J4 - // tests) - /** Gets the codec to run tests with. */ - public static final String TEST_CODEC = System.getProperty("tests.codec", "randomPerField"); - /** Gets the codecprovider to run tests with */ - public static final String TEST_CODECPROVIDER = System.getProperty("tests.codecprovider", "random"); - /** Gets the locale to run tests with */ - public static final String TEST_LOCALE = System.getProperty("tests.locale", "random"); - /** Gets the timezone to run tests with */ - public static final String TEST_TIMEZONE = System.getProperty("tests.timezone", "random"); - /** Gets the directory to run tests with */ - public static final String TEST_DIRECTORY = System.getProperty("tests.directory", "random"); - /** Get the number of times to run tests */ - public static final int TEST_ITER = Integer.parseInt(System.getProperty("tests.iter", "1")); - /** Get the minimum number of times to run tests until a failure happens */ - public static final int TEST_ITER_MIN = Integer.parseInt(System.getProperty("tests.iter.min", Integer.toString(TEST_ITER))); - /** Get the random seed for tests */ - public static final String TEST_SEED = System.getProperty("tests.seed", "random"); - /** whether or not nightly tests should run */ - public static final boolean TEST_NIGHTLY = Boolean.parseBoolean(System.getProperty("tests.nightly", "false")); - /** the line file used by LineFileDocs */ - public static final String TEST_LINE_DOCS_FILE = System.getProperty("tests.linedocsfile", "europarl.lines.txt.gz"); - /** whether or not to clean threads between test invocations: "false", "perMethod", "perClass" */ - public static final String TEST_CLEAN_THREADS = System.getProperty("tests.cleanthreads", "perClass"); - /** whether or not to clean threads between test invocations: "false", "perMethod", "perClass" */ - public static final Throttling TEST_THROTTLING = TEST_NIGHTLY ? Throttling.SOMETIMES : Throttling.NEVER; - - private static final Pattern codecWithParam = Pattern.compile("(.*)\\(\\s*(\\d+)\\s*\\)"); - - /** - * A random multiplier which you should use when writing random tests: - * multiply it by the number of iterations - */ - public static final int RANDOM_MULTIPLIER = Integer.parseInt(System.getProperty("tests.multiplier", "1")); - - private int savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount(); - - private volatile Thread.UncaughtExceptionHandler savedUncaughtExceptionHandler = null; - - /** Used to track if setUp and tearDown are called correctly from subclasses */ - private static State state = State.INITIAL; - - private static enum State { - INITIAL, // no tests ran yet - SETUP, // test has called setUp() - RANTEST, // test is running - TEARDOWN // test has called tearDown() - } - - /** - * Some tests expect the directory to contain a single segment, and want to do tests on that segment's reader. - * This is an utility method to help them. - */ - public static SegmentReader getOnlySegmentReader(IndexReader reader) { - if (reader instanceof SegmentReader) - return (SegmentReader) reader; - - IndexReader[] subReaders = reader.getSequentialSubReaders(); - if (subReaders.length != 1) - throw new IllegalArgumentException(reader + " has " + subReaders.length + " segments instead of exactly one"); - - return (SegmentReader) subReaders[0]; - } - - private static class UncaughtExceptionEntry { - public final Thread thread; - public final Throwable exception; - - public UncaughtExceptionEntry(Thread thread, Throwable exception) { - this.thread = thread; - this.exception = exception; - } - } - private List uncaughtExceptions = Collections.synchronizedList(new ArrayList()); - - // saves default codec: we do this statically as many build indexes in @beforeClass - private static String savedDefaultCodec; - // default codec: not set when we use a per-field provider. - private static Codec codec; - // default codec provider - private static CodecProvider savedCodecProvider; - - private static SimilarityProvider similarityProvider; - - private static Locale locale; - private static Locale savedLocale; - private static TimeZone timeZone; - private static TimeZone savedTimeZone; - - protected static Map stores; - - private static final String[] TEST_CODECS = new String[] {"MockSep", "MockFixedIntBlock", "MockVariableIntBlock", "MockRandom"}; - - private static void swapCodec(Codec c, CodecProvider cp) { - Codec prior = null; - try { - prior = cp.lookup(c.name); - } catch (IllegalArgumentException iae) { - } - if (prior != null) { - cp.unregister(prior); - } - cp.register(c); - } - - // returns current default codec - static Codec installTestCodecs(String codec, CodecProvider cp) { - savedDefaultCodec = cp.getDefaultFieldCodec(); - - final boolean codecHasParam; - int codecParam = 0; - if (codec.equals("randomPerField")) { - // lie - codec = "Standard"; - codecHasParam = false; - } else if (codec.equals("random")) { - codec = pickRandomCodec(random); - codecHasParam = false; - } else { - Matcher m = codecWithParam.matcher(codec); - if (m.matches()) { - // codec has a fixed param - codecHasParam = true; - codec = m.group(1); - codecParam = Integer.parseInt(m.group(2)); - } else { - codecHasParam = false; - } - } - - cp.setDefaultFieldCodec(codec); - - if (codec.equals("PreFlex")) { - // If we're running w/ PreFlex codec we must swap in the - // test-only PreFlexRW codec (since core PreFlex can - // only read segments): - swapCodec(new PreFlexRWCodec(), cp); - } - - swapCodec(new MockSepCodec(), cp); - // TODO: make it possible to specify min/max iterms per - // block via CL: - int minItemsPerBlock = _TestUtil.nextInt(random, 2, 100); - int maxItemsPerBlock = 2*(Math.max(2, minItemsPerBlock-1)) + random.nextInt(100); - swapCodec(new PulsingCodec(codecHasParam && "Pulsing".equals(codec) ? codecParam : 1 + random.nextInt(20), minItemsPerBlock, maxItemsPerBlock), cp); - swapCodec(new MockFixedIntBlockCodec(codecHasParam && "MockFixedIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 2000)), cp); - // baseBlockSize cannot be over 127: - swapCodec(new MockVariableIntBlockCodec(codecHasParam && "MockVariableIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 127)), cp); - swapCodec(new MockRandomCodec(random), cp); - - return cp.lookup(codec); - } - - // returns current PreFlex codec - static void removeTestCodecs(Codec codec, CodecProvider cp) { - if (codec.name.equals("PreFlex")) { - final Codec preFlex = cp.lookup("PreFlex"); - if (preFlex != null) { - cp.unregister(preFlex); - } - cp.register(new PreFlexCodec()); - } - cp.unregister(cp.lookup("MockSep")); - cp.unregister(cp.lookup("MockFixedIntBlock")); - cp.unregister(cp.lookup("MockVariableIntBlock")); - cp.unregister(cp.lookup("MockRandom")); - swapCodec(new PulsingCodec(), cp); - cp.setDefaultFieldCodec(savedDefaultCodec); - } - - // randomly picks from core and test codecs - static String pickRandomCodec(Random rnd) { - int idx = rnd.nextInt(CodecProvider.CORE_CODECS.length + - TEST_CODECS.length); - if (idx < CodecProvider.CORE_CODECS.length) { - return CodecProvider.CORE_CODECS[idx]; - } else { - return TEST_CODECS[idx - CodecProvider.CORE_CODECS.length]; - } - } - - /** @deprecated (4.0) until we fix no-fork problems in solr tests */ - @Deprecated - static List testClassesRun = new ArrayList(); - - private static void initRandom() { - assert !random.initialized; - staticSeed = "random".equals(TEST_SEED) ? seedRand.nextLong() : ThreeLongs.fromString(TEST_SEED).l1; - random.setSeed(staticSeed); - random.initialized = true; - } - - @Deprecated - private static boolean icuTested = false; - - @BeforeClass - public static void beforeClassLuceneTestCaseJ4() { - initRandom(); - state = State.INITIAL; - tempDirs.clear(); - stores = Collections.synchronizedMap(new IdentityHashMap()); - - // enable this by default, for IDE consistency with ant tests (as its the default from ant) - // TODO: really should be in solr base classes, but some extend LTC directly. - // we do this in beforeClass, because some tests currently disable it - if (System.getProperty("solr.directoryFactory") == null) { - System.setProperty("solr.directoryFactory", "org.apache.solr.core.MockDirectoryFactory"); - } - - savedCodecProvider = CodecProvider.getDefault(); - if ("random".equals(TEST_CODECPROVIDER)) { - if ("randomPerField".equals(TEST_CODEC)) { - if (random.nextInt(4) == 0) { // preflex-only setup - codec = installTestCodecs("PreFlex", CodecProvider.getDefault()); - } else { // per-field setup - CodecProvider.setDefault(new RandomCodecProvider(random, useNoMemoryExpensiveCodec)); - codec = installTestCodecs(TEST_CODEC, CodecProvider.getDefault()); - } - } else { // ordinary setup - codec = installTestCodecs(TEST_CODEC, CodecProvider.getDefault()); - } - } else { - // someone specified their own codecprovider by class - try { - Class cpClazz = Class.forName(TEST_CODECPROVIDER).asSubclass(CodecProvider.class); - CodecProvider cp = cpClazz.newInstance(); - String codecName; - if (TEST_CODEC.startsWith("random")) { // TODO: somehow do random per-field?! - Set codecSet = cp.listAll(); - String availableCodecs[] = codecSet.toArray(new String[codecSet.size()]); - codecName = availableCodecs[random.nextInt(availableCodecs.length)]; - } else { - codecName = TEST_CODEC; - } - - codec = cp.lookup(codecName); - cp.setDefaultFieldCodec(codecName); - CodecProvider.setDefault(cp); - } catch (Exception e) { - System.err.println("Could not instantiate CodecProvider: " + TEST_CODECPROVIDER); - throw new RuntimeException(e); - } - } - - savedLocale = Locale.getDefault(); - - // START hack to init ICU safely before we randomize locales. - // ICU fails during classloading when a special Java7-only locale is the default - // see: http://bugs.icu-project.org/trac/ticket/8734 - if (!icuTested) { - icuTested = true; - try { - Locale.setDefault(Locale.US); - Class.forName("com.ibm.icu.util.ULocale"); - } catch (ClassNotFoundException cnfe) { - // ignore if no ICU is in classpath - } - } - // END hack - - locale = TEST_LOCALE.equals("random") ? randomLocale(random) : localeForName(TEST_LOCALE); - Locale.setDefault(locale); - savedTimeZone = TimeZone.getDefault(); - timeZone = TEST_TIMEZONE.equals("random") ? randomTimeZone(random) : TimeZone.getTimeZone(TEST_TIMEZONE); - TimeZone.setDefault(timeZone); - similarityProvider = new RandomSimilarityProvider(random); - testsFailed = false; - - // verify assertions are enabled (do last, for smooth cleanup) - if (!Boolean.parseBoolean(System.getProperty("tests.asserts.gracious", "false"))) { - assertTrue("assertions are not enabled!", assertionsEnabled()); - } - } - - @AfterClass - public static void afterClassLuceneTestCaseJ4() { - State oldState = state; // capture test execution state - state = State.INITIAL; // set the state for subsequent tests - - Throwable problem = null; - try { - if (!testsFailed) { - assertTrue("ensure your setUp() calls super.setUp() and your tearDown() calls super.tearDown()!!!", - oldState == State.INITIAL || oldState == State.TEARDOWN); - } - } catch (Throwable t) { - if (problem == null) problem = t; - } - - if (! "false".equals(TEST_CLEAN_THREADS)) { - int rogueThreads = threadCleanup("test class"); - if (rogueThreads > 0) { - // TODO: fail here once the leaks are fixed. - System.err.println("RESOURCE LEAK: test class left " + rogueThreads + " thread(s) running"); - } - } - - String codecDescription = uninstallCodecsAfterClass(); - Locale.setDefault(savedLocale); - TimeZone.setDefault(savedTimeZone); - System.clearProperty("solr.solr.home"); - System.clearProperty("solr.data.dir"); - - try { - // now look for unclosed resources - if (!testsFailed) { - checkResourcesAfterClass(); - } - } catch (Throwable t) { - if (problem == null) problem = t; - } - - stores = null; - - try { - // clear out any temp directories if we can - if (!testsFailed) { - clearTempDirectoriesAfterClass(); - } - } catch (Throwable t) { - if (problem == null) problem = t; - } - - // if we had afterClass failures, get some debugging information - if (problem != null) { - reportPartialFailureInfo(); - } - - // if verbose or tests failed, report some information back - if (VERBOSE || testsFailed || problem != null) { - printDebuggingInformation(codecDescription); - } - - // reset seed - random.setSeed(0L); - random.initialized = false; - - if (problem != null) { - throw new RuntimeException(problem); - } - } - - /** print some useful debugging information about the environment */ - private static void printDebuggingInformation(String codecDescription) { - System.err.println("NOTE: test params are: codec=" + codecDescription + - ", sim=" + similarityProvider + - ", locale=" + locale + - ", timezone=" + (timeZone == null ? "(null)" : timeZone.getID())); - System.err.println("NOTE: all tests run in this JVM:"); - System.err.println(Arrays.toString(testClassesRun.toArray())); - System.err.println("NOTE: " + System.getProperty("os.name") + " " - + System.getProperty("os.version") + " " - + System.getProperty("os.arch") + "/" - + System.getProperty("java.vendor") + " " - + System.getProperty("java.version") + " " - + (Constants.JRE_IS_64BIT ? "(64-bit)" : "(32-bit)") + "/" - + "cpus=" + Runtime.getRuntime().availableProcessors() + "," - + "threads=" + Thread.activeCount() + "," - + "free=" + Runtime.getRuntime().freeMemory() + "," - + "total=" + Runtime.getRuntime().totalMemory()); - } - - /** uninstalls test codecs, returns description of the codec used for debugging */ - private static String uninstallCodecsAfterClass() { - String codecDescription; - CodecProvider cp = CodecProvider.getDefault(); - - if ("randomPerField".equals(TEST_CODEC) && cp instanceof RandomCodecProvider) { - codecDescription = cp.toString(); - } else { - codecDescription = codec.toString(); - } - - if ("random".equals(TEST_CODECPROVIDER) && CodecProvider.getDefault() == savedCodecProvider) - removeTestCodecs(codec, CodecProvider.getDefault()); - CodecProvider.setDefault(savedCodecProvider); - - return codecDescription; - } - - /** check that directories and their resources were closed */ - private static void checkResourcesAfterClass() { - for (MockDirectoryWrapper d : stores.keySet()) { - if (d.isOpen()) { - StackTraceElement elements[] = stores.get(d); - // Look for the first class that is not LuceneTestCase that requested - // a Directory. The first two items are of Thread's, so skipping over - // them. - StackTraceElement element = null; - for (int i = 2; i < elements.length; i++) { - StackTraceElement ste = elements[i]; - if (ste.getClassName().indexOf("LuceneTestCase") == -1) { - element = ste; - break; - } - } - fail("directory of test was not closed, opened from: " + element); - } - } - } - - /** clear temp directories: this will fail if its not successful */ - private static void clearTempDirectoriesAfterClass() { - for (Entry entry : tempDirs.entrySet()) { - try { - _TestUtil.rmDir(entry.getKey()); - } catch (IOException e) { - e.printStackTrace(); - System.err.println("path " + entry.getKey() + " allocated from"); - // first two STE's are Java's - StackTraceElement[] elements = entry.getValue(); - for (int i = 2; i < elements.length; i++) { - StackTraceElement ste = elements[i]; - // print only our code's stack information - if (ste.getClassName().indexOf("org.apache.lucene") == -1) break; - System.err.println("\t" + ste); - } - fail("could not remove temp dir: " + entry.getKey()); - } - } - } - - protected static boolean testsFailed; /* true if any tests failed */ - - // This is how we get control when errors occur. - // Think of this as start/end/success/failed - // events. - @Rule - public final TestWatchman intercept = new TestWatchman() { - - @Override - public void failed(Throwable e, FrameworkMethod method) { - // org.junit.internal.AssumptionViolatedException in older releases - // org.junit.Assume.AssumptionViolatedException in recent ones - if (e.getClass().getName().endsWith("AssumptionViolatedException")) { - if (e.getCause() instanceof _TestIgnoredException) - e = e.getCause(); - System.err.print("NOTE: Assume failed in '" + method.getName() + "' (ignored):"); - if (VERBOSE) { - System.err.println(); - e.printStackTrace(System.err); - } else { - System.err.print(" "); - System.err.println(e.getMessage()); - } - } else { - testsFailed = true; - reportAdditionalFailureInfo(); - } - super.failed(e, method); - } - - @Override - public void starting(FrameworkMethod method) { - // set current method name for logging - LuceneTestCase.this.name = method.getName(); - State s = state; // capture test execution state - state = State.RANTEST; // set the state for subsequent tests - if (!testsFailed) { - assertTrue("ensure your setUp() calls super.setUp()!!!", s == State.SETUP); - } - super.starting(method); - } - }; - - /** - * The thread executing the current test case. - * @see #isTestThread() - */ - volatile Thread testCaseThread; - - /** @see #testCaseThread */ - @Rule - public final MethodRule setTestThread = new MethodRule() { - public Statement apply(final Statement s, FrameworkMethod fm, Object target) { - return new Statement() { - public void evaluate() throws Throwable { - try { - LuceneTestCase.this.testCaseThread = Thread.currentThread(); - s.evaluate(); - } finally { - LuceneTestCase.this.testCaseThread = null; - } - } - }; - } - }; - - @Before - public void setUp() throws Exception { - seed = "random".equals(TEST_SEED) ? seedRand.nextLong() : ThreeLongs.fromString(TEST_SEED).l2; - random.setSeed(seed); - State s = state; // capture test execution state - state = State.SETUP; // set the state for subsequent tests - - savedUncaughtExceptionHandler = Thread.getDefaultUncaughtExceptionHandler(); - Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - public void uncaughtException(Thread t, Throwable e) { - testsFailed = true; - uncaughtExceptions.add(new UncaughtExceptionEntry(t, e)); - if (savedUncaughtExceptionHandler != null) - savedUncaughtExceptionHandler.uncaughtException(t, e); - } - }); - - savedBoolMaxClauseCount = BooleanQuery.getMaxClauseCount(); - - if (!testsFailed) { - assertTrue("ensure your tearDown() calls super.tearDown()!!!", (s == State.INITIAL || s == State.TEARDOWN)); - } - - if (useNoMemoryExpensiveCodec) { - final String defCodec = CodecProvider.getDefault().getDefaultFieldCodec(); - // Stupid: assumeFalse in setUp() does not print any information, because - // TestWatchman does not watch test during setUp() - getName() is also not defined... - // => print info directly and use assume without message: - if ("SimpleText".equals(defCodec) || "Memory".equals(defCodec)) { - System.err.println("NOTE: A test method in " + getClass().getSimpleName() + " was ignored, as it uses too much memory with " + defCodec + "."); - Assume.assumeTrue(false); - } - } - } - - /** - * Forcible purges all cache entries from the FieldCache. - *

    - * This method will be called by tearDown to clean up FieldCache.DEFAULT. - * If a (poorly written) test has some expectation that the FieldCache - * will persist across test methods (ie: a static IndexReader) this - * method can be overridden to do nothing. - *

    - * - * @see FieldCache#purgeAllCaches() - */ - protected void purgeFieldCache(final FieldCache fc) { - fc.purgeAllCaches(); - } - - protected String getTestLabel() { - return getClass().getName() + "." + getName(); - } - - /** - * Returns true if and only if the calling thread is the primary thread - * executing the test case. - */ - protected boolean isTestThread() { - assertNotNull("Test case thread not set?", testCaseThread); - return Thread.currentThread() == testCaseThread; - } - - @After - public void tearDown() throws Exception { - State oldState = state; // capture test execution state - state = State.TEARDOWN; // set the state for subsequent tests - - // NOTE: with junit 4.7, we don't get a reproduceWith because our Watchman - // does not know if something fails in tearDown. so we ensure this happens ourselves for now. - // we can remove this if we upgrade to 4.8 - Throwable problem = null; - - try { - if (!testsFailed) { - // Note: we allow a test to go straight from SETUP -> TEARDOWN (without ever entering the RANTEST state) - // because if you assume() inside setUp(), it skips the test and the TestWatchman has no way to know... - assertTrue("ensure your setUp() calls super.setUp()!!!", oldState == State.RANTEST || oldState == State.SETUP); - } - } catch (Throwable t) { - if (problem == null) problem = t; - } - - BooleanQuery.setMaxClauseCount(savedBoolMaxClauseCount); - - // this won't throw any exceptions or fail the test - // if we change this, then change this logic - checkRogueThreadsAfter(); - // restore the default uncaught exception handler - Thread.setDefaultUncaughtExceptionHandler(savedUncaughtExceptionHandler); - - try { - checkUncaughtExceptionsAfter(); - } catch (Throwable t) { - if (problem == null) problem = t; - } - - try { - // calling assertSaneFieldCaches here isn't as useful as having test - // classes call it directly from the scope where the index readers - // are used, because they could be gc'ed just before this tearDown - // method is called. - // - // But it's better then nothing. - // - // If you are testing functionality that you know for a fact - // "violates" FieldCache sanity, then you should either explicitly - // call purgeFieldCache at the end of your test method, or refactor - // your Test class so that the inconsistant FieldCache usages are - // isolated in distinct test methods - assertSaneFieldCaches(getTestLabel()); - } catch (Throwable t) { - if (problem == null) problem = t; - } - - purgeFieldCache(FieldCache.DEFAULT); - - if (problem != null) { - testsFailed = true; - reportAdditionalFailureInfo(); - throw new RuntimeException(problem); - } - } - - /** check if the test still has threads running, we don't want them to - * fail in a subsequent test and pass the blame to the wrong test */ - private void checkRogueThreadsAfter() { - if ("perMethod".equals(TEST_CLEAN_THREADS)) { - int rogueThreads = threadCleanup("test method: '" + getName() + "'"); - if (!testsFailed && rogueThreads > 0) { - System.err.println("RESOURCE LEAK: test method: '" + getName() - + "' left " + rogueThreads + " thread(s) running"); - // TODO: fail, but print seed for now - if (uncaughtExceptions.isEmpty()) { - reportAdditionalFailureInfo(); - } - } - } - } - - /** see if any other threads threw uncaught exceptions, and fail the test if so */ - private void checkUncaughtExceptionsAfter() { - if (!uncaughtExceptions.isEmpty()) { - System.err.println("The following exceptions were thrown by threads:"); - for (UncaughtExceptionEntry entry : uncaughtExceptions) { - System.err.println("*** Thread: " + entry.thread.getName() + " ***"); - entry.exception.printStackTrace(System.err); - } - fail("Some threads threw uncaught exceptions!"); - } - } - - private final static int THREAD_STOP_GRACE_MSEC = 50; - // jvm-wide list of 'rogue threads' we found, so they only get reported once. - private final static IdentityHashMap rogueThreads = new IdentityHashMap(); - - static { - // just a hack for things like eclipse test-runner threads - for (Thread t : Thread.getAllStackTraces().keySet()) { - rogueThreads.put(t, true); - } - - if (TEST_ITER > 1) { - System.out.println("WARNING: you are using -Dtests.iter=n where n > 1, not all tests support this option."); - System.out.println("Some may crash or fail: this is not a bug."); - } - } - - /** - * Looks for leftover running threads, trying to kill them off, - * so they don't fail future tests. - * returns the number of rogue threads that it found. - */ - private static int threadCleanup(String context) { - // educated guess - Thread[] stillRunning = new Thread[Thread.activeCount()+1]; - int threadCount = 0; - int rogueCount = 0; - - if ((threadCount = Thread.enumerate(stillRunning)) > 1) { - while (threadCount == stillRunning.length) { - // truncated response - stillRunning = new Thread[stillRunning.length*2]; - threadCount = Thread.enumerate(stillRunning); - } - - for (int i = 0; i < threadCount; i++) { - Thread t = stillRunning[i]; - - if (t.isAlive() && - !rogueThreads.containsKey(t) && - t != Thread.currentThread() && - /* its ok to keep your searcher across test cases */ - (t.getName().startsWith("LuceneTestCase") && context.startsWith("test method")) == false) { - System.err.println("WARNING: " + context + " left thread running: " + t); - rogueThreads.put(t, true); - rogueCount++; - if (t.getName().startsWith("LuceneTestCase")) { - System.err.println("PLEASE CLOSE YOUR INDEXSEARCHERS IN YOUR TEST!!!!"); - continue; - } else { - // wait on the thread to die of natural causes - try { - t.join(THREAD_STOP_GRACE_MSEC); - } catch (InterruptedException e) { e.printStackTrace(); } - } - // try to stop the thread: - t.setUncaughtExceptionHandler(null); - Thread.setDefaultUncaughtExceptionHandler(null); - if (!t.getName().startsWith("SyncThread")) // avoid zookeeper jre crash - t.interrupt(); - } - } - } - return rogueCount; - } - - /** - * Asserts that FieldCacheSanityChecker does not detect any - * problems with FieldCache.DEFAULT. - *

    - * If any problems are found, they are logged to System.err - * (allong with the msg) when the Assertion is thrown. - *

    - *

    - * This method is called by tearDown after every test method, - * however IndexReaders scoped inside test methods may be garbage - * collected prior to this method being called, causing errors to - * be overlooked. Tests are encouraged to keep their IndexReaders - * scoped at the class level, or to explicitly call this method - * directly in the same scope as the IndexReader. - *

    - * - * @see org.apache.lucene.util.FieldCacheSanityChecker - */ - protected void assertSaneFieldCaches(final String msg) { - final CacheEntry[] entries = FieldCache.DEFAULT.getCacheEntries(); - Insanity[] insanity = null; - try { - try { - insanity = FieldCacheSanityChecker.checkSanity(entries); - } catch (RuntimeException e) { - dumpArray(msg + ": FieldCache", entries, System.err); - throw e; - } - - assertEquals(msg + ": Insane FieldCache usage(s) found", - 0, insanity.length); - insanity = null; - } finally { - - // report this in the event of any exception/failure - // if no failure, then insanity will be null anyway - if (null != insanity) { - dumpArray(msg + ": Insane FieldCache usage(s)", insanity, System.err); - } - - } - } - - /** - * Returns a number of at least i - *

    - * The actual number returned will be influenced by whether {@link #TEST_NIGHTLY} - * is active and {@link #RANDOM_MULTIPLIER}, but also with some random fudge. - */ - public static int atLeast(Random random, int i) { - int min = (TEST_NIGHTLY ? 3*i : i) * RANDOM_MULTIPLIER; - int max = min+(min/2); - return _TestUtil.nextInt(random, min, max); - } - - public static int atLeast(int i) { - return atLeast(random, i); - } - - /** - * Returns true if something should happen rarely, - *

    - * The actual number returned will be influenced by whether {@link #TEST_NIGHTLY} - * is active and {@link #RANDOM_MULTIPLIER}. - */ - public static boolean rarely(Random random) { - int p = TEST_NIGHTLY ? 10 : 5; - p += (p * Math.log(RANDOM_MULTIPLIER)); - int min = 100 - Math.min(p, 50); // never more than 50 - return random.nextInt(100) >= min; - } - - public static boolean rarely() { - return rarely(random); - } - - public static boolean usually(Random random) { - return !rarely(random); - } - - public static boolean usually() { - return usually(random); - } - - public static void assumeTrue(String msg, boolean b) { - Assume.assumeNoException(b ? null : new _TestIgnoredException(msg)); - } - - public static void assumeFalse(String msg, boolean b) { - assumeTrue(msg, !b); - } - - public static void assumeNoException(String msg, Exception e) { - Assume.assumeNoException(e == null ? null : new _TestIgnoredException(msg, e)); - } - - public static Set asSet(T... args) { - return new HashSet(Arrays.asList(args)); - } - - /** - * Convenience method for logging an iterator. - * - * @param label String logged before/after the items in the iterator - * @param iter Each next() is toString()ed and logged on it's own line. If iter is null this is logged differnetly then an empty iterator. - * @param stream Stream to log messages to. - */ - public static void dumpIterator(String label, Iterator iter, - PrintStream stream) { - stream.println("*** BEGIN " + label + " ***"); - if (null == iter) { - stream.println(" ... NULL ..."); - } else { - while (iter.hasNext()) { - stream.println(iter.next().toString()); - } - } - stream.println("*** END " + label + " ***"); - } - - /** - * Convenience method for logging an array. Wraps the array in an iterator and delegates - * - * @see #dumpIterator(String,Iterator,PrintStream) - */ - public static void dumpArray(String label, Object[] objs, - PrintStream stream) { - Iterator iter = (null == objs) ? null : Arrays.asList(objs).iterator(); - dumpIterator(label, iter, stream); - } - - /** create a new index writer config with random defaults */ - public static IndexWriterConfig newIndexWriterConfig(Version v, Analyzer a) { - return newIndexWriterConfig(random, v, a); - } - - /** create a new index writer config with random defaults using the specified random */ - public static IndexWriterConfig newIndexWriterConfig(Random r, Version v, Analyzer a) { - IndexWriterConfig c = new IndexWriterConfig(v, a); - c.setSimilarityProvider(similarityProvider); - if (r.nextBoolean()) { - c.setMergeScheduler(new SerialMergeScheduler()); - } - if (r.nextBoolean()) { - if (rarely(r)) { - // crazy value - c.setMaxBufferedDocs(_TestUtil.nextInt(r, 2, 7)); - } else { - // reasonable value - c.setMaxBufferedDocs(_TestUtil.nextInt(r, 8, 1000)); - } - } - if (r.nextBoolean()) { - if (rarely(r)) { - // crazy value - c.setTermIndexInterval(r.nextBoolean() ? _TestUtil.nextInt(r, 1, 31) : _TestUtil.nextInt(r, 129, 1000)); - } else { - // reasonable value - c.setTermIndexInterval(_TestUtil.nextInt(r, 32, 128)); - } - } - if (r.nextBoolean()) { - c.setIndexerThreadPool(new ThreadAffinityDocumentsWriterThreadPool(_TestUtil.nextInt(r, 1, 20))); - } - - if (r.nextBoolean()) { - c.setMergePolicy(newTieredMergePolicy()); - } else if (r.nextBoolean()) { - c.setMergePolicy(newLogMergePolicy()); - } else { - c.setMergePolicy(new MockRandomMergePolicy(r)); - } - - c.setReaderPooling(r.nextBoolean()); - c.setReaderTermsIndexDivisor(_TestUtil.nextInt(r, 1, 4)); - return c; - } - - public static LogMergePolicy newLogMergePolicy() { - return newLogMergePolicy(random); - } - - public static TieredMergePolicy newTieredMergePolicy() { - return newTieredMergePolicy(random); - } - - public static LogMergePolicy newLogMergePolicy(Random r) { - LogMergePolicy logmp = r.nextBoolean() ? new LogDocMergePolicy() : new LogByteSizeMergePolicy(); - logmp.setUseCompoundFile(r.nextBoolean()); - logmp.setCalibrateSizeByDeletes(r.nextBoolean()); - if (rarely(r)) { - logmp.setMergeFactor(_TestUtil.nextInt(r, 2, 4)); - } else { - logmp.setMergeFactor(_TestUtil.nextInt(r, 5, 50)); - } - return logmp; - } - - public static TieredMergePolicy newTieredMergePolicy(Random r) { - TieredMergePolicy tmp = new TieredMergePolicy(); - if (rarely(r)) { - tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 2, 4)); - tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 2, 4)); - } else { - tmp.setMaxMergeAtOnce(_TestUtil.nextInt(r, 5, 50)); - tmp.setMaxMergeAtOnceExplicit(_TestUtil.nextInt(r, 5, 50)); - } - tmp.setMaxMergedSegmentMB(0.2 + r.nextDouble() * 2.0); - tmp.setFloorSegmentMB(0.2 + r.nextDouble() * 2.0); - tmp.setExpungeDeletesPctAllowed(0.0 + r.nextDouble() * 30.0); - tmp.setSegmentsPerTier(_TestUtil.nextInt(r, 2, 20)); - tmp.setUseCompoundFile(r.nextBoolean()); - tmp.setNoCFSRatio(0.1 + r.nextDouble()*0.8); - tmp.setReclaimDeletesWeight(r.nextDouble()*4); - return tmp; - } - - public static LogMergePolicy newLogMergePolicy(boolean useCFS) { - LogMergePolicy logmp = newLogMergePolicy(); - logmp.setUseCompoundFile(useCFS); - return logmp; - } - - public static LogMergePolicy newLogMergePolicy(boolean useCFS, int mergeFactor) { - LogMergePolicy logmp = newLogMergePolicy(); - logmp.setUseCompoundFile(useCFS); - logmp.setMergeFactor(mergeFactor); - return logmp; - } - - public static LogMergePolicy newLogMergePolicy(int mergeFactor) { - LogMergePolicy logmp = newLogMergePolicy(); - logmp.setMergeFactor(mergeFactor); - return logmp; - } - - /** - * Returns a new Directory instance. Use this when the test does not - * care about the specific Directory implementation (most tests). - *

    - * The Directory is wrapped with {@link MockDirectoryWrapper}. - * By default this means it will be picky, such as ensuring that you - * properly close it and all open files in your test. It will emulate - * some features of Windows, such as not allowing open files to be - * overwritten. - */ - public static MockDirectoryWrapper newDirectory() throws IOException { - return newDirectory(random); - } - - /** - * Returns a new Directory instance, using the specified random. - * See {@link #newDirectory()} for more information. - */ - public static MockDirectoryWrapper newDirectory(Random r) throws IOException { - Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); - MockDirectoryWrapper dir = new MockDirectoryWrapper(r, impl); - stores.put(dir, Thread.currentThread().getStackTrace()); - dir.setThrottling(TEST_THROTTLING); - return dir; - } - - /** - * Returns a new Directory instance, with contents copied from the - * provided directory. See {@link #newDirectory()} for more - * information. - */ - public static MockDirectoryWrapper newDirectory(Directory d) throws IOException { - return newDirectory(random, d); - } - - /** Returns a new FSDirectory instance over the given file, which must be a folder. */ - public static MockDirectoryWrapper newFSDirectory(File f) throws IOException { - return newFSDirectory(f, null); - } - - /** Returns a new FSDirectory instance over the given file, which must be a folder. */ - public static MockDirectoryWrapper newFSDirectory(File f, LockFactory lf) throws IOException { - String fsdirClass = TEST_DIRECTORY; - if (fsdirClass.equals("random")) { - fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)]; - } - - if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store - fsdirClass = "org.apache.lucene.store." + fsdirClass; - } - - Class clazz; - try { - try { - clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class); - } catch (ClassCastException e) { - // TEST_DIRECTORY is not a sub-class of FSDirectory, so draw one at random - fsdirClass = FS_DIRECTORIES[random.nextInt(FS_DIRECTORIES.length)]; - - if (fsdirClass.indexOf(".") == -1) {// if not fully qualified, assume .store - fsdirClass = "org.apache.lucene.store." + fsdirClass; - } - - clazz = Class.forName(fsdirClass).asSubclass(FSDirectory.class); - } - MockDirectoryWrapper dir = new MockDirectoryWrapper(random, newFSDirectoryImpl(clazz, f)); - if (lf != null) { - dir.setLockFactory(lf); - } - stores.put(dir, Thread.currentThread().getStackTrace()); - dir.setThrottling(TEST_THROTTLING); - return dir; - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** - * Returns a new Directory instance, using the specified random - * with contents copied from the provided directory. See - * {@link #newDirectory()} for more information. - */ - public static MockDirectoryWrapper newDirectory(Random r, Directory d) throws IOException { - Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); - for (String file : d.listAll()) { - d.copy(impl, file, file, newIOContext(r)); - } - MockDirectoryWrapper dir = new MockDirectoryWrapper(r, impl); - stores.put(dir, Thread.currentThread().getStackTrace()); - dir.setThrottling(TEST_THROTTLING); - return dir; - } - - public static Field newField(String name, String value, FieldType type) { - return newField(random, name, value, type); - } - - public static Field newField(Random random, String name, String value, FieldType type) { - if (usually(random) || !type.indexed()) { - // most of the time, don't modify the params - return new Field(name, value, type); - } - - FieldType newType = new FieldType(type); - if (!newType.stored() && random.nextBoolean()) { - newType.setStored(true); // randomly store it - } - - if (!newType.storeTermVectors() && random.nextBoolean()) { - newType.setStoreTermVectors(true); - if (!newType.storeTermVectorOffsets()) { - newType.setStoreTermVectorOffsets(random.nextBoolean()); - } - if (!newType.storeTermVectorPositions()) { - newType.setStoreTermVectorPositions(random.nextBoolean()); - } - } - - // TODO: we need to do this, but smarter, ie, most of - // the time we set the same value for a given field but - // sometimes (rarely) we change it up: - /* - if (newType.omitNorms()) { - newType.setOmitNorms(random.nextBoolean()); - } - */ - - return new Field(name, value, newType); - } - - /** return a random Locale from the available locales on the system */ - public static Locale randomLocale(Random random) { - Locale locales[] = Locale.getAvailableLocales(); - return locales[random.nextInt(locales.length)]; - } - - /** return a random TimeZone from the available timezones on the system */ - public static TimeZone randomTimeZone(Random random) { - String tzIds[] = TimeZone.getAvailableIDs(); - return TimeZone.getTimeZone(tzIds[random.nextInt(tzIds.length)]); - } - - /** return a Locale object equivalent to its programmatic name */ - public static Locale localeForName(String localeName) { - String elements[] = localeName.split("\\_"); - switch(elements.length) { - case 3: return new Locale(elements[0], elements[1], elements[2]); - case 2: return new Locale(elements[0], elements[1]); - case 1: return new Locale(elements[0]); - default: throw new IllegalArgumentException("Invalid Locale: " + localeName); - } - } - - private static final String FS_DIRECTORIES[] = { - "SimpleFSDirectory", - "NIOFSDirectory", - "MMapDirectory" - }; - - private static final String CORE_DIRECTORIES[] = { - "RAMDirectory", - FS_DIRECTORIES[0], FS_DIRECTORIES[1], FS_DIRECTORIES[2] - }; - - public static String randomDirectory(Random random) { - if (rarely(random)) { - return CORE_DIRECTORIES[random.nextInt(CORE_DIRECTORIES.length)]; - } else { - return "RAMDirectory"; - } - } - - private static Directory newFSDirectoryImpl( - Class clazz, File file) - throws IOException { - FSDirectory d = null; - try { - // Assuming every FSDirectory has a ctor(File), but not all may take a - // LockFactory too, so setting it afterwards. - Constructor ctor = clazz.getConstructor(File.class); - d = ctor.newInstance(file); - } catch (Exception e) { - d = FSDirectory.open(file); - } - return d; - } - - /** Registers a temp file that will be deleted when tests are done. */ - public static void registerTempFile(File tmpFile) { - tempDirs.put(tmpFile.getAbsoluteFile(), Thread.currentThread().getStackTrace()); - } - - static Directory newDirectoryImpl(Random random, String clazzName) { - if (clazzName.equals("random")) - clazzName = randomDirectory(random); - if (clazzName.indexOf(".") == -1) // if not fully qualified, assume .store - clazzName = "org.apache.lucene.store." + clazzName; - try { - final Class clazz = Class.forName(clazzName).asSubclass(Directory.class); - // If it is a FSDirectory type, try its ctor(File) - if (FSDirectory.class.isAssignableFrom(clazz)) { - final File tmpFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR); - tmpFile.delete(); - tmpFile.mkdir(); - registerTempFile(tmpFile); - return newFSDirectoryImpl(clazz.asSubclass(FSDirectory.class), tmpFile); - } - - // try empty ctor - return clazz.newInstance(); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - /** create a new searcher over the reader. - * This searcher might randomly use threads. */ - public static IndexSearcher newSearcher(IndexReader r) throws IOException { - return newSearcher(r, true); - } - - /** create a new searcher over the reader. - * This searcher might randomly use threads. - * if maybeWrap is true, this searcher might wrap the reader - * with one that returns null for getSequentialSubReaders. - */ - public static IndexSearcher newSearcher(IndexReader r, boolean maybeWrap) throws IOException { - if (random.nextBoolean()) { - if (maybeWrap && rarely()) { - r = new SlowMultiReaderWrapper(r); - } - IndexSearcher ret = random.nextBoolean() ? new AssertingIndexSearcher(random, r) : new AssertingIndexSearcher(random, r.getTopReaderContext()); - ret.setSimilarityProvider(similarityProvider); - return ret; - } else { - int threads = 0; - final ExecutorService ex = (random.nextBoolean()) ? null - : Executors.newFixedThreadPool(threads = _TestUtil.nextInt(random, 1, 8), - new NamedThreadFactory("LuceneTestCase")); - if (ex != null && VERBOSE) { - System.out.println("NOTE: newSearcher using ExecutorService with " + threads + " threads"); - } - IndexSearcher ret = random.nextBoolean() ? - new AssertingIndexSearcher(random, r, ex) { - @Override - public void close() throws IOException { - super.close(); - shutdownExecutorService(ex); - } - } : new AssertingIndexSearcher(random, r.getTopReaderContext(), ex) { - @Override - public void close() throws IOException { - super.close(); - shutdownExecutorService(ex); - } - }; - ret.setSimilarityProvider(similarityProvider); - return ret; - } - } - - static void shutdownExecutorService(ExecutorService ex) { - if (ex != null) { - ex.shutdown(); - try { - ex.awaitTermination(1000, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } - } - } - - public String getName() { - return this.name; - } - - /** Gets a resource from the classpath as {@link File}. This method should only be used, - * if a real file is needed. To get a stream, code should prefer - * {@link Class#getResourceAsStream} using {@code this.getClass()}. - */ - - protected File getDataFile(String name) throws IOException { - try { - return new File(this.getClass().getResource(name).toURI()); - } catch (Exception e) { - throw new IOException("Cannot find resource: " + name); - } - } - - // We get here from InterceptTestCaseEvents on the 'failed' event.... - public static void reportPartialFailureInfo() { - System.err.println("NOTE: reproduce with (hopefully): ant test -Dtestcase=" + testClassesRun.get(testClassesRun.size()-1) - + " -Dtests.seed=" + new ThreeLongs(staticSeed, 0L, LuceneTestCaseRunner.runnerSeed) - + reproduceWithExtraParams()); - } - - // We get here from InterceptTestCaseEvents on the 'failed' event.... - public void reportAdditionalFailureInfo() { - System.err.println("NOTE: reproduce with: ant test -Dtestcase=" + getClass().getSimpleName() - + " -Dtestmethod=" + getName() + " -Dtests.seed=" + new ThreeLongs(staticSeed, seed, LuceneTestCaseRunner.runnerSeed) - + reproduceWithExtraParams()); - } - - // extra params that were overridden needed to reproduce the command - private static String reproduceWithExtraParams() { - StringBuilder sb = new StringBuilder(); - if (!TEST_CODEC.equals("randomPerField")) sb.append(" -Dtests.codec=").append(TEST_CODEC); - if (!TEST_LOCALE.equals("random")) sb.append(" -Dtests.locale=").append(TEST_LOCALE); - if (!TEST_TIMEZONE.equals("random")) sb.append(" -Dtests.timezone=").append(TEST_TIMEZONE); - if (!TEST_DIRECTORY.equals("random")) sb.append(" -Dtests.directory=").append(TEST_DIRECTORY); - if (RANDOM_MULTIPLIER > 1) sb.append(" -Dtests.multiplier=").append(RANDOM_MULTIPLIER); - if (TEST_NIGHTLY) sb.append(" -Dtests.nightly=true"); - return sb.toString(); - } - - public static IOContext newIOContext(Random random) { - final int randomNumDocs = random.nextInt(4192); - final int size = random.nextInt(512) * randomNumDocs; - final IOContext context; - switch (random.nextInt(5)) { - case 0: - context = IOContext.DEFAULT; - break; - case 1: - context = IOContext.READ; - break; - case 2: - context = IOContext.READONCE; - break; - case 3: - context = new IOContext(new MergeInfo(randomNumDocs, size, true, false)); - break; - case 4: - context = new IOContext(new FlushInfo(randomNumDocs, size)); - break; - default: - context = IOContext.DEFAULT; - } - return context; - } - - // initialized by the TestRunner - static boolean useNoMemoryExpensiveCodec; - - // recorded seed: for beforeClass - private static long staticSeed; - // seed for individual test methods, changed in @before - private long seed; - - static final Random seedRand = new Random(); - protected static final SmartRandom random = new SmartRandom(0); - - private String name = ""; - - /** - * Annotation for tests that should only be run during nightly builds. - */ - @Documented - @Inherited - @Retention(RetentionPolicy.RUNTIME) - public @interface Nightly {} - - /** - * Annotation for test classes that should only use codecs that are not memory expensive (avoid SimpleText, MemoryCodec). - */ - @Documented - @Inherited - @Retention(RetentionPolicy.RUNTIME) - @Target(ElementType.TYPE) - public @interface UseNoMemoryExpensiveCodec {} - - @Ignore("just a hack") - public final void alwaysIgnoredTestMethod() {} - - /** check if assertions are enabled */ - private static boolean assertionsEnabled() { - try { - assert Boolean.FALSE.booleanValue(); - return false; // should never get here - } catch (AssertionError e) { - return true; - } - } - -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java lucene-2621/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java 2011-11-01 18:41:12.560614526 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,191 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.List; -import java.util.Random; - -import org.apache.lucene.util.LuceneTestCase.Nightly; -import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; -import org.junit.Ignore; -import org.junit.Test; -import org.junit.runner.Description; -import org.junit.runner.manipulation.Filter; -import org.junit.runner.manipulation.NoTestsRemainException; -import org.junit.runner.notification.Failure; -import org.junit.runner.notification.RunListener; -import org.junit.runner.notification.RunNotifier; -import org.junit.runners.BlockJUnit4ClassRunner; -import org.junit.runners.model.FrameworkMethod; -import org.junit.runners.model.InitializationError; - -// please don't reorganize these into a wildcard! -import static org.apache.lucene.util.LuceneTestCase.TEST_ITER; -import static org.apache.lucene.util.LuceneTestCase.TEST_ITER_MIN; -import static org.apache.lucene.util.LuceneTestCase.TEST_METHOD; -import static org.apache.lucene.util.LuceneTestCase.TEST_SEED; -import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY; -import static org.apache.lucene.util.LuceneTestCase.VERBOSE; - -/** optionally filters the tests to be run by TEST_METHOD */ -public class LuceneTestCaseRunner extends BlockJUnit4ClassRunner { - private List testMethods; - static final long runnerSeed; - static { - runnerSeed = "random".equals(TEST_SEED) ? LuceneTestCase.seedRand.nextLong() : ThreeLongs.fromString(TEST_SEED).l3; - } - - @Override - protected List computeTestMethods() { - if (testMethods != null) - return testMethods; - - Random r = new Random(runnerSeed); - - LuceneTestCase.testClassesRun.add(getTestClass().getJavaClass().getSimpleName()); - testMethods = new ArrayList(); - for (Method m : getTestClass().getJavaClass().getMethods()) { - // check if the current test's class has methods annotated with @Ignore - final Ignore ignored = m.getAnnotation(Ignore.class); - if (ignored != null && !m.getName().equals("alwaysIgnoredTestMethod")) { - System.err.println("NOTE: Ignoring test method '" + m.getName() + "': " + ignored.value()); - } - // add methods starting with "test" - final int mod = m.getModifiers(); - if (m.getAnnotation(Test.class) != null || - (m.getName().startsWith("test") && - !Modifier.isAbstract(mod) && - m.getParameterTypes().length == 0 && - m.getReturnType() == Void.TYPE)) - { - if (Modifier.isStatic(mod)) - throw new RuntimeException("Test methods must not be static."); - testMethods.add(new FrameworkMethod(m)); - } - } - - if (testMethods.isEmpty()) { - throw new RuntimeException("No runnable methods!"); - } - - if (TEST_NIGHTLY == false) { - if (getTestClass().getJavaClass().isAnnotationPresent(Nightly.class)) { - /* the test class is annotated with nightly, remove all methods */ - String className = getTestClass().getJavaClass().getSimpleName(); - System.err.println("NOTE: Ignoring nightly-only test class '" + className + "'"); - testMethods.clear(); - } else { - /* remove all nightly-only methods */ - for (int i = 0; i < testMethods.size(); i++) { - final FrameworkMethod m = testMethods.get(i); - if (m.getAnnotation(Nightly.class) != null) { - System.err.println("NOTE: Ignoring nightly-only test method '" + m.getName() + "'"); - testMethods.remove(i--); - } - } - } - /* dodge a possible "no-runnable methods" exception by adding a fake ignored test */ - if (testMethods.isEmpty()) { - try { - testMethods.add(new FrameworkMethod(LuceneTestCase.class.getMethod("alwaysIgnoredTestMethod"))); - } catch (Exception e) { throw new RuntimeException(e); } - } - } - // sort the test methods first before shuffling them, so that the shuffle is consistent - // across different implementations that might order the methods different originally. - Collections.sort(testMethods, new Comparator() { - @Override - public int compare(FrameworkMethod f1, FrameworkMethod f2) { - return f1.getName().compareTo(f2.getName()); - } - }); - Collections.shuffle(testMethods, r); - return testMethods; - } - - @Override - protected void runChild(FrameworkMethod arg0, RunNotifier arg1) { - if (VERBOSE) { - System.out.println("\nNOTE: running test " + arg0.getName()); - } - - // only print iteration info if the user requested more than one iterations - final boolean verbose = VERBOSE && TEST_ITER > 1; - - final int currentIter[] = new int[1]; - arg1.addListener(new RunListener() { - @Override - public void testFailure(Failure failure) throws Exception { - if (verbose) { - System.out.println("\nNOTE: iteration " + currentIter[0] + " failed! "); - } - } - }); - for (int i = 0; i < TEST_ITER; i++) { - currentIter[0] = i; - if (verbose) { - System.out.println("\nNOTE: running iter=" + (1+i) + " of " + TEST_ITER); - } - super.runChild(arg0, arg1); - if (LuceneTestCase.testsFailed) { - if (i >= TEST_ITER_MIN - 1) { // XXX is this still off-by-one? - break; - } - } - } - } - - public LuceneTestCaseRunner(Class clazz) throws InitializationError { - super(clazz); - - // This TestRunner can handle only LuceneTestCase subclasses - if (!LuceneTestCase.class.isAssignableFrom(clazz)) { - throw new UnsupportedOperationException("LuceneTestCaseRunner can only be used with LuceneTestCase."); - } - - final boolean useNoMemoryExpensiveCodec = LuceneTestCase.useNoMemoryExpensiveCodec = - clazz.isAnnotationPresent(UseNoMemoryExpensiveCodec.class); - if (useNoMemoryExpensiveCodec) { - System.err.println("NOTE: Using no memory expensive codecs (Memory, SimpleText) for " + - clazz.getSimpleName() + "."); - } - - // evil we cannot init our random here, because super() calls computeTestMethods!!!!; - Filter f = new Filter() { - - @Override - public String describe() { return "filters according to TEST_METHOD"; } - - @Override - public boolean shouldRun(Description d) { - return TEST_METHOD == null || d.getMethodName().equals(TEST_METHOD); - } - }; - - try { - f.apply(this); - } catch (NoTestsRemainException e) { - throw new RuntimeException(e); - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/SmartRandom.java lucene-2621/lucene/src/test-framework/org/apache/lucene/util/SmartRandom.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/SmartRandom.java 2011-11-01 18:41:12.564614526 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/util/SmartRandom.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,43 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Random; - -/** - * A random that tracks if its been initialized properly, - * and throws an exception if it hasn't. - */ -public class SmartRandom extends Random { - boolean initialized; - - SmartRandom(long seed) { - super(seed); - } - - @Override - protected int next(int bits) { - if (!initialized) { - System.err.println("!!! WARNING: test is using random from static initializer !!!"); - Thread.dumpStack(); - // I wish, but it causes JRE crashes - // throw new IllegalStateException("you cannot use this random from a static initializer in your test"); - } - return super.next(bits); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/_TestIgnoredException.java lucene-2621/lucene/src/test-framework/org/apache/lucene/util/_TestIgnoredException.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/_TestIgnoredException.java 2011-11-01 18:41:12.560614526 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/util/_TestIgnoredException.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,51 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.PrintStream; - -/** Replacement for Assume jUnit class, so we can add a message with explanation */ -final class _TestIgnoredException extends RuntimeException { - - _TestIgnoredException(String msg) { - super(msg); - } - - _TestIgnoredException(String msg, Throwable t) { - super(msg, t); - } - - @Override - public String getMessage() { - StringBuilder sb = new StringBuilder(super.getMessage()); - if (getCause() != null) - sb.append(" - ").append(getCause()); - return sb.toString(); - } - - // only this one is called by our code, exception is not used outside this class: - @Override - public void printStackTrace(PrintStream s) { - if (getCause() != null) { - s.println(super.toString() + " - Caused by:"); - getCause().printStackTrace(s); - } else { - super.printStackTrace(s); - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java lucene-2621/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java 2011-11-01 18:41:12.564614526 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,522 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.io.BufferedOutputStream; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PrintStream; -import java.lang.reflect.Method; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Map; -import java.util.Random; -import java.util.zip.ZipEntry; -import java.util.zip.ZipFile; - -import org.apache.lucene.document.Document; -import org.apache.lucene.document.Field; -import org.apache.lucene.index.CheckIndex; -import org.apache.lucene.index.ConcurrentMergeScheduler; -import org.apache.lucene.index.FieldInfos; -import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexableField; -import org.apache.lucene.index.LogMergePolicy; -import org.apache.lucene.index.MergePolicy; -import org.apache.lucene.index.MergeScheduler; -import org.apache.lucene.index.TieredMergePolicy; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.search.FieldDoc; -import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.TopDocs; -import org.apache.lucene.store.Directory; -import org.junit.Assert; - -public class _TestUtil { - - /** Returns temp dir, based on String arg in its name; - * does not create the directory. */ - public static File getTempDir(String desc) { - try { - File f = createTempFile(desc, "tmp", LuceneTestCase.TEMP_DIR); - f.delete(); - LuceneTestCase.registerTempFile(f); - return f; - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - /** - * Deletes a directory and everything underneath it. - */ - public static void rmDir(File dir) throws IOException { - if (dir.exists()) { - for (File f : dir.listFiles()) { - if (f.isDirectory()) { - rmDir(f); - } else { - if (!f.delete()) { - throw new IOException("could not delete " + f); - } - } - } - if (!dir.delete()) { - throw new IOException("could not delete " + dir); - } - } - } - - /** - * Convenience method: Unzip zipName + ".zip" under destDir, removing destDir first - */ - public static void unzip(File zipName, File destDir) throws IOException { - - ZipFile zipFile = new ZipFile(zipName); - - Enumeration entries = zipFile.entries(); - - rmDir(destDir); - - destDir.mkdir(); - LuceneTestCase.registerTempFile(destDir); - - while (entries.hasMoreElements()) { - ZipEntry entry = entries.nextElement(); - - InputStream in = zipFile.getInputStream(entry); - File targetFile = new File(destDir, entry.getName()); - if (entry.isDirectory()) { - // allow unzipping with directory structure - targetFile.mkdirs(); - } else { - if (targetFile.getParentFile()!=null) { - // be on the safe side: do not rely on that directories are always extracted - // before their children (although this makes sense, but is it guaranteed?) - targetFile.getParentFile().mkdirs(); - } - OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile)); - - byte[] buffer = new byte[8192]; - int len; - while((len = in.read(buffer)) >= 0) { - out.write(buffer, 0, len); - } - - in.close(); - out.close(); - } - } - - zipFile.close(); - } - - public static void syncConcurrentMerges(IndexWriter writer) { - syncConcurrentMerges(writer.getConfig().getMergeScheduler()); - } - - public static void syncConcurrentMerges(MergeScheduler ms) { - if (ms instanceof ConcurrentMergeScheduler) - ((ConcurrentMergeScheduler) ms).sync(); - } - - /** This runs the CheckIndex tool on the index in. If any - * issues are hit, a RuntimeException is thrown; else, - * true is returned. */ - public static CheckIndex.Status checkIndex(Directory dir) throws IOException { - return checkIndex(dir, CodecProvider.getDefault()); - } - - /** This runs the CheckIndex tool on the index in. If any - * issues are hit, a RuntimeException is thrown; else, - * true is returned. */ - public static CheckIndex.Status checkIndex(Directory dir, CodecProvider codecs) throws IOException { - ByteArrayOutputStream bos = new ByteArrayOutputStream(1024); - CheckIndex checker = new CheckIndex(dir); - checker.setInfoStream(new PrintStream(bos), false); - CheckIndex.Status indexStatus = checker.checkIndex(null, codecs); - if (indexStatus == null || indexStatus.clean == false) { - System.out.println("CheckIndex failed"); - System.out.println(bos.toString()); - throw new RuntimeException("CheckIndex failed"); - } else { - return indexStatus; - } - } - - // NOTE: only works for TMP and LMP!! - public static void setUseCompoundFile(MergePolicy mp, boolean v) { - if (mp instanceof TieredMergePolicy) { - ((TieredMergePolicy) mp).setUseCompoundFile(v); - } else if (mp instanceof LogMergePolicy) { - ((LogMergePolicy) mp).setUseCompoundFile(v); - } - } - - /** start and end are BOTH inclusive */ - public static int nextInt(Random r, int start, int end) { - return start + r.nextInt(end-start+1); - } - - public static String randomSimpleString(Random r) { - final int end = r.nextInt(10); - if (end == 0) { - // allow 0 length - return ""; - } - final char[] buffer = new char[end]; - for (int i = 0; i < end; i++) { - buffer[i] = (char) _TestUtil.nextInt(r, 97, 102); - } - return new String(buffer, 0, end); - } - - /** Returns random string, including full unicode range. */ - public static String randomUnicodeString(Random r) { - return randomUnicodeString(r, 20); - } - - /** - * Returns a random string up to a certain length. - */ - public static String randomUnicodeString(Random r, int maxLength) { - final int end = r.nextInt(maxLength); - if (end == 0) { - // allow 0 length - return ""; - } - final char[] buffer = new char[end]; - randomFixedLengthUnicodeString(r, buffer, 0, buffer.length); - return new String(buffer, 0, end); - } - - /** - * Fills provided char[] with valid random unicode code - * unit sequence. - */ - public static void randomFixedLengthUnicodeString(Random random, char[] chars, int offset, int length) { - int i = offset; - final int end = offset + length; - while(i < end) { - final int t = random.nextInt(5); - if (0 == t && i < length - 1) { - // Make a surrogate pair - // High surrogate - chars[i++] = (char) nextInt(random, 0xd800, 0xdbff); - // Low surrogate - chars[i++] = (char) nextInt(random, 0xdc00, 0xdfff); - } else if (t <= 1) { - chars[i++] = (char) random.nextInt(0x80); - } else if (2 == t) { - chars[i++] = (char) nextInt(random, 0x80, 0x7ff); - } else if (3 == t) { - chars[i++] = (char) nextInt(random, 0x800, 0xd7ff); - } else if (4 == t) { - chars[i++] = (char) nextInt(random, 0xe000, 0xffff); - } - } - } - - private static final int[] blockStarts = { - 0x0000, 0x0080, 0x0100, 0x0180, 0x0250, 0x02B0, 0x0300, 0x0370, 0x0400, - 0x0500, 0x0530, 0x0590, 0x0600, 0x0700, 0x0750, 0x0780, 0x07C0, 0x0800, - 0x0900, 0x0980, 0x0A00, 0x0A80, 0x0B00, 0x0B80, 0x0C00, 0x0C80, 0x0D00, - 0x0D80, 0x0E00, 0x0E80, 0x0F00, 0x1000, 0x10A0, 0x1100, 0x1200, 0x1380, - 0x13A0, 0x1400, 0x1680, 0x16A0, 0x1700, 0x1720, 0x1740, 0x1760, 0x1780, - 0x1800, 0x18B0, 0x1900, 0x1950, 0x1980, 0x19E0, 0x1A00, 0x1A20, 0x1B00, - 0x1B80, 0x1C00, 0x1C50, 0x1CD0, 0x1D00, 0x1D80, 0x1DC0, 0x1E00, 0x1F00, - 0x2000, 0x2070, 0x20A0, 0x20D0, 0x2100, 0x2150, 0x2190, 0x2200, 0x2300, - 0x2400, 0x2440, 0x2460, 0x2500, 0x2580, 0x25A0, 0x2600, 0x2700, 0x27C0, - 0x27F0, 0x2800, 0x2900, 0x2980, 0x2A00, 0x2B00, 0x2C00, 0x2C60, 0x2C80, - 0x2D00, 0x2D30, 0x2D80, 0x2DE0, 0x2E00, 0x2E80, 0x2F00, 0x2FF0, 0x3000, - 0x3040, 0x30A0, 0x3100, 0x3130, 0x3190, 0x31A0, 0x31C0, 0x31F0, 0x3200, - 0x3300, 0x3400, 0x4DC0, 0x4E00, 0xA000, 0xA490, 0xA4D0, 0xA500, 0xA640, - 0xA6A0, 0xA700, 0xA720, 0xA800, 0xA830, 0xA840, 0xA880, 0xA8E0, 0xA900, - 0xA930, 0xA960, 0xA980, 0xAA00, 0xAA60, 0xAA80, 0xABC0, 0xAC00, 0xD7B0, - 0xE000, 0xF900, 0xFB00, 0xFB50, 0xFE00, 0xFE10, - 0xFE20, 0xFE30, 0xFE50, 0xFE70, 0xFF00, 0xFFF0, - 0x10000, 0x10080, 0x10100, 0x10140, 0x10190, 0x101D0, 0x10280, 0x102A0, - 0x10300, 0x10330, 0x10380, 0x103A0, 0x10400, 0x10450, 0x10480, 0x10800, - 0x10840, 0x10900, 0x10920, 0x10A00, 0x10A60, 0x10B00, 0x10B40, 0x10B60, - 0x10C00, 0x10E60, 0x11080, 0x12000, 0x12400, 0x13000, 0x1D000, 0x1D100, - 0x1D200, 0x1D300, 0x1D360, 0x1D400, 0x1F000, 0x1F030, 0x1F100, 0x1F200, - 0x20000, 0x2A700, 0x2F800, 0xE0000, 0xE0100, 0xF0000, 0x100000 - }; - - private static final int[] blockEnds = { - 0x007F, 0x00FF, 0x017F, 0x024F, 0x02AF, 0x02FF, 0x036F, 0x03FF, 0x04FF, - 0x052F, 0x058F, 0x05FF, 0x06FF, 0x074F, 0x077F, 0x07BF, 0x07FF, 0x083F, - 0x097F, 0x09FF, 0x0A7F, 0x0AFF, 0x0B7F, 0x0BFF, 0x0C7F, 0x0CFF, 0x0D7F, - 0x0DFF, 0x0E7F, 0x0EFF, 0x0FFF, 0x109F, 0x10FF, 0x11FF, 0x137F, 0x139F, - 0x13FF, 0x167F, 0x169F, 0x16FF, 0x171F, 0x173F, 0x175F, 0x177F, 0x17FF, - 0x18AF, 0x18FF, 0x194F, 0x197F, 0x19DF, 0x19FF, 0x1A1F, 0x1AAF, 0x1B7F, - 0x1BBF, 0x1C4F, 0x1C7F, 0x1CFF, 0x1D7F, 0x1DBF, 0x1DFF, 0x1EFF, 0x1FFF, - 0x206F, 0x209F, 0x20CF, 0x20FF, 0x214F, 0x218F, 0x21FF, 0x22FF, 0x23FF, - 0x243F, 0x245F, 0x24FF, 0x257F, 0x259F, 0x25FF, 0x26FF, 0x27BF, 0x27EF, - 0x27FF, 0x28FF, 0x297F, 0x29FF, 0x2AFF, 0x2BFF, 0x2C5F, 0x2C7F, 0x2CFF, - 0x2D2F, 0x2D7F, 0x2DDF, 0x2DFF, 0x2E7F, 0x2EFF, 0x2FDF, 0x2FFF, 0x303F, - 0x309F, 0x30FF, 0x312F, 0x318F, 0x319F, 0x31BF, 0x31EF, 0x31FF, 0x32FF, - 0x33FF, 0x4DBF, 0x4DFF, 0x9FFF, 0xA48F, 0xA4CF, 0xA4FF, 0xA63F, 0xA69F, - 0xA6FF, 0xA71F, 0xA7FF, 0xA82F, 0xA83F, 0xA87F, 0xA8DF, 0xA8FF, 0xA92F, - 0xA95F, 0xA97F, 0xA9DF, 0xAA5F, 0xAA7F, 0xAADF, 0xABFF, 0xD7AF, 0xD7FF, - 0xF8FF, 0xFAFF, 0xFB4F, 0xFDFF, 0xFE0F, 0xFE1F, - 0xFE2F, 0xFE4F, 0xFE6F, 0xFEFF, 0xFFEF, 0xFFFF, - 0x1007F, 0x100FF, 0x1013F, 0x1018F, 0x101CF, 0x101FF, 0x1029F, 0x102DF, - 0x1032F, 0x1034F, 0x1039F, 0x103DF, 0x1044F, 0x1047F, 0x104AF, 0x1083F, - 0x1085F, 0x1091F, 0x1093F, 0x10A5F, 0x10A7F, 0x10B3F, 0x10B5F, 0x10B7F, - 0x10C4F, 0x10E7F, 0x110CF, 0x123FF, 0x1247F, 0x1342F, 0x1D0FF, 0x1D1FF, - 0x1D24F, 0x1D35F, 0x1D37F, 0x1D7FF, 0x1F02F, 0x1F09F, 0x1F1FF, 0x1F2FF, - 0x2A6DF, 0x2B73F, 0x2FA1F, 0xE007F, 0xE01EF, 0xFFFFF, 0x10FFFF - }; - - /** Returns random string of length between 0-20 codepoints, all codepoints within the same unicode block. */ - public static String randomRealisticUnicodeString(Random r) { - return randomRealisticUnicodeString(r, 20); - } - - /** Returns random string of length up to maxLength codepoints , all codepoints within the same unicode block. */ - public static String randomRealisticUnicodeString(Random r, int maxLength) { - return randomRealisticUnicodeString(r, 0, 20); - } - - /** Returns random string of length between min and max codepoints, all codepoints within the same unicode block. */ - public static String randomRealisticUnicodeString(Random r, int minLength, int maxLength) { - final int end = minLength + r.nextInt(maxLength); - final int block = r.nextInt(blockStarts.length); - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < end; i++) - sb.appendCodePoint(nextInt(r, blockStarts[block], blockEnds[block])); - return sb.toString(); - } - - /** Returns random string, with a given UTF-8 byte length*/ - public static String randomFixedByteLengthUnicodeString(Random r, int length) { - - final char[] buffer = new char[length*3]; - int bytes = length; - int i = 0; - for (; i < buffer.length && bytes != 0; i++) { - int t; - if (bytes >= 4) { - t = r.nextInt(5); - } else if (bytes >= 3) { - t = r.nextInt(4); - } else if (bytes >= 2) { - t = r.nextInt(2); - } else { - t = 0; - } - if (t == 0) { - buffer[i] = (char) r.nextInt(0x80); - bytes--; - } else if (1 == t) { - buffer[i] = (char) nextInt(r, 0x80, 0x7ff); - bytes -= 2; - } else if (2 == t) { - buffer[i] = (char) nextInt(r, 0x800, 0xd7ff); - bytes -= 3; - } else if (3 == t) { - buffer[i] = (char) nextInt(r, 0xe000, 0xffff); - bytes -= 3; - } else if (4 == t) { - // Make a surrogate pair - // High surrogate - buffer[i++] = (char) nextInt(r, 0xd800, 0xdbff); - // Low surrogate - buffer[i] = (char) nextInt(r, 0xdc00, 0xdfff); - bytes -= 4; - } - - } - return new String(buffer, 0, i); - } - - public static CodecProvider alwaysCodec(final Codec c) { - CodecProvider p = new CodecProvider() { - @Override - public Codec lookup(String name) { - // can't do this until we fix PreFlexRW to not - //impersonate PreFlex: - if (name.equals(c.name)) { - return c; - } else { - return CodecProvider.getDefault().lookup(name); - } - } - }; - p.setDefaultFieldCodec(c.name); - return p; - } - - /** Return a CodecProvider that can read any of the - * default codecs, but always writes in the specified - * codec. */ - public static CodecProvider alwaysCodec(final String codec) { - return alwaysCodec(CodecProvider.getDefault().lookup(codec)); - } - - public static boolean anyFilesExceptWriteLock(Directory dir) throws IOException { - String[] files = dir.listAll(); - if (files.length > 1 || (files.length == 1 && !files[0].equals("write.lock"))) { - return true; - } else { - return false; - } - } - - /** just tries to configure things to keep the open file - * count lowish */ - public static void reduceOpenFiles(IndexWriter w) { - // keep number of open files lowish - MergePolicy mp = w.getConfig().getMergePolicy(); - if (mp instanceof LogMergePolicy) { - LogMergePolicy lmp = (LogMergePolicy) mp; - lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor())); - } else if (mp instanceof TieredMergePolicy) { - TieredMergePolicy tmp = (TieredMergePolicy) mp; - tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce())); - tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier())); - } - MergeScheduler ms = w.getConfig().getMergeScheduler(); - if (ms instanceof ConcurrentMergeScheduler) { - ((ConcurrentMergeScheduler) ms).setMaxThreadCount(2); - ((ConcurrentMergeScheduler) ms).setMaxMergeCount(3); - } - } - - /** Checks some basic behaviour of an AttributeImpl - * @param reflectedValues contains a map with "AttributeClass#key" as values - */ - public static void assertAttributeReflection(final AttributeImpl att, Map reflectedValues) { - final Map map = new HashMap(); - att.reflectWith(new AttributeReflector() { - public void reflect(Class attClass, String key, Object value) { - map.put(attClass.getName() + '#' + key, value); - } - }); - Assert.assertEquals("Reflection does not produce same map", reflectedValues, map); - } - - public static void keepFullyDeletedSegments(IndexWriter w) { - try { - // Carefully invoke what is a package-private (test - // only, internal) method on IndexWriter: - Method m = IndexWriter.class.getDeclaredMethod("keepFullyDeletedSegments"); - m.setAccessible(true); - m.invoke(w); - } catch (Exception e) { - // Should not happen? - throw new RuntimeException(e); - } - } - - /** Adds field info for a Document. */ - public static void add(Document doc, FieldInfos fieldInfos) { - for (IndexableField field : doc) { - fieldInfos.addOrUpdate(field.name(), field.fieldType(), false, field.docValuesType()); - } - } - - /** - * insecure, fast version of File.createTempFile - * uses Random instead of SecureRandom. - */ - public static File createTempFile(String prefix, String suffix, File directory) - throws IOException { - // Force a prefix null check first - if (prefix.length() < 3) { - throw new IllegalArgumentException("prefix must be 3"); - } - String newSuffix = suffix == null ? ".tmp" : suffix; - File result; - do { - result = genTempFile(prefix, newSuffix, directory); - } while (!result.createNewFile()); - return result; - } - - /* Temp file counter */ - private static int counter = 0; - - /* identify for differnt VM processes */ - private static int counterBase = 0; - - private static class TempFileLocker {}; - private static TempFileLocker tempFileLocker = new TempFileLocker(); - - private static File genTempFile(String prefix, String suffix, File directory) { - int identify = 0; - - synchronized (tempFileLocker) { - if (counter == 0) { - int newInt = new Random().nextInt(); - counter = ((newInt / 65535) & 0xFFFF) + 0x2710; - counterBase = counter; - } - identify = counter++; - } - - StringBuilder newName = new StringBuilder(); - newName.append(prefix); - newName.append(counterBase); - newName.append(identify); - newName.append(suffix); - return new File(directory, newName.toString()); - } - - public static void assertEquals(TopDocs expected, TopDocs actual) { - Assert.assertEquals("wrong total hits", expected.totalHits, actual.totalHits); - Assert.assertEquals("wrong maxScore", expected.getMaxScore(), actual.getMaxScore(), 0.0); - Assert.assertEquals("wrong hit count", expected.scoreDocs.length, actual.scoreDocs.length); - for(int hitIDX=0;hitIDX - *

  • The test class's random seed: this is what the test sees in its beforeClass methods - *
  • The test method's random seed: this is what the test method sees starting in its befores - *
  • The test runner's random seed (controls the shuffling of test methods) - * - */ -class ThreeLongs { - public final long l1, l2, l3; - - public ThreeLongs(long l1, long l2, long l3) { - this.l1 = l1; - this.l2 = l2; - this.l3 = l3; - } - - @Override - public String toString() { - return Long.toString(l1, 16) + ":" + Long.toString(l2, 16) + ":" + Long.toString(l3, 16); - } - - public static ThreeLongs fromString(String s) { - String parts[] = s.split(":"); - assert parts.length == 3; - return new ThreeLongs(Long.parseLong(parts[0], 16), Long.parseLong(parts[1], 16), Long.parseLong(parts[2], 16)); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java lucene-2621/lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java --- lucene-clean-trunk/lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java 2011-11-01 18:41:12.560614526 -0400 +++ lucene-2621/lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,149 +0,0 @@ -package org.apache.lucene.util; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.io.IOException; - -import org.apache.lucene.store.DataInput; -import org.apache.lucene.store.IndexOutput; - -public class ThrottledIndexOutput extends IndexOutput { - public static final int DEFAULT_MIN_WRITTEN_BYTES = 1024; - private final int bytesPerSecond; - private IndexOutput delegate; - private long flushDelayMillis; - private long closeDelayMillis; - private long seekDelayMillis; - private long pendingBytes; - private long minBytesWritten; - private long timeElapsed; - private final byte[] bytes = new byte[1]; - - public ThrottledIndexOutput newFromDelegate(IndexOutput output) { - return new ThrottledIndexOutput(bytesPerSecond, flushDelayMillis, - closeDelayMillis, seekDelayMillis, minBytesWritten, output); - } - - public ThrottledIndexOutput(int bytesPerSecond, long delayInMillis, - IndexOutput delegate) { - this(bytesPerSecond, delayInMillis, delayInMillis, delayInMillis, - DEFAULT_MIN_WRITTEN_BYTES, delegate); - } - - public ThrottledIndexOutput(int bytesPerSecond, long delays, - int minBytesWritten, IndexOutput delegate) { - this(bytesPerSecond, delays, delays, delays, minBytesWritten, delegate); - } - - public static final int mBitsToBytes(int mbits) { - return mbits * 125000; - } - - public ThrottledIndexOutput(int bytesPerSecond, long flushDelayMillis, - long closeDelayMillis, long seekDelayMillis, long minBytesWritten, - IndexOutput delegate) { - assert bytesPerSecond > 0; - this.delegate = delegate; - this.bytesPerSecond = bytesPerSecond; - this.flushDelayMillis = flushDelayMillis; - this.closeDelayMillis = closeDelayMillis; - this.seekDelayMillis = seekDelayMillis; - this.minBytesWritten = minBytesWritten; - } - - @Override - public void flush() throws IOException { - sleep(flushDelayMillis); - delegate.flush(); - } - - @Override - public void close() throws IOException { - try { - sleep(closeDelayMillis + getDelay(true)); - } finally { - delegate.close(); - } - } - - @Override - public long getFilePointer() { - return delegate.getFilePointer(); - } - - @Override - public void seek(long pos) throws IOException { - sleep(seekDelayMillis); - delegate.seek(pos); - } - - @Override - public long length() throws IOException { - return delegate.length(); - } - - @Override - public void writeByte(byte b) throws IOException { - bytes[0] = b; - writeBytes(bytes, 0, 1); - } - - @Override - public void writeBytes(byte[] b, int offset, int length) throws IOException { - final long before = System.nanoTime(); - delegate.writeBytes(b, offset, length); - timeElapsed += System.nanoTime() - before; - pendingBytes += length; - sleep(getDelay(false)); - - } - - protected long getDelay(boolean closing) { - if (pendingBytes > 0 && (closing || pendingBytes > minBytesWritten)) { - long actualBps = (timeElapsed / pendingBytes) * 1000000000l; // nano to sec - if (actualBps > bytesPerSecond) { - long expected = (pendingBytes * 1000l / bytesPerSecond) ; - final long delay = expected - (timeElapsed / 1000000l) ; - pendingBytes = 0; - timeElapsed = 0; - return delay; - } - } - return 0; - - } - - private static final void sleep(long ms) { - if (ms <= 0) - return; - try { - Thread.sleep(ms); - } catch (InterruptedException e) { - throw new ThreadInterruptedException(e); - } - } - - @Override - public void setLength(long length) throws IOException { - delegate.setLength(length); - } - - @Override - public void copyBytes(DataInput input, long numBytes) throws IOException { - delegate.copyBytes(input, numBytes); - } -} diff -ruN -x .svn -x build lucene-clean-trunk/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java lucene-2621/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java --- lucene-clean-trunk/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java 2011-08-17 23:39:02.011460945 -0400 +++ lucene-2621/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java 2011-11-01 21:10:40.464609903 -0400 @@ -20,7 +20,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CollationTestBase; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.util.BytesRef; import java.text.Collator; @@ -47,7 +47,7 @@ @Override public void setUp() throws Exception { super.setUp(); - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); } public void testFarsiRangeFilterCollating() throws Exception { diff -ruN -x .svn -x build lucene-clean-trunk/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java lucene-2621/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java --- lucene-clean-trunk/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java 2011-08-17 23:39:01.981460945 -0400 +++ lucene-2621/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java 2011-11-01 21:10:40.456609904 -0400 @@ -22,7 +22,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CollationTestBase; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.util.BytesRef; import java.util.Locale; @@ -45,7 +45,7 @@ @Override public void setUp() throws Exception { super.setUp(); - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); } public void testFarsiRangeFilterCollating() throws Exception { diff -ruN -x .svn -x build lucene-clean-trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java lucene-2621/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java --- lucene-clean-trunk/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java 2011-05-08 10:52:11.550078711 -0400 +++ lucene-2621/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java 2011-11-02 21:30:07.520564791 -0400 @@ -33,7 +33,7 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.NoMergeScheduler; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.Version; @@ -133,7 +133,12 @@ final String defaultCodec = config.get("default.codec", null); if (defaultCodec != null) { - CodecProvider.getDefault().setDefaultFieldCodec(defaultCodec); + try { + Class clazz = Class.forName(defaultCodec).asSubclass(Codec.class); + Codec.setDefault(clazz.newInstance()); + } catch (Exception e) { + throw new RuntimeException("Couldn't instantiate Codec: " + defaultCodec, e); + } } final String mergePolicy = config.get("merge.policy", diff -ruN -x .svn -x build lucene-clean-trunk/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java lucene-2621/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java --- lucene-clean-trunk/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java 2011-11-03 14:37:00.628533051 -0400 +++ lucene-2621/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java 2011-11-02 00:19:12.916604078 -0400 @@ -486,11 +486,17 @@ * @throws IOException */ public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB, boolean optimize) throws IOException { + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null) + .setRAMBufferSizeMB(ramMB); + ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(mergeFactor); + indexDictionary(dict, config, optimize); + } + + public final void indexDictionary(Dictionary dict, IndexWriterConfig config, boolean optimize) throws IOException { synchronized (modifyCurrentIndexLock) { ensureOpen(); final Directory dir = this.spellIndex; - final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, null).setRAMBufferSizeMB(ramMB)); - ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setMaxMergeAtOnce(mergeFactor); + final IndexWriter writer = new IndexWriter(dir, config); IndexSearcher indexSearcher = obtainSearcher(); final List termsEnums = new ArrayList(); diff -ruN -x .svn -x build lucene-clean-trunk/solr/CHANGES.txt lucene-2621/solr/CHANGES.txt --- lucene-clean-trunk/solr/CHANGES.txt 2011-11-03 13:25:07.100535273 -0400 +++ lucene-2621/solr/CHANGES.txt 2011-11-03 13:44:31.468534674 -0400 @@ -19,7 +19,7 @@ See the tutorial at http://lucene.apache.org/solr/tutorial.html -$Id: CHANGES.txt 1197154 2011-11-03 14:38:13Z mvg $ +$Id: CHANGES.txt 1197240 2011-11-03 17:44:17Z rmuir $ ================== 4.0.0-dev ================== Versions of Major Components @@ -133,10 +133,8 @@ fq={!join from=name to=parent}eyes:blue (yonik) -* SOLR-1942: Added the ability to select codec per fieldType in schema.xml - as well as support custom CodecProviders in solrconfig.xml. - NOTE: IndexReaderFactory now has a codecProvider that should be passed - to IndexReader.open (in the case you have a custom IndexReaderFactory). +* SOLR-1942: Added the ability to select postings format per fieldType in schema.xml + as well as support custom Codecs in solrconfig.xml. (simonw via rmuir) * SOLR-2136: Boolean type added to function queries, along with diff -ruN -x .svn -x build lucene-clean-trunk/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java lucene-2621/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java --- lucene-clean-trunk/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java 2011-07-24 19:00:21.602328885 -0400 +++ lucene-2621/solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java 2011-11-01 21:10:40.424609906 -0400 @@ -22,7 +22,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; @@ -37,7 +37,7 @@ @BeforeClass public static void beforeClass() throws Exception { - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); String home = setupSolrHome(); initCore("solrconfig.xml","schema.xml", home); // add some docs diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/CodecFactory.java lucene-2621/solr/core/src/java/org/apache/solr/core/CodecFactory.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/CodecFactory.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/solr/core/src/java/org/apache/solr/core/CodecFactory.java 2011-11-02 21:31:19.100564754 -0400 @@ -0,0 +1,33 @@ +package org.apache.solr.core; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.util.plugin.NamedListInitializedPlugin; + +/** + * Factory for plugging in a custom {@link Codec} + */ +public abstract class CodecFactory implements NamedListInitializedPlugin { + public void init(NamedList args) { + } + + public abstract Codec create(IndexSchema Schema); +} diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/CodecProviderFactory.java lucene-2621/solr/core/src/java/org/apache/solr/core/CodecProviderFactory.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/CodecProviderFactory.java 2011-07-11 21:29:55.171819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/core/CodecProviderFactory.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,32 +0,0 @@ -package org.apache.solr.core; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.util.plugin.NamedListInitializedPlugin; - -/** - * Factory for plugging in a custom {@link CodecProvider} - */ -public abstract class CodecProviderFactory implements NamedListInitializedPlugin { - public void init(NamedList args) { - } - - public abstract CodecProvider create(); -} diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java lucene-2621/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/solr/core/src/java/org/apache/solr/core/DefaultCodecFactory.java 2011-11-01 23:57:42.360604742 -0400 @@ -0,0 +1,59 @@ +package org.apache.solr.core; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.schema.SchemaField; + +/** + * Default CodecFactory implementation, extends Lucene's + * and returns postings format implementations according to the + * schema configuration. + * @lucene.experimental + */ +public class DefaultCodecFactory extends CodecFactory { + + // TODO: we need to change how solr does this? + // rather than a string like "Pulsing" you need to be able to pass parameters + // and everything to a field in the schema, e.g. we should provide factories for + // the Lucene's core formats (Memory, Pulsing, ...) and such. + // + // So I think a FieldType should return PostingsFormat, not a String. + // how it constructs this from the XML... i don't care. + + @Override + public Codec create(final IndexSchema schema) { + return new Lucene40Codec() { + @Override + public PostingsFormat getPostingsFormatForField(String field) { + final SchemaField fieldOrNull = schema.getFieldOrNull(field); + if (fieldOrNull == null) { + throw new IllegalArgumentException("no such field " + field); + } + String postingsFormatName = fieldOrNull.getType().getPostingsFormat(); + if (postingsFormatName != null) { + return PostingsFormat.forName(postingsFormatName); + } + return super.getPostingsFormatForField(field); + } + }; + } +} diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java lucene-2621/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java 2011-07-11 21:29:55.171819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java 2011-11-01 23:57:57.448604735 -0400 @@ -19,7 +19,6 @@ import java.io.IOException; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.solr.common.util.NamedList; import org.apache.solr.util.plugin.NamedListInitializedPlugin; @@ -29,7 +28,6 @@ */ public abstract class IndexReaderFactory implements NamedListInitializedPlugin { protected int termInfosIndexDivisor = 1;//IndexReader.DEFAULT_TERMS_INDEX_DIVISOR; Set this once Lucene makes this public. - protected CodecProvider provider; /** * Potentially initializes {@link #termInfosIndexDivisor}. Overriding classes should call super.init() in order * to make sure termInfosIndexDivisor is set. @@ -65,11 +63,4 @@ */ public abstract IndexReader newReader(Directory indexDir, boolean readOnly) throws IOException; - - /** - * Sets the codec provider for this IndexReaderFactory - */ - public void setCodecProvider(CodecProvider provider) { - this.provider = provider; - } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/RequestHandlers.java lucene-2621/solr/core/src/java/org/apache/solr/core/RequestHandlers.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/RequestHandlers.java 2011-10-27 10:39:10.604851980 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/core/RequestHandlers.java 2011-11-02 13:11:13.924580214 -0400 @@ -283,7 +283,7 @@ } public String getVersion() { - String rev = "$Revision: 1189568 $"; + String rev = "$Revision: 1196701 $"; if( _handler != null ) { rev += " :: " + _handler.getVersion(); } @@ -291,7 +291,7 @@ } public String getSourceId() { - String rev = "$Id: RequestHandlers.java 1189568 2011-10-27 01:14:47Z ehatcher $"; + String rev = "$Id: RequestHandlers.java 1196701 2011-11-02 17:11:00Z rmuir $"; if( _handler != null ) { rev += " :: " + _handler.getSourceId(); } @@ -299,7 +299,7 @@ } public String getSource() { - String rev = "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/RequestHandlers.java $"; + String rev = "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/core/RequestHandlers.java $"; if( _handler != null ) { rev += "\n" + _handler.getSource(); } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecProvider.java lucene-2621/solr/core/src/java/org/apache/solr/core/SchemaCodecProvider.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/SchemaCodecProvider.java 2011-07-11 21:29:55.161819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/core/SchemaCodecProvider.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,155 +0,0 @@ -package org.apache.solr.core; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import java.util.Collection; -import java.util.Set; - -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.SegmentInfosReader; -import org.apache.lucene.index.codecs.SegmentInfosWriter; -import org.apache.solr.schema.IndexSchema; -import org.apache.solr.schema.SchemaField; - -/** - * Selects a codec based on a {@link IndexSchema}. This {@link CodecProvider} - * also supports dynamic fields such that not all field codecs need to be known - * in advance - */ -final class SchemaCodecProvider extends CodecProvider { - private final IndexSchema schema; - private final CodecProvider delegate; - - SchemaCodecProvider(IndexSchema schema, CodecProvider delegate) { - this.schema = schema; - this.delegate = delegate; - } - - @Override - public Codec lookup(String name) { - synchronized (delegate) { - return delegate.lookup(name); - } - } - - @Override - public String getFieldCodec(String name) { - synchronized (delegate) { - if (!delegate.hasFieldCodec(name)) { - final SchemaField fieldOrNull = schema.getFieldOrNull(name); - if (fieldOrNull == null) { - throw new IllegalArgumentException("no such field " + name); - } - String codecName = fieldOrNull.getType().getCodec(); - if (codecName == null) { - codecName = delegate.getDefaultFieldCodec(); - } - delegate.setFieldCodec(name, codecName); - return codecName; - } - return delegate.getFieldCodec(name); - } - } - - @Override - public int hashCode() { - return delegate.hashCode(); - } - - @Override - public void register(Codec codec) { - throw new UnsupportedOperationException(); - } - - @Override - public void unregister(Codec codec) { - throw new UnsupportedOperationException(); - } - - @Override - public Collection getAllExtensions() { - return delegate.getAllExtensions(); - } - - @Override - public SegmentInfosWriter getSegmentInfosWriter() { - return delegate.getSegmentInfosWriter(); - } - - @Override - public SegmentInfosReader getSegmentInfosReader() { - return delegate.getSegmentInfosReader(); - } - - @Override - public boolean equals(Object obj) { - return delegate.equals(obj); - } - - @Override - public void setFieldCodec(String field, String codec) { - throw new UnsupportedOperationException(); - } - - @Override - public String getDefaultFieldCodec() { - return delegate.getDefaultFieldCodec(); - } - - @Override - public boolean isCodecRegistered(String name) { - synchronized (delegate) { - return delegate.isCodecRegistered(name); - } - } - - @Override - public void setDefaultFieldCodec(String codec) { - throw new UnsupportedOperationException(); - } - - @Override - public boolean hasFieldCodec(String fieldName) { - synchronized (delegate) { - if (!delegate.hasFieldCodec(fieldName)) { - final SchemaField fieldOrNull = schema.getFieldOrNull(fieldName); - if (fieldOrNull == null) { - return false; - } - String codecName = fieldOrNull.getType().getCodec(); - if (codecName == null) { - codecName = delegate.getDefaultFieldCodec(); - } - delegate.setFieldCodec(fieldName, codecName); - } - return true; - } - } - - @Override - public String toString() { - return "SchemaCodecProvider(" + delegate.toString() + ")"; - } - - @Override - public Set listAll() { - synchronized (delegate) { - return delegate.listAll(); - } - } -} diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java lucene-2621/solr/core/src/java/org/apache/solr/core/SolrConfig.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/SolrConfig.java 2011-09-08 17:00:37.176036247 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/core/SolrConfig.java 2011-11-02 21:31:11.244564760 -0400 @@ -35,8 +35,6 @@ import org.apache.solr.spelling.QueryConverter; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.index.IndexDeletionPolicy; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.Version; import org.slf4j.Logger; @@ -200,7 +198,7 @@ loadPluginInfo(DirectoryFactory.class,"directoryFactory",false, true); loadPluginInfo(IndexDeletionPolicy.class,"mainIndex/deletionPolicy",false, true); - loadPluginInfo(CodecProviderFactory.class,"mainIndex/codecProviderFactory",false, false); + loadPluginInfo(CodecFactory.class,"mainIndex/codecFactory",false, false); loadPluginInfo(IndexReaderFactory.class,"indexReaderFactory",false, true); loadPluginInfo(UpdateRequestProcessorChain.class,"updateRequestProcessorChain",false, false); loadPluginInfo(UpdateLog.class,"updateHandler/updateLog",false, false); diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java lucene-2621/solr/core/src/java/org/apache/solr/core/SolrCore.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java 2011-11-02 22:08:42.032563601 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/core/SolrCore.java 2011-11-03 13:44:31.480534671 -0400 @@ -20,7 +20,7 @@ import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.store.Directory; import org.apache.solr.common.SolrException; @@ -91,7 +91,7 @@ private IndexDeletionPolicyWrapper solrDelPolicy; private DirectoryFactory directoryFactory; private IndexReaderFactory indexReaderFactory; - private final CodecProvider codecProvider; + private final Codec codec; public long getStartTime() { return startTime; } @@ -347,7 +347,6 @@ indexReaderFactory = new StandardIndexReaderFactory(); } this.indexReaderFactory = indexReaderFactory; - this.indexReaderFactory.setCodecProvider(codecProvider); } // protect via synchronized(SolrCore.class) @@ -383,7 +382,7 @@ log.warn(logid+"Solr index directory '" + new File(indexDir) + "' doesn't exist." + " Creating new index..."); - SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.mainIndexConfig, solrDelPolicy, codecProvider, false); + SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.mainIndexConfig, solrDelPolicy, codec, false); writer.close(); } @@ -560,7 +559,7 @@ initDeletionPolicy(); - this.codecProvider = initCodecProvider(solrConfig, schema); + this.codec= initCodec(solrConfig, schema); if (updateHandler == null) { initDirectoryFactory(); @@ -638,18 +637,16 @@ resourceLoader.inform(infoRegistry); } - private CodecProvider initCodecProvider(SolrConfig solrConfig, IndexSchema schema) { - final PluginInfo info = solrConfig.getPluginInfo(CodecProviderFactory.class.getName()); - CodecProvider cp; + private Codec initCodec(SolrConfig solrConfig, final IndexSchema schema) { + final PluginInfo info = solrConfig.getPluginInfo(CodecFactory.class.getName()); + final CodecFactory factory; if (info != null) { - CodecProviderFactory factory = (CodecProviderFactory) schema.getResourceLoader().newInstance(info.className); + factory = (CodecFactory) schema.getResourceLoader().newInstance(info.className); factory.init(info.initArgs); - cp = factory.create(); } else { - // make sure we use the default if nothing is configured - cp = CodecProvider.getDefault(); + factory = new DefaultCodecFactory(); } - return new SchemaCodecProvider(schema, cp); + return factory.create(schema); } /** @@ -1830,11 +1827,11 @@ } public String getSourceId() { - return "$Id: SolrCore.java 1196797 2011-11-02 21:02:17Z yonik $"; + return "$Id: SolrCore.java 1197240 2011-11-03 17:44:17Z rmuir $"; } public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/core/SolrCore.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/core/SolrCore.java $"; } public URL[] getDocs() { @@ -1853,8 +1850,8 @@ return lst; } - public CodecProvider getCodecProvider() { - return codecProvider; + public Codec getCodec() { + return codec; } public final class LazyQueryResponseWriterWrapper implements QueryResponseWriter { diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java lucene-2621/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java 2011-07-11 21:29:55.161819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java 2011-11-01 23:59:37.492604683 -0400 @@ -35,6 +35,6 @@ @Override public IndexReader newReader(Directory indexDir, boolean readOnly) throws IOException { - return IndexReader.open(indexDir, null, readOnly, termInfosIndexDivisor, provider); + return IndexReader.open(indexDir, null, readOnly, termInfosIndexDivisor); } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java 2011-07-11 21:30:07.701819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java 2011-09-11 11:01:09.366123944 -0400 @@ -121,7 +121,7 @@ } public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/AdminHandlers.java $"; } public Category getCategory() { diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java 2011-09-16 09:41:18.076281604 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java 2011-11-02 00:46:26.668603235 -0400 @@ -213,6 +213,7 @@ for (int i = 0; i < dirNames.length; i++) { Directory dir = dirFactory.get(dirNames[i], core.getSolrConfig().mainIndexConfig.lockType); dirsToBeReleased[i] = dir; + // TODO: why doesn't this use the IR factory? what is going on here? readersToBeClosed[i] = IndexReader.open(dir, true); } } @@ -603,16 +604,16 @@ @Override public String getVersion() { - return "$Revision: 1171304 $"; + return "$Revision: 1196453 $"; } @Override public String getSourceId() { - return "$Id: CoreAdminHandler.java 1171304 2011-09-15 22:41:18Z janhoy $"; + return "$Id: CoreAdminHandler.java 1196453 2011-11-02 04:46:14Z rmuir $"; } @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java 2011-07-11 21:30:07.701819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java 2011-09-11 11:01:09.366123944 -0400 @@ -111,6 +111,6 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/PluginInfoHandler.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java 2011-07-11 21:30:07.701819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java 2011-09-11 11:01:09.366123944 -0400 @@ -66,6 +66,6 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/PropertiesRequestHandler.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java 2011-07-17 06:27:34.458948135 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java 2011-09-11 11:01:09.366123944 -0400 @@ -233,6 +233,6 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/ShowFileRequestHandler.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java 2011-07-11 21:30:07.701819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java 2011-09-11 11:01:09.366123944 -0400 @@ -107,7 +107,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/SolrInfoMBeanHandler.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java 2011-07-11 21:30:07.701819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java 2011-09-11 11:01:09.366123944 -0400 @@ -292,7 +292,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/SystemInfoHandler.java $"; } private static final long ONE_KB = 1024; diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java 2011-07-11 21:30:07.701819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java 2011-09-11 11:01:09.366123944 -0400 @@ -144,6 +144,6 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/admin/ThreadDumpHandler.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java lucene-2621/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java 2011-07-11 21:30:20.691819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java 2011-09-11 11:01:09.366123944 -0400 @@ -260,7 +260,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/DebugComponent.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java lucene-2621/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java 2011-07-31 23:12:09.602328018 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java 2011-09-11 11:01:09.366123944 -0400 @@ -593,17 +593,17 @@ @Override public String getVersion() { - return "$Revision: 1152530 $"; + return "$Revision: 1160700 $"; } @Override public String getSourceId() { - return "$Id: FacetComponent.java 1152530 2011-07-31 00:30:19Z koji $"; + return "$Id: FacetComponent.java 1160700 2011-08-23 14:06:58Z rmuir $"; } @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java lucene-2621/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java 2011-07-11 21:30:20.701819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java 2011-09-11 11:01:09.366123944 -0400 @@ -199,7 +199,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/HighlightComponent.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java lucene-2621/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java 2011-07-11 21:30:20.701819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java 2011-09-11 11:01:09.366123944 -0400 @@ -128,7 +128,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/MoreLikeThisComponent.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java lucene-2621/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java 2011-07-11 21:30:20.691819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java 2011-09-11 11:01:09.366123944 -0400 @@ -249,7 +249,7 @@ // } // // public String getSource() { -// return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java $"; +// return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/PivotFacetHelper.java $"; // } // // public String getVersion() { diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java lucene-2621/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java 2011-11-02 22:08:42.036563601 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java 2011-11-03 13:44:31.476534671 -0400 @@ -223,17 +223,17 @@ @Override public String getVersion() { - return "$Revision: 1196797 $"; + return "$Revision: 1197240 $"; } @Override public String getSourceId() { - return "$Id: RealTimeGetComponent.java 1196797 2011-11-02 21:02:17Z yonik $"; + return "$Id: RealTimeGetComponent.java 1197240 2011-11-03 17:44:17Z rmuir $"; } @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/component/RealTimeGetComponent.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java 2011-07-11 21:30:20.691819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java 2011-09-11 11:01:09.376123944 -0400 @@ -330,7 +330,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java lucene-2621/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java 2011-07-11 21:30:20.701819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java 2011-09-11 11:01:09.376123944 -0400 @@ -163,7 +163,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/StatsComponent.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java lucene-2621/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java 2011-07-11 21:30:20.701819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java 2011-09-11 11:01:09.376123944 -0400 @@ -483,7 +483,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/component/TermsComponent.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java 2011-10-27 10:39:10.612851980 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java 2011-11-02 13:11:13.968580214 -0400 @@ -122,17 +122,17 @@ @Override public String getVersion() { - return "$Revision: 1189719 $"; + return "$Revision: 1196701 $"; } @Override public String getSourceId() { - return "$Id: DocumentAnalysisRequestHandler.java 1189719 2011-10-27 12:31:49Z ehatcher $"; + return "$Id: DocumentAnalysisRequestHandler.java 1196701 2011-11-02 17:11:00Z rmuir $"; } @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java $"; } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java 2011-07-11 21:30:24.421819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java 2011-09-11 11:01:09.376123944 -0400 @@ -79,6 +79,6 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/DumpRequestHandler.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java 2011-07-11 21:30:24.421819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java 2011-09-11 11:01:09.376123944 -0400 @@ -118,7 +118,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java $"; } // ================================================= Helper methods ================================================ diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java 2011-07-11 21:30:24.421819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java 2011-09-11 11:01:09.376123944 -0400 @@ -59,7 +59,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/JsonUpdateRequestHandler.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java 2011-07-11 21:30:24.411819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java 2011-09-11 11:01:09.376123944 -0400 @@ -100,6 +100,6 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/PingRequestHandler.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/StandardRequestHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/StandardRequestHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/StandardRequestHandler.java 2011-07-11 21:30:24.411819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/StandardRequestHandler.java 2011-09-11 11:01:09.376123944 -0400 @@ -62,7 +62,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/StandardRequestHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/StandardRequestHandler.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java lucene-2621/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java 2011-07-11 21:30:24.411819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java 2011-09-11 11:01:09.386123944 -0400 @@ -93,7 +93,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/handler/XsltUpdateRequestHandler.java $"; } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/DefaultEncoder.java lucene-2621/solr/core/src/java/org/apache/solr/highlight/DefaultEncoder.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/DefaultEncoder.java 2011-07-11 21:29:41.951819802 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/highlight/DefaultEncoder.java 2011-09-11 11:01:09.386123944 -0400 @@ -42,7 +42,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/DefaultEncoder.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/DefaultEncoder.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/GapFragmenter.java lucene-2621/solr/core/src/java/org/apache/solr/highlight/GapFragmenter.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/GapFragmenter.java 2011-07-11 21:29:41.951819802 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/highlight/GapFragmenter.java 2011-09-11 11:01:09.386123944 -0400 @@ -61,7 +61,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/GapFragmenter.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/GapFragmenter.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/HtmlEncoder.java lucene-2621/solr/core/src/java/org/apache/solr/highlight/HtmlEncoder.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/HtmlEncoder.java 2011-07-11 21:29:41.951819802 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/highlight/HtmlEncoder.java 2011-09-11 11:01:09.386123944 -0400 @@ -42,7 +42,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/HtmlEncoder.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/HtmlEncoder.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/HtmlFormatter.java lucene-2621/solr/core/src/java/org/apache/solr/highlight/HtmlFormatter.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/HtmlFormatter.java 2011-07-11 21:29:41.951819802 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/highlight/HtmlFormatter.java 2011-09-11 11:01:09.386123944 -0400 @@ -60,6 +60,6 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/HtmlFormatter.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/HtmlFormatter.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/RegexFragmenter.java lucene-2621/solr/core/src/java/org/apache/solr/highlight/RegexFragmenter.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/RegexFragmenter.java 2011-07-11 21:29:41.951819802 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/highlight/RegexFragmenter.java 2011-09-11 11:01:09.386123944 -0400 @@ -107,7 +107,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/RegexFragmenter.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/RegexFragmenter.java $"; } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/SimpleFragListBuilder.java lucene-2621/solr/core/src/java/org/apache/solr/highlight/SimpleFragListBuilder.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/SimpleFragListBuilder.java 2011-07-11 21:29:41.951819802 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/highlight/SimpleFragListBuilder.java 2011-09-11 11:01:09.386123944 -0400 @@ -44,7 +44,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/SimpleFragListBuilder.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/SimpleFragListBuilder.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/SingleFragListBuilder.java lucene-2621/solr/core/src/java/org/apache/solr/highlight/SingleFragListBuilder.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/highlight/SingleFragListBuilder.java 2011-07-11 21:29:41.951819802 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/highlight/SingleFragListBuilder.java 2011-09-11 11:01:09.386123944 -0400 @@ -44,7 +44,7 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/highlight/SingleFragListBuilder.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/highlight/SingleFragListBuilder.java $"; } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java lucene-2621/solr/core/src/java/org/apache/solr/schema/FieldType.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/schema/FieldType.java 2011-09-28 16:10:03.538773855 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/schema/FieldType.java 2011-10-28 20:52:16.244788517 -0400 @@ -167,10 +167,10 @@ initArgs.remove("positionIncrementGap"); } - final String codec = initArgs.get("codec"); - if (codec != null) { - this.codec = codec; - initArgs.remove("codec"); + final String postingsFormat = initArgs.get("postingsFormat"); + if (postingsFormat != null) { + this.postingsFormat = postingsFormat; + initArgs.remove("postingsFormat"); } if (initArgs.size() > 0) { @@ -527,12 +527,12 @@ } /** - * The codec ID used for this field type + * The postings format used for this field type */ - protected String codec; + protected String postingsFormat; - public String getCodec() { - return codec; + public String getPostingsFormat() { + return postingsFormat; } /** diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java lucene-2621/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java 2011-07-17 06:27:34.438948135 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java 2011-09-11 11:01:09.386123944 -0400 @@ -327,17 +327,17 @@ @Override public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java $"; } @Override public String getSourceId() { - return "$Id: FileFloatSource.java 1147284 2011-07-15 19:14:01Z hossman $"; + return "$Id: FileFloatSource.java 1160700 2011-08-23 14:06:58Z rmuir $"; } @Override public String getVersion() { - return "$Revision: 1147284 $"; + return "$Revision: 1160700 $"; } } } diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/search/LRUCache.java lucene-2621/solr/core/src/java/org/apache/solr/search/LRUCache.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/search/LRUCache.java 2011-07-11 21:29:32.201819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/search/LRUCache.java 2011-09-11 11:01:09.386123944 -0400 @@ -231,7 +231,7 @@ } public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/LRUCache.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/search/LRUCache.java $"; } public URL[] getDocs() { diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java lucene-2621/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java 2011-07-11 21:29:32.181819804 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java 2011-09-11 11:01:09.396123944 -0400 @@ -50,7 +50,7 @@ return "$Id: SolrFieldCacheMBean.java 1144761 2011-07-09 23:01:53Z sarowe $"; } public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/flexscoring/solr/core/src/java/org/apache/solr/search/SolrFieldCacheMBean.java $"; } public URL[] getDocs() { return null; diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java lucene-2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java 2011-11-03 14:37:00.600533051 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java 2011-11-03 13:44:31.484534673 -0400 @@ -1865,11 +1865,11 @@ } public String getSourceId() { - return "$Id: SolrIndexSearcher.java 1196797 2011-11-02 21:02:17Z yonik $"; + return "$Id: SolrIndexSearcher.java 1197240 2011-11-03 17:44:17Z rmuir $"; } public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java $"; } public URL[] getDocs() { diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java lucene-2621/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java 2011-11-03 08:48:29.608543825 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java 2011-11-02 00:26:47.780603843 -0400 @@ -29,6 +29,7 @@ import org.apache.lucene.search.spell.HighFrequencyDictionary; import org.apache.lucene.search.spell.PlainTextDictionary; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.schema.FieldType; @@ -62,7 +63,14 @@ try { loadExternalFileDictionary(core); spellChecker.clearIndex(); - spellChecker.indexDictionary(dictionary); + // TODO: you should be able to specify the IWC params? + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null); + // TODO: if we enable this, codec gets angry since field won't exist in the schema + // config.setCodec(core.getCodec()); + ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(300); + // TODO: does Solr really want to continue passing 'optimize=true' to the spellchecker here? + // (its been doing this behind the scenes all along, but its wasteful. + spellChecker.indexDictionary(dictionary, config, true); } catch (IOException e) { throw new RuntimeException(e); } @@ -94,6 +102,8 @@ setMaxBufferedDocs(150). setMergePolicy(mp). setOpenMode(IndexWriterConfig.OpenMode.CREATE) + // TODO: if we enable this, codec gets angry since field won't exist in the schema + // .setCodec(core.getCodec()) ); List lines = core.getResourceLoader().getLines(sourceLocation, characterEncoding); @@ -106,7 +116,7 @@ writer.optimize(); writer.close(); - dictionary = new HighFrequencyDictionary(IndexReader.open(ramDir), + dictionary = new HighFrequencyDictionary(IndexReader.open(ramDir, true), WORD_FIELD_NAME, 0.0f); } else { // check if character encoding is defined diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java lucene-2621/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java 2011-11-03 08:48:31.008543823 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java 2011-11-02 00:22:18.872603981 -0400 @@ -17,8 +17,11 @@ */ import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.search.spell.HighFrequencyDictionary; +import org.apache.lucene.util.Version; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; @@ -63,7 +66,7 @@ if (sourceLocation != null) { try { FSDirectory luceneIndexDir = FSDirectory.open(new File(sourceLocation)); - this.reader = IndexReader.open(luceneIndexDir); + this.reader = IndexReader.open(luceneIndexDir, true); } catch (IOException e) { throw new RuntimeException(e); } @@ -85,8 +88,19 @@ // Create the dictionary dictionary = new HighFrequencyDictionary(reader, field, threshold); + // TODO: maybe whether or not to clear the index should be configurable? + // an incremental update is faster (just adds new terms), but if you 'expunged' + // old terms I think they might hang around. spellChecker.clearIndex(); - spellChecker.indexDictionary(dictionary); + // TODO: you should be able to specify the IWC params? + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null); + + // TODO: if we enable this, codec gets angry since field won't exist in the schema + // config.setCodec(core.getCodec()); + ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(300); + // TODO: does Solr really want to continue passing 'optimize=true' to the spellchecker here? + // (its been doing this behind the scenes all along, but its wasteful. + spellChecker.indexDictionary(dictionary, config, true); } catch (IOException e) { throw new RuntimeException(e); diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java lucene-2621/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java 2011-08-18 16:38:29.981460859 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java 2011-11-02 00:03:54.788604547 -0400 @@ -78,7 +78,7 @@ boolean removeAllExisting, boolean forceNewDirectory) throws IOException { return new SolrIndexWriter(name, core.getNewIndexDir(), core.getDirectoryFactory(), removeAllExisting, core.getSchema(), - core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy(), core.getCodecProvider(), forceNewDirectory); + core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy(), core.getCodec(), forceNewDirectory); } @Override diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java lucene-2621/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java 2011-11-02 22:08:42.024563598 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java 2011-11-03 13:44:31.528534671 -0400 @@ -491,11 +491,11 @@ } public String getSourceId() { - return "$Id: DirectUpdateHandler2.java 1196797 2011-11-02 21:02:17Z yonik $"; + return "$Id: DirectUpdateHandler2.java 1197240 2011-11-03 17:44:17Z rmuir $"; } public String getSource() { - return "$URL: https://svn.apache.org/repos/asf/lucene/dev/trunk/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java $"; + return "$URL: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/update/DirectUpdateHandler2.java $"; } public URL[] getDocs() { diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java lucene-2621/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java --- lucene-clean-trunk/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java 2011-08-18 16:38:29.981460859 -0400 +++ lucene-2621/solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java 2011-11-02 00:04:26.924604534 -0400 @@ -29,7 +29,7 @@ import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.Directory; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.schema.IndexSchema; @@ -52,12 +52,12 @@ private PrintStream infoStream; private DirectoryFactory directoryFactory; - public SolrIndexWriter(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, CodecProvider codecProvider, boolean forceNewDirectory) throws IOException { + public SolrIndexWriter(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec, boolean forceNewDirectory) throws IOException { super( directoryFactory.get(path, config.lockType, forceNewDirectory), config.toIndexWriterConfig(schema). setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND). - setIndexDeletionPolicy(delPolicy).setCodecProvider(codecProvider) + setIndexDeletionPolicy(delPolicy).setCodec(codec) ); log.debug("Opened Writer " + name); this.name = name; diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test/org/apache/solr/core/MockCodecProviderFactory.java lucene-2621/solr/core/src/test/org/apache/solr/core/MockCodecProviderFactory.java --- lucene-clean-trunk/solr/core/src/test/org/apache/solr/core/MockCodecProviderFactory.java 2011-07-11 21:26:08.351819805 -0400 +++ lucene-2621/solr/core/src/test/org/apache/solr/core/MockCodecProviderFactory.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,63 +0,0 @@ -package org.apache.solr.core; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.pulsing.PulsingCodec; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.solr.common.util.NamedList; - -/** - * CodecProviderFactory for testing, it inits a CP with Standard and Pulsing, - * and also adds any codecs specified by classname in solrconfig. - */ -public class MockCodecProviderFactory extends CodecProviderFactory { - private String defaultCodec; - private NamedList codecs; - - @Override - public void init(NamedList args) { - super.init(args); - defaultCodec = (String) args.get("defaultCodec"); - codecs = (NamedList) args.get("codecs"); - } - - @Override - public CodecProvider create() { - CodecProvider cp = new CodecProvider(); - cp.register(new StandardCodec()); - cp.register(new PulsingCodec()); - if (codecs != null) { - for (Object codec : codecs.getAll("name")) { - if (!cp.isCodecRegistered((String)codec)) { - try { - Class clazz = Class.forName((String)codec).asSubclass(Codec.class); - cp.register(clazz.newInstance()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - } - if (defaultCodec != null) { - cp.setDefaultFieldCodec(defaultCodec); - } - return cp; - } -} diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test/org/apache/solr/core/TestCodecProviderSupport.java lucene-2621/solr/core/src/test/org/apache/solr/core/TestCodecProviderSupport.java --- lucene-clean-trunk/solr/core/src/test/org/apache/solr/core/TestCodecProviderSupport.java 2011-07-11 21:26:08.351819805 -0400 +++ lucene-2621/solr/core/src/test/org/apache/solr/core/TestCodecProviderSupport.java 1969-12-31 19:00:00.000000000 -0500 @@ -1,106 +0,0 @@ -package org.apache.solr.core; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import java.util.Map; - -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.schema.SchemaField; -import org.junit.BeforeClass; - -public class TestCodecProviderSupport extends SolrTestCaseJ4 { - - @BeforeClass - public static void beforeClass() throws Exception { - initCore("solrconfig_codec.xml", "schema_codec.xml"); - } - - public void testCodecs() { - CodecProvider codecProvider = h.getCore().getCodecProvider(); - Map fields = h.getCore().getSchema().getFields(); - SchemaField schemaField = fields.get("string_pulsing_f"); - assertEquals("Pulsing", codecProvider.getFieldCodec(schemaField.getName())); - schemaField = fields.get("string_simpletext_f"); - assertEquals("SimpleText", - codecProvider.getFieldCodec(schemaField.getName())); - schemaField = fields.get("string_standard_f"); - assertEquals("Standard", codecProvider.getFieldCodec(schemaField.getName())); - schemaField = fields.get("string_f"); - assertEquals("Pulsing", codecProvider.getFieldCodec(schemaField.getName())); - - assertTrue(codecProvider.hasFieldCodec("string_simpletext_f")); - assertTrue(codecProvider.hasFieldCodec("string_standard_f")); - assertTrue(codecProvider.hasFieldCodec("string_f")); - } - - public void testDynamicFields() { - CodecProvider codecProvider = h.getCore().getCodecProvider(); - - assertTrue(codecProvider.hasFieldCodec("bar_simple")); - assertTrue(codecProvider.hasFieldCodec("bar_pulsing")); - assertTrue(codecProvider.hasFieldCodec("bar_standard")); - - assertEquals("SimpleText", codecProvider.getFieldCodec("foo_simple")); - assertEquals("Pulsing", codecProvider.getFieldCodec("foo_pulsing")); - assertEquals("Standard", codecProvider.getFieldCodec("foo_standard")); - } - - public void testUnmodifiable() { - CodecProvider codecProvider = h.getCore().getCodecProvider(); - try { - codecProvider.setDefaultFieldCodec("foo"); - fail("should be unmodifiable"); - } catch (UnsupportedOperationException e) { - // - } - - try { - codecProvider.setFieldCodec("foo", "bar"); - fail("should be unmodifiable"); - } catch (UnsupportedOperationException e) { - // - } - - try { - codecProvider.register(new StandardCodec()); - fail("should be unmodifiable"); - } catch (UnsupportedOperationException e) { - // - } - - try { - codecProvider.unregister(new StandardCodec()); - fail("should be unmodifiable"); - } catch (UnsupportedOperationException e) { - // - } - } - - public void testUnknownField() { - CodecProvider codecProvider = h.getCore().getCodecProvider(); - try { - codecProvider.getFieldCodec("notexisting"); - fail("field is not existing"); - } catch (IllegalArgumentException e) { - // - } - - } -} diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java lucene-2621/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java --- lucene-clean-trunk/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java 1969-12-31 19:00:00.000000000 -0500 +++ lucene-2621/solr/core/src/test/org/apache/solr/core/TestCodecSupport.java 2011-11-02 21:43:26.720564381 -0400 @@ -0,0 +1,73 @@ +package org.apache.solr.core; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.util.Map; + +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.schema.SchemaField; +import org.junit.BeforeClass; + +public class TestCodecSupport extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig-basic.xml", "schema_codec.xml"); + } + + public void testPostingsFormats() { + Codec codec = h.getCore().getCodec(); + Map fields = h.getCore().getSchema().getFields(); + SchemaField schemaField = fields.get("string_pulsing_f"); + PerFieldPostingsFormat format = (PerFieldPostingsFormat) codec.postingsFormat(); + assertEquals("Pulsing40", format.getPostingsFormatForField(schemaField.getName()).getName()); + schemaField = fields.get("string_simpletext_f"); + assertEquals("SimpleText", + format.getPostingsFormatForField(schemaField.getName()).getName()); + schemaField = fields.get("string_standard_f"); + assertEquals("Lucene40", format.getPostingsFormatForField(schemaField.getName()).getName()); + schemaField = fields.get("string_f"); + assertEquals("Lucene40", format.getPostingsFormatForField(schemaField.getName()).getName()); + } + + public void testDynamicFields() { + Codec codec = h.getCore().getCodec(); + PerFieldPostingsFormat format = (PerFieldPostingsFormat) codec.postingsFormat(); + + assertEquals("SimpleText", format.getPostingsFormatForField("foo_simple").getName()); + assertEquals("SimpleText", format.getPostingsFormatForField("bar_simple").getName()); + assertEquals("Pulsing40", format.getPostingsFormatForField("foo_pulsing").getName()); + assertEquals("Pulsing40", format.getPostingsFormatForField("bar_pulsing").getName()); + assertEquals("Lucene40", format.getPostingsFormatForField("foo_standard").getName()); + assertEquals("Lucene40", format.getPostingsFormatForField("bar_standard").getName()); + } + + public void testUnknownField() { + Codec codec = h.getCore().getCodec(); + PerFieldPostingsFormat format = (PerFieldPostingsFormat) codec.postingsFormat(); + try { + format.getPostingsFormatForField("notexisting"); + fail("field is not existing"); + } catch (IllegalArgumentException e) { + // + } + + } +} diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test/org/apache/solr/schema/TestCollationField.java lucene-2621/solr/core/src/test/org/apache/solr/schema/TestCollationField.java --- lucene-clean-trunk/solr/core/src/test/org/apache/solr/schema/TestCollationField.java 2011-09-27 14:52:41.398773982 -0400 +++ lucene-2621/solr/core/src/test/org/apache/solr/schema/TestCollationField.java 2011-11-02 00:05:16.396604507 -0400 @@ -25,7 +25,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; @@ -36,7 +36,8 @@ @BeforeClass public static void beforeClass() throws Exception { - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", + "Lucene3x".equals(Codec.getDefault().getName())); String home = setupSolrHome(); initCore("solrconfig.xml","schema.xml", home); // add some docs diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java lucene-2621/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java --- lucene-clean-trunk/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java 2011-09-27 14:52:41.398773982 -0400 +++ lucene-2621/solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java 2011-11-02 00:04:56.152604517 -0400 @@ -18,7 +18,7 @@ package org.apache.solr.search.function; import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.TFIDFSimilarity; @@ -413,7 +413,7 @@ */ public void testTotalTermFreq() throws Exception { assumeFalse("PreFlex codec does not support collection-level term stats", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + "Lucene3x".equals(Codec.getDefault().getName())); clearIndex(); diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test-files/solr/conf/schema_codec.xml lucene-2621/solr/core/src/test-files/solr/conf/schema_codec.xml --- lucene-clean-trunk/solr/core/src/test-files/solr/conf/schema_codec.xml 2011-07-11 21:27:32.411819805 -0400 +++ lucene-2621/solr/core/src/test-files/solr/conf/schema_codec.xml 2011-11-02 00:10:53.308604333 -0400 @@ -17,9 +17,9 @@ --> - - - + + + diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig_codec.xml lucene-2621/solr/core/src/test-files/solr/conf/solrconfig_codec.xml --- lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig_codec.xml 2011-08-17 09:20:42.090062764 -0400 +++ lucene-2621/solr/core/src/test-files/solr/conf/solrconfig_codec.xml 2011-11-02 00:09:23.884604381 -0400 @@ -19,15 +19,5 @@ ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - Pulsing - - org.apache.lucene.index.codecs.simpletext.SimpleTextCodec - org.apache.lucene.index.codecs.preflex.PreFlexCodec - - - - diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-master1.xml lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-master1.xml --- lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-master1.xml 2011-11-03 13:25:07.004535273 -0400 +++ lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-master1.xml 2011-11-03 13:44:31.508534673 -0400 @@ -17,7 +17,7 @@ limitations under the License. --> - diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-master2.xml lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-master2.xml --- lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-master2.xml 2011-11-03 13:25:07.004535273 -0400 +++ lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-master2.xml 2011-11-03 13:44:31.508534673 -0400 @@ -17,7 +17,7 @@ limitations under the License. --> - diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-master.xml lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-master.xml --- lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-master.xml 2011-11-03 13:25:07.004535273 -0400 +++ lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-master.xml 2011-11-03 13:44:31.508534673 -0400 @@ -17,7 +17,7 @@ limitations under the License. --> - diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-repeater.xml lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-repeater.xml --- lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-repeater.xml 2011-11-03 13:25:07.008535273 -0400 +++ lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-repeater.xml 2011-11-03 13:44:31.508534673 -0400 @@ -17,7 +17,7 @@ limitations under the License. --> - diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-slave1.xml lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-slave1.xml --- lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-slave1.xml 2011-11-03 13:25:07.004535273 -0400 +++ lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-slave1.xml 2011-11-03 13:44:31.512534674 -0400 @@ -17,7 +17,7 @@ limitations under the License. --> - diff -ruN -x .svn -x build lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-slave.xml lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-slave.xml --- lucene-clean-trunk/solr/core/src/test-files/solr/conf/solrconfig-slave.xml 2011-11-03 13:25:07.004535273 -0400 +++ lucene-2621/solr/core/src/test-files/solr/conf/solrconfig-slave.xml 2011-11-03 13:44:31.512534674 -0400 @@ -17,7 +17,7 @@ limitations under the License. --> - diff -ruN -x .svn -x build lucene-clean-trunk/solr/solrj/src/test-files/solrj/solr/conf/schema-replication1.xml lucene-2621/solr/solrj/src/test-files/solrj/solr/conf/schema-replication1.xml --- lucene-clean-trunk/solr/solrj/src/test-files/solrj/solr/conf/schema-replication1.xml 2011-07-24 19:00:11.232328885 -0400 +++ lucene-2621/solr/solrj/src/test-files/solrj/solr/conf/schema-replication1.xml 2011-09-11 11:01:09.406123944 -0400 @@ -23,7 +23,7 @@ kitchen sink thrown in. See example/solr/conf/schema.xml for a more concise example. - $Id: schema-replication1.xml 1144761 2011-07-09 23:01:53Z sarowe $ + $Id: schema-replication1.xml 1160700 2011-08-23 14:06:58Z rmuir $ $Source$ $Name$ --> diff -ruN -x .svn -x build lucene-clean-trunk/solr/solrj/src/test-files/solrj/solr/conf/solrconfig-slave1.xml lucene-2621/solr/solrj/src/test-files/solrj/solr/conf/solrconfig-slave1.xml --- lucene-clean-trunk/solr/solrj/src/test-files/solrj/solr/conf/solrconfig-slave1.xml 2011-11-03 13:25:07.020535273 -0400 +++ lucene-2621/solr/solrj/src/test-files/solrj/solr/conf/solrconfig-slave1.xml 2011-11-03 13:44:31.504534671 -0400 @@ -17,7 +17,7 @@ limitations under the License. --> -