Index: lucene/src/test/org/apache/lucene/index/TestAddIndexes.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (revision 1029583) +++ lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (working copy) @@ -24,6 +24,12 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.PerFieldCodecWrapper; +import org.apache.lucene.index.codecs.mocksep.MockSepCodec; +import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; +import org.apache.lucene.index.codecs.standard.StandardCodec; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.store.RAMDirectory; @@ -33,7 +39,7 @@ import org.apache.lucene.search.PhraseQuery; public class TestAddIndexes extends LuceneTestCase { - + public void testSimpleCase() throws IOException { // main directory Directory dir = newDirectory(); @@ -43,31 +49,39 @@ IndexWriter writer = null; - writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer()) - .setOpenMode(OpenMode.CREATE)); + writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE)); // add 100 documents addDocs(writer, 100); assertEquals(100, writer.maxDoc()); writer.close(); _TestUtil.checkIndex(dir); - writer = newWriter(aux, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file + writer = newWriter(aux, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE)); + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundFile(false); // use one without a compound file + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundDocStore(false); // use one without a compound file // add 40 documents in separate files addDocs(writer, 40); assertEquals(40, writer.maxDoc()); writer.close(); - writer = newWriter(aux2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE)); + writer = newWriter(aux2, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE)); // add 40 documents in compound files addDocs2(writer, 50); assertEquals(50, writer.maxDoc()); writer.close(); // test doc count before segments are merged - writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND)); + writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND)); assertEquals(100, writer.maxDoc()); writer.addIndexes(new Directory[] { aux, aux2 }); assertEquals(190, writer.maxDoc()); @@ -82,14 +96,17 @@ // now add another set in. Directory aux3 = newDirectory(); - writer = newWriter(aux3, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())); + writer = newWriter(aux3, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())); // add 40 documents addDocs(writer, 40); assertEquals(40, writer.maxDoc()); writer.close(); // test doc count before segments are merged/index is optimized - writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND)); + writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND)); assertEquals(190, writer.maxDoc()); writer.addIndexes(new Directory[] { aux3 }); assertEquals(230, writer.maxDoc()); @@ -103,7 +120,9 @@ verifyTermDocs(dir, new Term("content", "bbb"), 50); // now optimize it. - writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND)); + writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND)); writer.optimize(); writer.close(); @@ -116,11 +135,14 @@ // now add a single document Directory aux4 = newDirectory(); - writer = newWriter(aux4, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())); + writer = newWriter(aux4, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())); addDocs2(writer, 1); writer.close(); - writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND)); + writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND)); assertEquals(230, writer.maxDoc()); writer.addIndexes(new Directory[] { aux4 }); assertEquals(231, writer.maxDoc()); @@ -143,17 +165,20 @@ Directory aux = newDirectory(); setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND)); - writer.addIndexes(new Directory[] {aux}); + IndexWriter writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND)); + writer.addIndexes(new Directory[] { aux }); // Adds 10 docs, then replaces them with another 10 // docs, so 10 pending deletes: for (int i = 0; i < 20; i++) { Document doc = new Document(); - doc.add(newField("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED)); + doc.add(newField("id", "" + (i % 10), Field.Store.NO, + Field.Index.NOT_ANALYZED)); doc.add(newField("content", "bbb " + i, Field.Store.NO, - Field.Index.ANALYZED)); - writer.updateDocument(new Term("id", "" + (i%10)), doc); + Field.Index.ANALYZED)); + writer.updateDocument(new Term("id", "" + (i % 10)), doc); } // Deletes one of the 10 added docs, leaving 9: PhraseQuery q = new PhraseQuery(); @@ -180,19 +205,23 @@ Directory aux = newDirectory(); setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND)); + IndexWriter writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND)); // Adds 10 docs, then replaces them with another 10 // docs, so 10 pending deletes: for (int i = 0; i < 20; i++) { Document doc = new Document(); - doc.add(newField("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED)); - doc.add(newField("content", "bbb " + i, Field.Store.NO, Field.Index.ANALYZED)); - writer.updateDocument(new Term("id", "" + (i%10)), doc); + doc.add(newField("id", "" + (i % 10), Field.Store.NO, + Field.Index.NOT_ANALYZED)); + doc.add(newField("content", "bbb " + i, Field.Store.NO, + Field.Index.ANALYZED)); + writer.updateDocument(new Term("id", "" + (i % 10)), doc); } - - writer.addIndexes(new Directory[] {aux}); - + + writer.addIndexes(new Directory[] { aux }); + // Deletes one of the 10 added docs, leaving 9: PhraseQuery q = new PhraseQuery(); q.add(new Term("content", "bbb")); @@ -218,16 +247,19 @@ Directory aux = newDirectory(); setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND)); + IndexWriter writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND)); // Adds 10 docs, then replaces them with another 10 // docs, so 10 pending deletes: for (int i = 0; i < 20; i++) { Document doc = new Document(); - doc.add(newField("id", "" + (i % 10), Field.Store.NO, Field.Index.NOT_ANALYZED)); + doc.add(newField("id", "" + (i % 10), Field.Store.NO, + Field.Index.NOT_ANALYZED)); doc.add(newField("content", "bbb " + i, Field.Store.NO, - Field.Index.ANALYZED)); - writer.updateDocument(new Term("id", "" + (i%10)), doc); + Field.Index.ANALYZED)); + writer.updateDocument(new Term("id", "" + (i % 10)), doc); } // Deletes one of the 10 added docs, leaving 9: @@ -236,7 +268,7 @@ q.add(new Term("content", "14")); writer.deleteDocuments(q); - writer.addIndexes(new Directory[] {aux}); + writer.addIndexes(new Directory[] { aux }); writer.optimize(); writer.commit(); @@ -259,31 +291,41 @@ IndexWriter writer = null; - writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())); + writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())); // add 100 documents addDocs(writer, 100); assertEquals(100, writer.maxDoc()); writer.close(); - writer = newWriter(aux, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file + writer = newWriter(aux, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundFile(false); // use one without a compound file + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundDocStore(false); // use one without a compound file // add 140 documents in separate files addDocs(writer, 40); writer.close(); - writer = newWriter(aux, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file + writer = newWriter(aux, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundFile(false); // use one without a compound file + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundDocStore(false); // use one without a compound file addDocs(writer, 100); writer.close(); - writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND)); + writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND)); try { // cannot add self writer.addIndexes(new Directory[] { aux, dir }); assertTrue(false); - } - catch (IllegalArgumentException e) { + } catch (IllegalArgumentException e) { assertEquals(100, writer.maxDoc()); } writer.close(); @@ -305,9 +347,9 @@ setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer()) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10)); + IndexWriter writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); addDocs(writer, 10); @@ -331,7 +373,9 @@ setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(9)); + IndexWriter writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(9)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); addDocs(writer, 2); @@ -355,12 +399,13 @@ setUpDirs(dir, aux); - IndexWriter writer = newWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer()) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10)); + IndexWriter writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - writer.addIndexes(new Directory[] { aux, new MockDirectoryWrapper(new RAMDirectory(aux)) }); + writer.addIndexes(new Directory[] { aux, + new MockDirectoryWrapper(new RAMDirectory(aux)) }); assertEquals(1060, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -387,12 +432,13 @@ assertEquals(10, reader.numDocs()); reader.close(); - IndexWriter writer = newWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer()) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(4)); + IndexWriter writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(4)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); - writer.addIndexes(new Directory[] { aux, new MockDirectoryWrapper(new RAMDirectory(aux)) }); + writer.addIndexes(new Directory[] { aux, + new MockDirectoryWrapper(new RAMDirectory(aux)) }); assertEquals(1060, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -410,9 +456,9 @@ setUpDirs(dir, aux); - IndexWriter writer = newWriter(aux2, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer()) - .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(100)); + IndexWriter writer = newWriter(aux2, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(100)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); writer.addIndexes(new Directory[] { aux }); assertEquals(30, writer.maxDoc()); @@ -433,8 +479,9 @@ assertEquals(22, reader.numDocs()); reader.close(); - writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) - .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(6)); + writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(6)); ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(4); writer.addIndexes(new Directory[] { aux, aux2 }); @@ -456,8 +503,7 @@ private void addDocs(IndexWriter writer, int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); - doc.add(newField("content", "aaa", Field.Store.NO, - Field.Index.ANALYZED)); + doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED)); writer.addDocument(doc); } } @@ -465,8 +511,16 @@ private void addDocs2(IndexWriter writer, int numDocs) throws IOException { for (int i = 0; i < numDocs; i++) { Document doc = new Document(); - doc.add(newField("content", "bbb", Field.Store.NO, - Field.Index.ANALYZED)); + doc.add(newField("content", "bbb", Field.Store.NO, Field.Index.ANALYZED)); + writer.addDocument(doc); + } + } + + private void addDocs3(IndexWriter writer, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED)); + doc.add(newField("id", "" + i, Field.Store.YES, Field.Index.ANALYZED)); writer.addDocument(doc); } } @@ -481,7 +535,8 @@ private void verifyTermDocs(Directory dir, Term term, int numDocs) throws IOException { IndexReader reader = IndexReader.open(dir, true); - DocsEnum docsEnum = MultiFields.getTermDocsEnum(reader, null, term.field, term.bytes); + DocsEnum docsEnum = MultiFields.getTermDocsEnum(reader, null, term.field, + term.bytes); int count = 0; while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) count++; @@ -492,24 +547,34 @@ private void setUpDirs(Directory dir, Directory aux) throws IOException { IndexWriter writer = null; - writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); + writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(1000)); // add 1000 documents in 1 segment addDocs(writer, 1000); assertEquals(1000, writer.maxDoc()); assertEquals(1, writer.getSegmentCount()); writer.close(); - writer = newWriter(aux, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(100)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file + writer = newWriter(aux, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE).setMaxBufferedDocs(100)); + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundFile(false); // use one without a compound file + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundDocStore(false); // use one without a compound file ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); // add 30 documents in 3 segments for (int i = 0; i < 3; i++) { addDocs(writer, 10); writer.close(); - writer = newWriter(aux, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(100)); - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundFile(false); // use one without a compound file - ((LogMergePolicy) writer.getConfig().getMergePolicy()).setUseCompoundDocStore(false); // use one without a compound file + writer = newWriter(aux, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(100)); + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundFile(false); // use one without a compound file + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundDocStore(false); // use one without a compound file ((LogMergePolicy) writer.getConfig().getMergePolicy()).setMergeFactor(10); } assertEquals(30, writer.maxDoc()); @@ -526,25 +591,26 @@ lmp.setUseCompoundDocStore(false); lmp.setMergeFactor(100); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new MockAnalyzer()) - .setMaxBufferedDocs(5).setMergePolicy(lmp)); + TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(5) + .setMergePolicy(lmp)); Document doc = new Document(); - doc.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.ANALYZED, Field.TermVector.WITH_POSITIONS_OFFSETS)); - for(int i=0;i<60;i++) + doc.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", + Field.Store.YES, Field.Index.ANALYZED, + Field.TermVector.WITH_POSITIONS_OFFSETS)); + for (int i = 0; i < 60; i++) writer.addDocument(doc); Document doc2 = new Document(); - doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", Field.Store.YES, - Field.Index.NO)); - for(int i=0;i<10;i++) + doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", + Field.Store.YES, Field.Index.NO)); + doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", + Field.Store.YES, Field.Index.NO)); + doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", + Field.Store.YES, Field.Index.NO)); + doc2.add(newField("content", "aaa bbb ccc ddd eee fff ggg hhh iii", + Field.Store.YES, Field.Index.NO)); + for (int i = 0; i < 10; i++) writer.addDocument(doc2); writer.close(); @@ -555,12 +621,86 @@ lmp.setUseCompoundDocStore(false); lmp.setMergeFactor(4); writer = new IndexWriter(dir2, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer()) - .setMergeScheduler(new SerialMergeScheduler()).setMergePolicy(lmp)); - writer.addIndexes(new Directory[] {dir}); + new MockAnalyzer()).setMergeScheduler(new SerialMergeScheduler()) + .setMergePolicy(lmp)); + writer.addIndexes(new Directory[] { dir }); writer.close(); dir.close(); dir2.close(); } + public void testSimpleCaseCustomCodecProvider() throws IOException { + // main directory + Directory dir = newDirectory(); + // two auxiliary directories + Directory aux = newDirectory(); + Directory aux2 = newDirectory(); + CodecProvider provider = new MockCodecProvider(); + IndexWriter writer = null; + + writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE).setCodecProvider(provider)); + // add 100 documents + addDocs3(writer, 100); + assertEquals(100, writer.maxDoc()); + writer.commit(); + writer.close(); + _TestUtil.checkIndex(dir, provider); + + writer = newWriter(aux, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE).setCodecProvider(provider)); + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundFile(false); // use one without a compound file + ((LogMergePolicy) writer.getConfig().getMergePolicy()) + .setUseCompoundDocStore(false); // use one without a compound file + // add 40 documents in separate files + addDocs(writer, 40); + assertEquals(40, writer.maxDoc()); + writer.commit(); + writer.close(); + + writer = newWriter(aux2, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE).setCodecProvider(provider)); + // add 40 documents in compound files + addDocs2(writer, 50); + assertEquals(50, writer.maxDoc()); + writer.commit(); + writer.close(); + + // test doc count before segments are merged + writer = newWriter(dir, + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.APPEND).setCodecProvider(provider)); + assertEquals(100, writer.maxDoc()); + writer.addIndexes(new Directory[] { aux, aux2 }); + assertEquals(190, writer.maxDoc()); + writer.close(); + _TestUtil.checkIndex(dir, provider); + + dir.close(); + aux.close(); + aux2.close(); + } + + public static class MockCodecProvider extends CodecProvider { + final PerFieldCodecWrapper perField; + + public MockCodecProvider() { + StandardCodec standardCodec = new StandardCodec(); + perField = new PerFieldCodecWrapper(standardCodec); + perField.add("id", new SimpleTextCodec()); + perField.add("content", new MockSepCodec()); + register(perField); + } + + @Override + public Codec getWriter(SegmentWriteState state) { + return perField; + } + + } + } Index: lucene/src/test/org/apache/lucene/index/TestPerFieldCodecWrapper.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestPerFieldCodecWrapper.java (revision 0) +++ lucene/src/test/org/apache/lucene/index/TestPerFieldCodecWrapper.java (revision 0) @@ -0,0 +1,116 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.PerFieldCodecWrapper; +import org.apache.lucene.index.codecs.mocksep.MockSepCodec; +import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; +import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util._TestUtil; + +/** + * + * + */ +public class TestPerFieldCodecWrapper extends LuceneTestCase { + + private IndexWriter newWriter(Directory dir, IndexWriterConfig conf) + throws IOException { + conf.setMergePolicy(new LogDocMergePolicy()); + final IndexWriter writer = new IndexWriter(dir, conf); + return writer; + } + + private void addDocs(IndexWriter writer, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(newField("content", "aaa", Field.Store.NO, Field.Index.ANALYZED)); + writer.addDocument(doc); + } + } + + private void addDocs2(IndexWriter writer, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(newField("content", "bbb", Field.Store.NO, Field.Index.ANALYZED)); + writer.addDocument(doc); + } + } + + private void addDocs3(IndexWriter writer, int numDocs) throws IOException { + for (int i = 0; i < numDocs; i++) { + Document doc = new Document(); + doc.add(newField("content", "ccc", Field.Store.NO, Field.Index.ANALYZED)); + doc.add(newField("id", "" + i, Field.Store.YES, Field.Index.ANALYZED)); + writer.addDocument(doc); + } + } + + /* + * Test is hetrogenous index segements are merge sucessfully + */ + public void testMergeUnusedPerFieldCodec() throws IOException { + Directory dir = newDirectory(); + CodecProvider provider = new MockCodecProvider(); + IndexWriterConfig iwconf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()) + .setOpenMode(OpenMode.CREATE).setCodecProvider(provider); + ((LogMergePolicy)iwconf.getMergePolicy()).setMaxMergeDocs(10); + IndexWriter writer = newWriter(dir,iwconf + ); + addDocs(writer, 10); + writer.commit(); + addDocs3(writer, 10); + writer.commit(); + addDocs2(writer, 10); + writer.commit(); + assertEquals(30, writer.maxDoc()); + _TestUtil.checkIndex(dir, provider); + writer.optimize(); + assertEquals(30, writer.maxDoc()); + writer.close(); + dir.close(); + } + + public static class MockCodecProvider extends CodecProvider { + final PerFieldCodecWrapper perField; + + public MockCodecProvider() { + StandardCodec standardCodec = new StandardCodec(); + perField = new PerFieldCodecWrapper(standardCodec); + perField.add("id", new SimpleTextCodec()); + perField.add("content", new MockSepCodec()); + register(perField); + } + + @Override + public Codec getWriter(SegmentWriteState state) { + return perField; + } + + } + +}