Property changes on: . ___________________________________________________________________ Modified: svn:ignore - classes build dist *~ velocity.log build.properties .idea *.iml *.ipr *.iws .project .classpath .settings prj.el bin pom.xml + classes build dist *~ velocity.log build.properties .idea *.iml *.ipr *.iws .project .classpath .settings prj.el bin bin.* pom.xml Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621:r1188713-1197363 Property changes on: solr ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr:r1188713-1197363 Property changes on: solr/testlogging.properties ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/testlogging.properties:r1188713-1197363 Property changes on: solr/common-build.xml ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/common-build.xml:r1188713-1197363 Index: solr/CHANGES.txt =================================================================== --- solr/CHANGES.txt (revision 1197230) +++ solr/CHANGES.txt (working copy) @@ -133,10 +133,8 @@ fq={!join from=name to=parent}eyes:blue (yonik) -* SOLR-1942: Added the ability to select codec per fieldType in schema.xml - as well as support custom CodecProviders in solrconfig.xml. - NOTE: IndexReaderFactory now has a codecProvider that should be passed - to IndexReader.open (in the case you have a custom IndexReaderFactory). +* SOLR-1942: Added the ability to select postings format per fieldType in schema.xml + as well as support custom Codecs in solrconfig.xml. (simonw via rmuir) * SOLR-2136: Boolean type added to function queries, along with Property changes on: solr/CHANGES.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/CHANGES.txt:r1188713-1197363 Property changes on: solr/scripts ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/scripts:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/scripts:r1188713-1197363 Property changes on: solr/core ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/core:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/core:r1188713-1197363 Property changes on: solr/core/src/test ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/core/src/test:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/core/src/test:r1188713-1197363 Index: solr/core/src/test/org/apache/solr/core/TestCodecProviderSupport.java (deleted) =================================================================== Index: solr/core/src/test/org/apache/solr/core/MockCodecProviderFactory.java (deleted) =================================================================== Index: solr/core/src/test/org/apache/solr/schema/TestCollationField.java =================================================================== --- solr/core/src/test/org/apache/solr/schema/TestCollationField.java (revision 1197230) +++ solr/core/src/test/org/apache/solr/schema/TestCollationField.java (working copy) @@ -25,7 +25,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; @@ -36,7 +36,8 @@ @BeforeClass public static void beforeClass() throws Exception { - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", + "Lucene3x".equals(Codec.getDefault().getName())); String home = setupSolrHome(); initCore("solrconfig.xml","schema.xml", home); // add some docs Index: solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java =================================================================== --- solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java (revision 1197230) +++ solr/core/src/test/org/apache/solr/search/function/TestFunctionQuery.java (working copy) @@ -18,7 +18,7 @@ package org.apache.solr.search.function; import org.apache.lucene.index.FieldInvertState; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.FieldCache; import org.apache.lucene.search.similarities.DefaultSimilarity; import org.apache.lucene.search.similarities.TFIDFSimilarity; @@ -413,7 +413,7 @@ */ public void testTotalTermFreq() throws Exception { assumeFalse("PreFlex codec does not support collection-level term stats", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + "Lucene3x".equals(Codec.getDefault().getName())); clearIndex(); Index: solr/core/src/test-files/solr/conf/solrconfig_codec.xml =================================================================== --- solr/core/src/test-files/solr/conf/solrconfig_codec.xml (revision 1197230) +++ solr/core/src/test-files/solr/conf/solrconfig_codec.xml (working copy) @@ -19,15 +19,5 @@ ${tests.luceneMatchVersion:LUCENE_CURRENT} - - - Pulsing - - org.apache.lucene.index.codecs.simpletext.SimpleTextCodec - org.apache.lucene.index.codecs.preflex.PreFlexCodec - - - - Index: solr/core/src/test-files/solr/conf/schema_codec.xml =================================================================== --- solr/core/src/test-files/solr/conf/schema_codec.xml (revision 1197230) +++ solr/core/src/test-files/solr/conf/schema_codec.xml (working copy) @@ -17,9 +17,9 @@ --> - - - + + + Property changes on: solr/core/src/java ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/core/src/java:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/core/src/java:r1188713-1197363 Index: solr/core/src/java/org/apache/solr/schema/FieldType.java =================================================================== --- solr/core/src/java/org/apache/solr/schema/FieldType.java (revision 1197230) +++ solr/core/src/java/org/apache/solr/schema/FieldType.java (working copy) @@ -167,10 +167,10 @@ initArgs.remove("positionIncrementGap"); } - final String codec = initArgs.get("codec"); - if (codec != null) { - this.codec = codec; - initArgs.remove("codec"); + final String postingsFormat = initArgs.get("postingsFormat"); + if (postingsFormat != null) { + this.postingsFormat = postingsFormat; + initArgs.remove("postingsFormat"); } if (initArgs.size() > 0) { @@ -527,12 +527,12 @@ } /** - * The codec ID used for this field type + * The postings format used for this field type */ - protected String codec; + protected String postingsFormat; - public String getCodec() { - return codec; + public String getPostingsFormat() { + return postingsFormat; } /** Index: solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java =================================================================== --- solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java (revision 1197230) +++ solr/core/src/java/org/apache/solr/update/SolrIndexWriter.java (working copy) @@ -29,7 +29,7 @@ import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.Directory; import org.apache.solr.core.DirectoryFactory; import org.apache.solr.schema.IndexSchema; @@ -52,12 +52,12 @@ private PrintStream infoStream; private DirectoryFactory directoryFactory; - public SolrIndexWriter(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, CodecProvider codecProvider, boolean forceNewDirectory) throws IOException { + public SolrIndexWriter(String name, String path, DirectoryFactory directoryFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy, Codec codec, boolean forceNewDirectory) throws IOException { super( directoryFactory.get(path, config.lockType, forceNewDirectory), config.toIndexWriterConfig(schema). setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND). - setIndexDeletionPolicy(delPolicy).setCodecProvider(codecProvider) + setIndexDeletionPolicy(delPolicy).setCodec(codec) ); log.debug("Opened Writer " + name); this.name = name; Index: solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java =================================================================== --- solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (revision 1197230) +++ solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java (working copy) @@ -78,7 +78,7 @@ boolean removeAllExisting, boolean forceNewDirectory) throws IOException { return new SolrIndexWriter(name, core.getNewIndexDir(), core.getDirectoryFactory(), removeAllExisting, core.getSchema(), - core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy(), core.getCodecProvider(), forceNewDirectory); + core.getSolrConfig().mainIndexConfig, core.getDeletionPolicy(), core.getCodec(), forceNewDirectory); } @Override Index: solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java =================================================================== --- solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java (revision 1197230) +++ solr/core/src/java/org/apache/solr/core/StandardIndexReaderFactory.java (working copy) @@ -35,6 +35,6 @@ @Override public IndexReader newReader(Directory indexDir, boolean readOnly) throws IOException { - return IndexReader.open(indexDir, null, readOnly, termInfosIndexDivisor, provider); + return IndexReader.open(indexDir, null, readOnly, termInfosIndexDivisor); } } Index: solr/core/src/java/org/apache/solr/core/SchemaCodecProvider.java (deleted) =================================================================== Index: solr/core/src/java/org/apache/solr/core/CodecProviderFactory.java (deleted) =================================================================== Index: solr/core/src/java/org/apache/solr/core/SolrCore.java =================================================================== --- solr/core/src/java/org/apache/solr/core/SolrCore.java (revision 1197230) +++ solr/core/src/java/org/apache/solr/core/SolrCore.java (working copy) @@ -20,7 +20,7 @@ import org.apache.lucene.index.IndexDeletionPolicy; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.store.Directory; import org.apache.solr.common.SolrException; @@ -91,7 +91,7 @@ private IndexDeletionPolicyWrapper solrDelPolicy; private DirectoryFactory directoryFactory; private IndexReaderFactory indexReaderFactory; - private final CodecProvider codecProvider; + private final Codec codec; public long getStartTime() { return startTime; } @@ -347,7 +347,6 @@ indexReaderFactory = new StandardIndexReaderFactory(); } this.indexReaderFactory = indexReaderFactory; - this.indexReaderFactory.setCodecProvider(codecProvider); } // protect via synchronized(SolrCore.class) @@ -383,7 +382,7 @@ log.warn(logid+"Solr index directory '" + new File(indexDir) + "' doesn't exist." + " Creating new index..."); - SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.mainIndexConfig, solrDelPolicy, codecProvider, false); + SolrIndexWriter writer = new SolrIndexWriter("SolrCore.initIndex", indexDir, getDirectoryFactory(), true, schema, solrConfig.mainIndexConfig, solrDelPolicy, codec, false); writer.close(); } @@ -560,7 +559,7 @@ initDeletionPolicy(); - this.codecProvider = initCodecProvider(solrConfig, schema); + this.codec= initCodec(solrConfig, schema); if (updateHandler == null) { initDirectoryFactory(); @@ -638,18 +637,16 @@ resourceLoader.inform(infoRegistry); } - private CodecProvider initCodecProvider(SolrConfig solrConfig, IndexSchema schema) { - final PluginInfo info = solrConfig.getPluginInfo(CodecProviderFactory.class.getName()); - CodecProvider cp; + private Codec initCodec(SolrConfig solrConfig, final IndexSchema schema) { + final PluginInfo info = solrConfig.getPluginInfo(CodecFactory.class.getName()); + final CodecFactory factory; if (info != null) { - CodecProviderFactory factory = (CodecProviderFactory) schema.getResourceLoader().newInstance(info.className); + factory = (CodecFactory) schema.getResourceLoader().newInstance(info.className); factory.init(info.initArgs); - cp = factory.create(); } else { - // make sure we use the default if nothing is configured - cp = CodecProvider.getDefault(); + factory = new DefaultCodecFactory(); } - return new SchemaCodecProvider(schema, cp); + return factory.create(schema); } /** @@ -1853,8 +1850,8 @@ return lst; } - public CodecProvider getCodecProvider() { - return codecProvider; + public Codec getCodec() { + return codec; } public final class LazyQueryResponseWriterWrapper implements QueryResponseWriter { Index: solr/core/src/java/org/apache/solr/core/SolrConfig.java =================================================================== --- solr/core/src/java/org/apache/solr/core/SolrConfig.java (revision 1197230) +++ solr/core/src/java/org/apache/solr/core/SolrConfig.java (working copy) @@ -35,8 +35,6 @@ import org.apache.solr.spelling.QueryConverter; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.index.IndexDeletionPolicy; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.Version; import org.slf4j.Logger; @@ -200,7 +198,7 @@ loadPluginInfo(DirectoryFactory.class,"directoryFactory",false, true); loadPluginInfo(IndexDeletionPolicy.class,"mainIndex/deletionPolicy",false, true); - loadPluginInfo(CodecProviderFactory.class,"mainIndex/codecProviderFactory",false, false); + loadPluginInfo(CodecFactory.class,"mainIndex/codecFactory",false, false); loadPluginInfo(IndexReaderFactory.class,"indexReaderFactory",false, true); loadPluginInfo(UpdateRequestProcessorChain.class,"updateRequestProcessorChain",false, false); loadPluginInfo(UpdateLog.class,"updateHandler/updateLog",false, false); Index: solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java =================================================================== --- solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java (revision 1197230) +++ solr/core/src/java/org/apache/solr/core/IndexReaderFactory.java (working copy) @@ -19,7 +19,6 @@ import java.io.IOException; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.solr.common.util.NamedList; import org.apache.solr.util.plugin.NamedListInitializedPlugin; @@ -29,7 +28,6 @@ */ public abstract class IndexReaderFactory implements NamedListInitializedPlugin { protected int termInfosIndexDivisor = 1;//IndexReader.DEFAULT_TERMS_INDEX_DIVISOR; Set this once Lucene makes this public. - protected CodecProvider provider; /** * Potentially initializes {@link #termInfosIndexDivisor}. Overriding classes should call super.init() in order * to make sure termInfosIndexDivisor is set. @@ -65,11 +63,4 @@ */ public abstract IndexReader newReader(Directory indexDir, boolean readOnly) throws IOException; - - /** - * Sets the codec provider for this IndexReaderFactory - */ - public void setCodecProvider(CodecProvider provider) { - this.provider = provider; - } } Index: solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java =================================================================== --- solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (revision 1197230) +++ solr/core/src/java/org/apache/solr/handler/admin/CoreAdminHandler.java (working copy) @@ -213,6 +213,7 @@ for (int i = 0; i < dirNames.length; i++) { Directory dir = dirFactory.get(dirNames[i], core.getSolrConfig().mainIndexConfig.lockType); dirsToBeReleased[i] = dir; + // TODO: why doesn't this use the IR factory? what is going on here? readersToBeClosed[i] = IndexReader.open(dir, true); } } Index: solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java =================================================================== --- solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java (revision 1197230) +++ solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java (working copy) @@ -29,6 +29,7 @@ import org.apache.lucene.search.spell.HighFrequencyDictionary; import org.apache.lucene.search.spell.PlainTextDictionary; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.util.Version; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; import org.apache.solr.schema.FieldType; @@ -62,7 +63,14 @@ try { loadExternalFileDictionary(core); spellChecker.clearIndex(); - spellChecker.indexDictionary(dictionary); + // TODO: you should be able to specify the IWC params? + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null); + // TODO: if we enable this, codec gets angry since field won't exist in the schema + // config.setCodec(core.getCodec()); + ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(300); + // TODO: does Solr really want to continue passing 'optimize=true' to the spellchecker here? + // (its been doing this behind the scenes all along, but its wasteful. + spellChecker.indexDictionary(dictionary, config, true); } catch (IOException e) { throw new RuntimeException(e); } @@ -94,6 +102,8 @@ setMaxBufferedDocs(150). setMergePolicy(mp). setOpenMode(IndexWriterConfig.OpenMode.CREATE) + // TODO: if we enable this, codec gets angry since field won't exist in the schema + // .setCodec(core.getCodec()) ); List lines = core.getResourceLoader().getLines(sourceLocation, characterEncoding); @@ -106,7 +116,7 @@ writer.optimize(); writer.close(); - dictionary = new HighFrequencyDictionary(IndexReader.open(ramDir), + dictionary = new HighFrequencyDictionary(IndexReader.open(ramDir, true), WORD_FIELD_NAME, 0.0f); } else { // check if character encoding is defined Index: solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java =================================================================== --- solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java (revision 1197230) +++ solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java (working copy) @@ -17,8 +17,11 @@ */ import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.TieredMergePolicy; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.search.spell.HighFrequencyDictionary; +import org.apache.lucene.util.Version; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; @@ -63,7 +66,7 @@ if (sourceLocation != null) { try { FSDirectory luceneIndexDir = FSDirectory.open(new File(sourceLocation)); - this.reader = IndexReader.open(luceneIndexDir); + this.reader = IndexReader.open(luceneIndexDir, true); } catch (IOException e) { throw new RuntimeException(e); } @@ -85,8 +88,19 @@ // Create the dictionary dictionary = new HighFrequencyDictionary(reader, field, threshold); + // TODO: maybe whether or not to clear the index should be configurable? + // an incremental update is faster (just adds new terms), but if you 'expunged' + // old terms I think they might hang around. spellChecker.clearIndex(); - spellChecker.indexDictionary(dictionary); + // TODO: you should be able to specify the IWC params? + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null); + + // TODO: if we enable this, codec gets angry since field won't exist in the schema + // config.setCodec(core.getCodec()); + ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(300); + // TODO: does Solr really want to continue passing 'optimize=true' to the spellchecker here? + // (its been doing this behind the scenes all along, but its wasteful. + spellChecker.indexDictionary(dictionary, config, true); } catch (IOException e) { throw new RuntimeException(e); Property changes on: solr/solrj ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/solrj:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/solrj:r1188713-1197363 Property changes on: solr/solrj/src/test/org/apache/solr/common ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/solrj/src/test/org/apache/solr/common:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/solrj/src/test/org/apache/solr/common:r1188713-1197363 Property changes on: solr/solrj/src/test/org/apache/solr/client ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/solrj/src/test/org/apache/solr/client:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/solrj/src/test/org/apache/solr/client:r1188713-1197363 Property changes on: solr/solrj/src/test/org/apache/solr/client/solrj ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/solrj/src/test/org/apache/solr/client/solrj:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/solrj/src/test/org/apache/solr/client/solrj:r1188713-1197363 Property changes on: solr/solrj/src/java ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/solrj/src/java:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/solrj/src/java:r1188713-1197363 Property changes on: solr/site-src ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/site-src:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/site-src:r1188713-1197363 Property changes on: solr/example ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/example:r1188713-1197363 Property changes on: solr/build.xml ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/build.xml:r1188713-1197363 Property changes on: solr/NOTICE.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/NOTICE.txt:r1188713-1197363 Property changes on: solr/contrib ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/contrib:r1188713-1197363 Property changes on: solr/contrib/clustering/src/test-files ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/contrib/clustering/src/test-files:r1129205-1144716 Merged /lucene/dev/branches/lucene2621/solr/contrib/clustering/src/test-files:r1188713-1197363 Property changes on: solr/contrib/dataimporthandler/src/test/org ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/contrib/dataimporthandler/src/test/org:r1129205-1144716 Merged /lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/test/org:r1188713-1197363 Property changes on: solr/contrib/dataimporthandler/src/test-files ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/contrib/dataimporthandler/src/test-files:r1129205-1144716 Merged /lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/test-files:r1188713-1197363 Property changes on: solr/contrib/dataimporthandler/src/java ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/contrib/dataimporthandler/src/java:r1129205-1144716 Merged /lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/java:r1188713-1197363 Property changes on: solr/contrib/dataimporthandler-extras/src/java ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/contrib/dataimporthandler-extras/src/java:r1129205-1144716 Merged /lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler-extras/src/java:r1188713-1197363 Index: solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java =================================================================== --- solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java (revision 1197230) +++ solr/contrib/analysis-extras/src/test/org/apache/solr/schema/TestICUCollationField.java (working copy) @@ -22,7 +22,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.solr.SolrTestCaseJ4; import org.junit.BeforeClass; @@ -37,7 +37,7 @@ @BeforeClass public static void beforeClass() throws Exception { - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); String home = setupSolrHome(); initCore("solrconfig.xml","schema.xml", home); // add some docs Property changes on: solr/contrib/uima/src/test-files ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/contrib/uima/src/test-files:r1129205-1144716 Merged /lucene/dev/branches/lucene2621/solr/contrib/uima/src/test-files:r1188713-1197363 Property changes on: solr/contrib/uima/src/java ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/contrib/uima/src/java:r1129205-1144716 Merged /lucene/dev/branches/lucene2621/solr/contrib/uima/src/java:r1188713-1197363 Property changes on: solr/LICENSE.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/LICENSE.txt:r1188713-1197363 Property changes on: solr/site ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/site:r1188713-1197363 Property changes on: solr/lib ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/lib:r1188713-1197363 Property changes on: solr/test-framework ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/test-framework:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/test-framework:r1188713-1197363 Property changes on: solr/README.txt ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/README.txt:r1188713-1197363 Property changes on: solr/dev-tools ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/dev-tools:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/dev-tools:r1188713-1197363 Property changes on: solr/client ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/solr/client:r1188713-1197363 Property changes on: solr/webapp ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/branches/solr2452/solr/webapp:r1144174-1144716 Merged /lucene/dev/branches/lucene2621/solr/webapp:r1188713-1197363 Index: modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java =================================================================== --- modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java (revision 1197230) +++ modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java (working copy) @@ -486,11 +486,17 @@ * @throws IOException */ public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB, boolean optimize) throws IOException { + IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null) + .setRAMBufferSizeMB(ramMB); + ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(mergeFactor); + indexDictionary(dict, config, optimize); + } + + public final void indexDictionary(Dictionary dict, IndexWriterConfig config, boolean optimize) throws IOException { synchronized (modifyCurrentIndexLock) { ensureOpen(); final Directory dir = this.spellIndex; - final IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(Version.LUCENE_CURRENT, null).setRAMBufferSizeMB(ramMB)); - ((TieredMergePolicy) writer.getConfig().getMergePolicy()).setMaxMergeAtOnce(mergeFactor); + final IndexWriter writer = new IndexWriter(dir, config); IndexSearcher indexSearcher = obtainSearcher(); final List termsEnums = new ArrayList(); Property changes on: modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java:r1188713-1197363 Index: modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java =================================================================== --- modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java (revision 1197230) +++ modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java (working copy) @@ -22,7 +22,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CollationTestBase; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.util.BytesRef; import java.util.Locale; @@ -45,7 +45,7 @@ @Override public void setUp() throws Exception { super.setUp(); - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); } public void testFarsiRangeFilterCollating() throws Exception { Index: modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java =================================================================== --- modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java (revision 1197230) +++ modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java (working copy) @@ -20,7 +20,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.CollationTestBase; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.util.BytesRef; import java.text.Collator; @@ -47,7 +47,7 @@ @Override public void setUp() throws Exception { super.setUp(); - assumeFalse("preflex format only supports UTF-8 encoded bytes", "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + assumeFalse("preflex format only supports UTF-8 encoded bytes", "Lucene3x".equals(Codec.getDefault().getName())); } public void testFarsiRangeFilterCollating() throws Exception { Index: modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java =================================================================== --- modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java (revision 1197230) +++ modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java (working copy) @@ -33,7 +33,7 @@ import org.apache.lucene.index.NoMergePolicy; import org.apache.lucene.index.NoMergeScheduler; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.util.Version; @@ -133,7 +133,12 @@ final String defaultCodec = config.get("default.codec", null); if (defaultCodec != null) { - CodecProvider.getDefault().setDefaultFieldCodec(defaultCodec); + try { + Class clazz = Class.forName(defaultCodec).asSubclass(Codec.class); + Codec.setDefault(clazz.newInstance()); + } catch (Exception e) { + throw new RuntimeException("Couldn't instantiate Codec: " + defaultCodec, e); + } } final String mergePolicy = config.get("merge.policy", Property changes on: lucene ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/lucene:r1188713-1197363 Index: lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java =================================================================== --- lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (revision 1197230) +++ lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (working copy) @@ -34,6 +34,7 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.queryparser.classic.QueryParser; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TopDocs; @@ -107,7 +108,7 @@ Directory ramdir = newDirectory(); Analyzer analyzer = randomAnalyzer(); IndexWriter writer = new IndexWriter(ramdir, - new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + new IndexWriterConfig(TEST_VERSION_CURRENT, analyzer).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); Document doc = new Document(); Field field1 = newField("foo", fooField.toString(), TextField.TYPE_UNSTORED); Field field2 = newField("term", termField.toString(), TextField.TYPE_UNSTORED); Index: lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java (revision 1197230) +++ lucene/contrib/misc/src/test/org/apache/lucene/search/TestSearcherManager.java (working copy) @@ -34,9 +34,11 @@ import org.apache.lucene.index.ThreadedIndexingAndSearchingTestCase; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; import org.apache.lucene.util.NamedThreadFactory; import org.apache.lucene.util._TestUtil; +@UseNoMemoryExpensiveCodec public class TestSearcherManager extends ThreadedIndexingAndSearchingTestCase { boolean warmCalled; Index: lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (revision 1197230) +++ lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (working copy) @@ -34,11 +34,6 @@ import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.DefaultSegmentInfosReader; -import org.apache.lucene.index.codecs.SegmentInfosReader; -import org.apache.lucene.index.codecs.SegmentInfosWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; @@ -50,30 +45,7 @@ public class TestAppendingCodec extends LuceneTestCase { - static class AppendingCodecProvider extends CodecProvider { - Codec appending = new AppendingCodec(); - SegmentInfosWriter infosWriter = new AppendingSegmentInfosWriter(); - SegmentInfosReader infosReader = new DefaultSegmentInfosReader(); - public AppendingCodecProvider() { - setDefaultFieldCodec(appending.name); - } - @Override - public Codec lookup(String name) { - return appending; - } - - @Override - public SegmentInfosReader getSegmentInfosReader() { - return infosReader; - } - @Override - public SegmentInfosWriter getSegmentInfosWriter() { - return infosWriter; - } - - } - - private static class AppendingIndexOutputWrapper extends IndexOutput { + private static class AppendingIndexOutputWrapper extends IndexOutput { IndexOutput wrapped; public AppendingIndexOutputWrapper(IndexOutput wrapped) { @@ -137,7 +109,7 @@ Directory dir = new AppendingRAMDirectory(random, new RAMDirectory()); IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(random)); - cfg.setCodecProvider(new AppendingCodecProvider()); + cfg.setCodec(new AppendingCodec()); ((TieredMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false); IndexWriter writer = new IndexWriter(dir, cfg); Document doc = new Document(); @@ -151,7 +123,7 @@ writer.addDocument(doc); writer.optimize(); writer.close(); - IndexReader reader = IndexReader.open(dir, null, true, 1, new AppendingCodecProvider()); + IndexReader reader = IndexReader.open(dir, null, true, 1); assertEquals(2, reader.numDocs()); Document doc2 = reader.document(0); assertEquals(text, doc2.get("f")); Index: lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java =================================================================== --- lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java (revision 1197230) +++ lucene/contrib/misc/src/test/org/apache/lucene/index/TestNRTManager.java (working copy) @@ -27,7 +27,9 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NRTCachingDirectory; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; +@UseNoMemoryExpensiveCodec public class TestNRTManager extends ThreadedIndexingAndSearchingTestCase { private final ThreadLocal lastGens = new ThreadLocal(); Index: lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java (revision 1197230) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java (working copy) @@ -20,8 +20,6 @@ import java.io.IOException; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; -import org.apache.lucene.store.Directory; -import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexOutput; public class AppendingSegmentInfosWriter extends DefaultSegmentInfosWriter { Index: lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java (revision 1197230) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java (working copy) @@ -34,9 +34,9 @@ public AppendingTermsDictReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext context, - int termsCacheSize, int codecId) throws IOException { + int termsCacheSize, String segmentSuffix) throws IOException { super(indexReader, dir, fieldInfos, segment, postingsReader, context, - termsCacheSize, codecId); + termsCacheSize, segmentSuffix); } @Override Index: lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java (revision 1197230) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java (working copy) @@ -31,9 +31,9 @@ public class AppendingTermsIndexReader extends FixedGapTermsIndexReader { public AppendingTermsIndexReader(Directory dir, FieldInfos fieldInfos, - String segment, int indexDivisor, Comparator termComp, int codecId, IOContext context) + String segment, int indexDivisor, Comparator termComp, String segmentSuffix, IOContext context) throws IOException { - super(dir, fieldInfos, segment, indexDivisor, termComp, codecId, context); + super(dir, fieldInfos, segment, indexDivisor, termComp, segmentSuffix, context); } @Override Index: lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java =================================================================== --- lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java (revision 1197230) +++ lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java (working copy) @@ -27,7 +27,6 @@ import java.util.List; import org.apache.lucene.index.IndexWriter; // Required for javadocs -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.FSDirectory; /** @@ -54,8 +53,6 @@ */ public class IndexSplitter { public SegmentInfos infos; - - private final CodecProvider codecs; FSDirectory fsDir; @@ -96,17 +93,12 @@ is.split(targetDir, segs.toArray(new String[0])); } } - - public IndexSplitter(File dir) throws IOException { - this(dir, CodecProvider.getDefault()); - } - public IndexSplitter(File dir, CodecProvider codecs) throws IOException { + public IndexSplitter(File dir) throws IOException { this.dir = dir; - this.codecs = codecs; fsDir = FSDirectory.open(dir); - infos = new SegmentInfos(codecs); - infos.read(fsDir, codecs); + infos = new SegmentInfos(); + infos.read(fsDir); } public void listSegments() throws IOException { @@ -140,13 +132,13 @@ infos.remove(idx); } infos.changed(); - infos.commit(fsDir); + infos.commit(fsDir, infos.codecFormat()); } public void split(File destDir, String[] segs) throws IOException { destDir.mkdirs(); FSDirectory destFSDir = FSDirectory.open(destDir); - SegmentInfos destInfos = new SegmentInfos(codecs); + SegmentInfos destInfos = new SegmentInfos(); destInfos.counter = infos.counter; for (String n : segs) { SegmentInfo info = getInfo(n); @@ -160,7 +152,7 @@ } } destInfos.changed(); - destInfos.commit(destFSDir); + destInfos.commit(destFSDir, infos.codecFormat()); // System.out.println("destDir:"+destDir.getAbsolutePath()); } Index: lucene/contrib/contrib-build.xml =================================================================== --- lucene/contrib/contrib-build.xml (revision 1197230) +++ lucene/contrib/contrib-build.xml (working copy) @@ -39,8 +39,8 @@ + - Property changes on: lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java:r1188713-1197363 Index: lucene/common-build.xml =================================================================== --- lucene/common-build.xml (revision 1197230) +++ lucene/common-build.xml (working copy) @@ -83,8 +83,7 @@ - - + @@ -473,8 +472,13 @@ - + + + + + @@ -551,9 +555,7 @@ - - - + Index: lucene/CHANGES.txt =================================================================== --- lucene/CHANGES.txt (revision 1197230) +++ lucene/CHANGES.txt (working copy) @@ -215,7 +215,7 @@ * LUCENE-2881: FieldInfos is now tracked per segment. Before it was tracked per IndexWriter session, which resulted in FieldInfos that had the FieldInfo properties from all previous segments combined. Field numbers are now tracked - globally across IndexWriter sessions and persisted into a X.fnx file on + globally across IndexWriter sessions and persisted into a _X.fnx file on successful commit. The corresponding file format changes are backwards- compatible. (Michael Busch, Simon Willnauer) @@ -375,9 +375,8 @@ * LUCENE-1458, LUCENE-2111: With flexible indexing it is now possible for an application to create its own postings codec, to alter how fields, terms, docs and positions are encoded into the index. The - standard codec is the default codec. Both IndexWriter and - IndexReader accept a CodecProvider class to obtain codecs for newly - written segments as well as existing segments opened for reading. + standard codec is the default codec. IndexWriter accepts a Codec + class to obtain codecs for newly written segments. * LUCENE-1458, LUCENE-2111: Some experimental codecs have been added for flexible indexing, including pulsing codec (inlines @@ -403,7 +402,7 @@ * LUCENE-2489: Added PerFieldCodecWrapper (in oal.index.codecs) which lets you set the Codec per field (Mike McCandless) -* LUCENE-2373: Extend CodecProvider to use SegmentInfosWriter and +* LUCENE-2373: Extend Codec to use SegmentInfosWriter and SegmentInfosReader to allow customization of SegmentInfos data. (Andrzej Bialecki) @@ -445,10 +444,10 @@ (i.e. \* or "*") Custom QueryParser subclasses overriding getRangeQuery() will be passed null for any open endpoint. (Adriano Crestani, yonik) -* LUCENE-2742: Add native per-field codec support. CodecProvider lets you now - register a codec for each field and which is in turn recorded in the segment - and field information. Codecs are maintained on a per-segment basis and be - resolved without knowing the actual codec used for writing the segment. +* LUCENE-2742: Add native per-field postings format support. Codec lets you now + register a postings format for each field and which is in turn recorded + into the index. Postings formtas are maintained on a per-segment basis and be + resolved without knowing the actual postings format used for writing the segment. (Simon Willnauer) * LUCENE-2741: Add support for multiple codecs that use the same file Index: lucene/src/test/org/apache/lucene/TestExternalCodecs.java =================================================================== --- lucene/src/test/org/apache/lucene/TestExternalCodecs.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/TestExternalCodecs.java (working copy) @@ -17,512 +17,47 @@ * limitations under the License. */ -import org.apache.lucene.util.*; -import org.apache.lucene.util.Bits; -import org.apache.lucene.index.*; -import org.apache.lucene.document.*; -import org.apache.lucene.search.*; +import java.io.*; +import java.util.*; + import org.apache.lucene.analysis.*; +import org.apache.lucene.document.*; +import org.apache.lucene.index.*; import org.apache.lucene.index.codecs.*; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.search.*; import org.apache.lucene.store.*; -import java.util.*; -import java.io.*; +import org.apache.lucene.util.*; +import org.apache.lucene.util.Bits; /* Intentionally outside of oal.index to verify fully external codecs work fine */ public class TestExternalCodecs extends LuceneTestCase { - // For fun, test that we can override how terms are - // sorted, and basic things still work -- this comparator - // sorts in reversed unicode code point order: - private static final Comparator reverseUnicodeComparator = new Comparator() { - public int compare(BytesRef t1, BytesRef t2) { - byte[] b1 = t1.bytes; - byte[] b2 = t2.bytes; - int b1Stop; - int b1Upto = t1.offset; - int b2Upto = t2.offset; - if (t1.length < t2.length) { - b1Stop = t1.offset + t1.length; - } else { - b1Stop = t1.offset + t2.length; - } - while(b1Upto < b1Stop) { - final int bb1 = b1[b1Upto++] & 0xff; - final int bb2 = b2[b2Upto++] & 0xff; - if (bb1 != bb2) { - //System.out.println("cmp 1=" + t1 + " 2=" + t2 + " return " + (bb2-bb1)); - return bb2 - bb1; - } - } - - // One is prefix of another, or they are equal - return t2.length-t1.length; - } - - @Override - public boolean equals(Object other) { - return this == other; - } - }; - - // TODO - // - good improvement would be to write through to disk, - // and then load into ram from disk - public static class RAMOnlyCodec extends Codec { + private static final class CustomPerFieldCodec extends Lucene40Codec { - public RAMOnlyCodec() { - super("RamOnly"); - } - // Postings state: - static class RAMPostings extends FieldsProducer { - final Map fieldToTerms = new TreeMap(); + private final PostingsFormat ramFormat = PostingsFormat.forName("RAMOnly"); + private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene40"); + private final PostingsFormat pulsingFormat = PostingsFormat.forName("Pulsing40"); - @Override - public Terms terms(String field) { - return fieldToTerms.get(field); - } - - @Override - public FieldsEnum iterator() { - return new RAMFieldsEnum(this); - } - - @Override - public void close() { - } - } - - static class RAMField extends Terms { - final String field; - final SortedMap termToDocs = new TreeMap(); - long sumTotalTermFreq; - long sumDocFreq; - int docCount; - - RAMField(String field) { - this.field = field; - } - - @Override - public long getUniqueTermCount() { - return termToDocs.size(); - } - - @Override - public long getSumTotalTermFreq() { - return sumTotalTermFreq; - } - - @Override - public long getSumDocFreq() throws IOException { - return sumDocFreq; - } - - @Override - public int getDocCount() throws IOException { - return docCount; - } - - @Override - public TermsEnum iterator() { - return new RAMTermsEnum(RAMOnlyCodec.RAMField.this); - } - - @Override - public Comparator getComparator() { - return reverseUnicodeComparator; - } - } - - static class RAMTerm { - final String term; - long totalTermFreq; - final List docs = new ArrayList(); - public RAMTerm(String term) { - this.term = term; - } - } - - static class RAMDoc { - final int docID; - final int[] positions; - byte[][] payloads; - - public RAMDoc(int docID, int freq) { - this.docID = docID; - positions = new int[freq]; - } - } - - // Classes for writing to the postings state - private static class RAMFieldsConsumer extends FieldsConsumer { - - private final RAMPostings postings; - private final RAMTermsConsumer termsConsumer = new RAMTermsConsumer(); - - public RAMFieldsConsumer(RAMPostings postings) { - this.postings = postings; - } - - @Override - public TermsConsumer addField(FieldInfo field) { - RAMField ramField = new RAMField(field.name); - postings.fieldToTerms.put(field.name, ramField); - termsConsumer.reset(ramField); - return termsConsumer; - } - - @Override - public void close() { - // TODO: finalize stuff - } - } - - private static class RAMTermsConsumer extends TermsConsumer { - private RAMField field; - private final RAMPostingsWriterImpl postingsWriter = new RAMPostingsWriterImpl(); - RAMTerm current; - - void reset(RAMField field) { - this.field = field; - } - - @Override - public PostingsConsumer startTerm(BytesRef text) { - final String term = text.utf8ToString(); - current = new RAMTerm(term); - postingsWriter.reset(current); - return postingsWriter; - } - - - @Override - public Comparator getComparator() { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } - - @Override - public void finishTerm(BytesRef text, TermStats stats) { - assert stats.docFreq > 0; - assert stats.docFreq == current.docs.size(); - current.totalTermFreq = stats.totalTermFreq; - field.termToDocs.put(current.term, current); - } - - @Override - public void finish(long sumTotalTermFreq, long sumDocFreq, int docCount) { - field.sumTotalTermFreq = sumTotalTermFreq; - field.sumDocFreq = sumDocFreq; - field.docCount = docCount; - } - } - - public static class RAMPostingsWriterImpl extends PostingsConsumer { - private RAMTerm term; - private RAMDoc current; - private int posUpto = 0; - - public void reset(RAMTerm term) { - this.term = term; - } - - @Override - public void startDoc(int docID, int freq) { - current = new RAMDoc(docID, freq); - term.docs.add(current); - posUpto = 0; - } - - @Override - public void addPosition(int position, BytesRef payload) { - current.positions[posUpto] = position; - if (payload != null && payload.length > 0) { - if (current.payloads == null) { - current.payloads = new byte[current.positions.length][]; - } - byte[] bytes = current.payloads[posUpto] = new byte[payload.length]; - System.arraycopy(payload.bytes, payload.offset, bytes, 0, payload.length); - } - posUpto++; - } - - @Override - public void finishDoc() { - assert posUpto == current.positions.length; - } - } - - // Classes for reading from the postings state - static class RAMFieldsEnum extends FieldsEnum { - private final RAMPostings postings; - private final Iterator it; - private String current; - - public RAMFieldsEnum(RAMPostings postings) { - this.postings = postings; - this.it = postings.fieldToTerms.keySet().iterator(); - } - - @Override - public String next() { - if (it.hasNext()) { - current = it.next(); - } else { - current = null; - } - return current; - } - - @Override - public TermsEnum terms() { - return new RAMTermsEnum(postings.fieldToTerms.get(current)); - } - } - - static class RAMTermsEnum extends TermsEnum { - Iterator it; - String current; - private final RAMField ramField; - - public RAMTermsEnum(RAMField field) { - this.ramField = field; - } - - @Override - public Comparator getComparator() { - return BytesRef.getUTF8SortedAsUnicodeComparator(); - } - - @Override - public BytesRef next() { - if (it == null) { - if (current == null) { - it = ramField.termToDocs.keySet().iterator(); - } else { - it = ramField.termToDocs.tailMap(current).keySet().iterator(); - } - } - if (it.hasNext()) { - current = it.next(); - return new BytesRef(current); - } else { - return null; - } - } - - @Override - public SeekStatus seekCeil(BytesRef term, boolean useCache) { - current = term.utf8ToString(); - it = null; - if (ramField.termToDocs.containsKey(current)) { - return SeekStatus.FOUND; - } else { - if (current.compareTo(ramField.termToDocs.lastKey()) > 0) { - return SeekStatus.END; - } else { - return SeekStatus.NOT_FOUND; - } - } - } - - @Override - public void seekExact(long ord) { - throw new UnsupportedOperationException(); - } - - @Override - public long ord() { - throw new UnsupportedOperationException(); - } - - @Override - public BytesRef term() { - // TODO: reuse BytesRef - return new BytesRef(current); - } - - @Override - public int docFreq() { - return ramField.termToDocs.get(current).docs.size(); - } - - @Override - public long totalTermFreq() { - return ramField.termToDocs.get(current).totalTermFreq; - } - - @Override - public DocsEnum docs(Bits liveDocs, DocsEnum reuse) { - return new RAMDocsEnum(ramField.termToDocs.get(current), liveDocs); - } - - @Override - public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) { - return new RAMDocsAndPositionsEnum(ramField.termToDocs.get(current), liveDocs); - } - } - - private static class RAMDocsEnum extends DocsEnum { - private final RAMTerm ramTerm; - private final Bits liveDocs; - private RAMDoc current; - int upto = -1; - int posUpto = 0; - - public RAMDocsEnum(RAMTerm ramTerm, Bits liveDocs) { - this.ramTerm = ramTerm; - this.liveDocs = liveDocs; - } - - @Override - public int advance(int targetDocID) { - do { - nextDoc(); - } while (upto < ramTerm.docs.size() && current.docID < targetDocID); - return NO_MORE_DOCS; - } - - // TODO: override bulk read, for better perf - @Override - public int nextDoc() { - while(true) { - upto++; - if (upto < ramTerm.docs.size()) { - current = ramTerm.docs.get(upto); - if (liveDocs == null || liveDocs.get(current.docID)) { - posUpto = 0; - return current.docID; - } - } else { - return NO_MORE_DOCS; - } - } - } - - @Override - public int freq() { - return current.positions.length; - } - - @Override - public int docID() { - return current.docID; - } - } - - private static class RAMDocsAndPositionsEnum extends DocsAndPositionsEnum { - private final RAMTerm ramTerm; - private final Bits liveDocs; - private RAMDoc current; - int upto = -1; - int posUpto = 0; - - public RAMDocsAndPositionsEnum(RAMTerm ramTerm, Bits liveDocs) { - this.ramTerm = ramTerm; - this.liveDocs = liveDocs; - } - - @Override - public int advance(int targetDocID) { - do { - nextDoc(); - } while (upto < ramTerm.docs.size() && current.docID < targetDocID); - return NO_MORE_DOCS; - } - - // TODO: override bulk read, for better perf - @Override - public int nextDoc() { - while(true) { - upto++; - if (upto < ramTerm.docs.size()) { - current = ramTerm.docs.get(upto); - if (liveDocs == null || liveDocs.get(current.docID)) { - posUpto = 0; - return current.docID; - } - } else { - return NO_MORE_DOCS; - } - } - } - - @Override - public int freq() { - return current.positions.length; - } - - @Override - public int docID() { - return current.docID; - } - - @Override - public int nextPosition() { - return current.positions[posUpto++]; - } - - @Override - public boolean hasPayload() { - return current.payloads != null && current.payloads[posUpto-1] != null; - } - - @Override - public BytesRef getPayload() { - return new BytesRef(current.payloads[posUpto-1]); - } - } - - // Holds all indexes created - private final Map state = new HashMap(); - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState writeState) { - RAMPostings postings = new RAMPostings(); - RAMFieldsConsumer consumer = new RAMFieldsConsumer(postings); - synchronized(state) { - state.put(writeState.segmentName, postings); + public PostingsFormat getPostingsFormatForField(String field) { + if (field.equals("field2") || field.equals("id")) { + return pulsingFormat; + } else if (field.equals("field1")) { + return defaultFormat; + } else { + return ramFormat; } - return consumer; } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState readState) - throws IOException { - - synchronized(state) { - return state.get(readState.segmentInfo.name); - } - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return null; - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return null; - } - - @Override - public void getExtensions(Set extensions) { - } - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) { - } } // tests storing "id" and "field2" fields as pulsing codec, // whose term sort is backwards unicode code point, and // storing "field1" as a custom entirely-in-RAM codec public void testPerFieldCodec() throws Exception { - CodecProvider provider = new CoreCodecProvider(); - provider.register(new RAMOnlyCodec()); - provider.setDefaultFieldCodec("RamOnly"); final int NUM_DOCS = atLeast(173); MockDirectoryWrapper dir = newDirectory(); @@ -530,7 +65,7 @@ IndexWriter w = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setCodecProvider(provider). + setCodec(new CustomPerFieldCodec()). setMergePolicy(newLogMergePolicy(3)) ); w.setInfoStream(VERBOSE ? System.out : null); @@ -539,11 +74,9 @@ doc.add(newField("field1", "this field uses the standard codec as the test", TextField.TYPE_UNSTORED)); // uses pulsing codec: Field field2 = newField("field2", "this field uses the pulsing codec as the test", TextField.TYPE_UNSTORED); - provider.setFieldCodec(field2.name(), "Pulsing"); doc.add(field2); Field idField = newField("id", "", StringField.TYPE_UNSTORED); - provider.setFieldCodec(idField.name(), "Pulsing"); doc.add(idField); for(int i=0;i filesToDelete = merger.createCompoundFile(merged + ".cfs", info, newIOContext(random)); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterCommit.java (working copy) @@ -28,7 +28,6 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -169,8 +168,10 @@ // them, the merged result can easily be larger than the // sum because the merged FST may use array encoding for // some arcs (which uses more space): - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory")); - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory")); + + final String idFormat = _TestUtil.getPostingsFormat("id"); + final String contentFormat = _TestUtil.getPostingsFormat("content"); + assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory")); MockDirectoryWrapper dir = newDirectory(); Analyzer analyzer; if (random.nextBoolean()) { Index: lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestSegmentTermEnum.java (working copy) @@ -26,6 +26,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.store.Directory; @@ -74,7 +75,7 @@ public void testPrevTermAtEnd() throws IOException { - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); addDoc(writer, "aaa bbb"); writer.close(); SegmentReader reader = getOnlySegmentReader(IndexReader.open(dir, false)); Index: lucene/src/test/org/apache/lucene/index/Test2BPostings.java =================================================================== --- lucene/src/test/org/apache/lucene/index/Test2BPostings.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/Test2BPostings.java (working copy) @@ -27,23 +27,20 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; +import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec; /** * Test indexes ~82M docs with 26 terms each, so you get > Integer.MAX_VALUE terms/docs pairs * @lucene.experimental */ +@UseNoMemoryExpensiveCodec public class Test2BPostings extends LuceneTestCase { @Nightly public void test() throws Exception { - - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory")); - assumeFalse("This test is super-slow and very disk-space-consuming with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText")); - MockDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BPostings")); dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER); dir.setCheckIndexOnClose(false); // don't double-checkindex Index: lucene/src/test/org/apache/lucene/index/TestFlex.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestFlex.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestFlex.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.store.*; import org.apache.lucene.analysis.*; import org.apache.lucene.document.*; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.util.*; public class TestFlex extends LuceneTestCase { @@ -64,7 +65,7 @@ public void testTermOrd() throws Exception { Directory d = newDirectory(); IndexWriter w = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); Document doc = new Document(); doc.add(newField("f", "a b c", TextField.TYPE_UNSTORED)); w.addDocument(doc); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterOnDiskFull.java (working copy) @@ -24,7 +24,7 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -154,9 +154,11 @@ // them, the merged result can easily be larger than the // sum because the merged FST may use array encoding for // some arcs (which uses more space): - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("id").equals("Memory")); - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("content").equals("Memory")); + final String idFormat = _TestUtil.getPostingsFormat("id"); + final String contentFormat = _TestUtil.getPostingsFormat("content"); + assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory")); + int START_COUNT = 57; int NUM_DIR = TEST_NIGHTLY ? 50 : 5; int END_COUNT = START_COUNT + NUM_DIR* (TEST_NIGHTLY ? 25 : 5); Index: lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestGlobalFieldNumbers.java (working copy) @@ -58,25 +58,25 @@ } writer.commit(); Collection files = writer.getIndexFileNames(); - files.remove("1.fnx"); + files.remove("_1.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); writer.commit(); files = writer.getIndexFileNames(); - files.remove("2.fnx"); + files.remove("_2.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } { @@ -89,12 +89,12 @@ writer.addDocument(d); writer.close(); Collection files = writer.getIndexFileNames(); - files.remove("2.fnx"); + files.remove("_2.fnx"); for (String string : files) { assertFalse(string.endsWith(".fnx")); } - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( @@ -102,7 +102,7 @@ writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); dir.close(); } @@ -121,29 +121,29 @@ d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } IndexReader reader = IndexReader.open(dir, false); reader.deleteDocument(0); reader.commit(); reader.close(); // make sure this reader can not modify the field map - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); dir.close(); } @@ -162,7 +162,7 @@ d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); @@ -170,9 +170,9 @@ writer.commit(); writer.commit(); writer.commit(); - assertFNXFiles(dir, "1.fnx", "2.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx"); writer.close(); - assertFNXFiles(dir, "1.fnx", "2.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx"); } { @@ -184,14 +184,14 @@ d.add(new BinaryField("f3", new byte[] { 1, 2, 3, 4, 5 })); writer.addDocument(d); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); } IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.optimize(); assertFalse(" field numbers got mixed up", writer.anyNonBulkMerges); writer.close(); - assertFNXFiles(dir, "2.fnx"); + assertFNXFiles(dir, "_2.fnx"); dir.close(); } } @@ -208,14 +208,14 @@ d.add(new Field("f2", "d1 second field", TextField.TYPE_STORED)); writer.addDocument(d); writer.commit(); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); d = new Document(); d.add(new Field("f1", "d2 first field", TextField.TYPE_STORED)); d.add(new BinaryField("f3", new byte[] { 1, 2, 3 })); writer.addDocument(d); - assertFNXFiles(dir, "1.fnx"); + assertFNXFiles(dir, "_1.fnx"); writer.close(); - assertFNXFiles(dir, "1.fnx", "2.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx"); // open first commit List listCommits = IndexReader.listCommits(dir); assertEquals(2, listCommits.size()); @@ -229,18 +229,18 @@ writer.addDocument(d); writer.commit(); // now we have 3 files since f3 is not present in the first commit - assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx", "_3.fnx"); writer.close(); - assertFNXFiles(dir, "1.fnx", "2.fnx", "3.fnx"); + assertFNXFiles(dir, "_1.fnx", "_2.fnx", "_3.fnx"); writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); writer.commit(); listCommits = IndexReader.listCommits(dir); assertEquals(1, listCommits.size()); - assertFNXFiles(dir, "3.fnx"); + assertFNXFiles(dir, "_3.fnx"); writer.close(); - assertFNXFiles(dir, "3.fnx"); + assertFNXFiles(dir, "_3.fnx"); dir.close(); } } @@ -494,7 +494,7 @@ assertEquals(1, segmentInfos.getGlobalFieldMapVersion()); assertEquals(1, segmentInfos.getLastGlobalFieldMapVersion()); files = writer.getIndexFileNames(); - assertTrue(files.remove("1.fnx")); + assertTrue(files.remove("_1.fnx")); for (String string : files) { assertFalse(string.endsWith(".fnx")); } Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (working copy) @@ -23,6 +23,7 @@ import java.io.Reader; import java.io.StringReader; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Random; @@ -927,10 +928,10 @@ } catch (RuntimeException re) { // Expected } - assertTrue(dir.fileExists("1.fnx")); + assertTrue(dir.fileExists("_1.fnx")); assertTrue(failure.failOnCommit && failure.failOnDeleteFile); w.rollback(); - assertFalse(dir.fileExists("1.fnx")); + assertFalse(dir.fileExists("_1.fnx")); assertEquals(0, dir.listAll().length); dir.close(); } Index: lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java =================================================================== --- lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/values/TestTypePromotion.java (working copy) @@ -15,7 +15,7 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.NoMergePolicy; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; @@ -42,8 +42,7 @@ @Before public void setUp() throws Exception { super.setUp(); - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault() - .getDefaultFieldCodec().equals("PreFlex")); + assumeFalse("cannot work with preflex codec", Codec.getDefault().getName().equals("Lucene3x")); } private static EnumSet INTEGERS = EnumSet.of(ValueType.VAR_INTS, Index: lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java =================================================================== --- lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/values/TestDocValuesIndexing.java (working copy) @@ -39,7 +39,7 @@ import org.apache.lucene.index.MultiPerDocValues; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.search.*; @@ -65,7 +65,7 @@ @Before public void setUp() throws Exception { super.setUp(); - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); + assumeFalse("cannot work with preflex codec", Codec.getDefault().getName().equals("Lucene3x")); } /* @@ -140,7 +140,7 @@ indexValues(w_1, valuesPerIndex, first, values, false, 7); w_1.commit(); assertEquals(valuesPerIndex, w_1.maxDoc()); - _TestUtil.checkIndex(d_1, w_1.getConfig().getCodecProvider()); + _TestUtil.checkIndex(d_1); // index second index Directory d_2 = newDirectory(); @@ -148,7 +148,7 @@ indexValues(w_2, valuesPerIndex, second, values, false, 7); w_2.commit(); assertEquals(valuesPerIndex, w_2.maxDoc()); - _TestUtil.checkIndex(d_2, w_2.getConfig().getCodecProvider()); + _TestUtil.checkIndex(d_2); Directory target = newDirectory(); IndexWriter w = new IndexWriter(target, writerConfig(random.nextBoolean())); @@ -162,7 +162,7 @@ w.optimize(true); w.commit(); - _TestUtil.checkIndex(target, w.getConfig().getCodecProvider()); + _TestUtil.checkIndex(target); assertEquals(valuesPerIndex * 2, w.maxDoc()); // check values @@ -546,7 +546,6 @@ } public void testMultiValuedIndexDocValuesField() throws Exception { - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); Directory d = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random, d); Document doc = new Document(); @@ -575,7 +574,6 @@ } public void testDifferentTypedDocValuesField() throws Exception { - assumeFalse("cannot work with preflex codec", CodecProvider.getDefault().getDefaultFieldCodec().equals("PreFlex")); Directory d = newDirectory(); RandomIndexWriter w = new RandomIndexWriter(random, d); Document doc = new Document(); Index: lucene/src/test/org/apache/lucene/index/TestCompoundFile.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestCompoundFile.java (working copy) @@ -637,9 +637,9 @@ CompoundFileDirectory csw = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random), true); int size = 5 + random.nextInt(128); for (int j = 0; j < 2; j++) { - IndexOutput os = csw.createOutput("seg" + j + "_foo.txt", newIOContext(random)); + IndexOutput os = csw.createOutput("seg_" + j + "_foo.txt", newIOContext(random)); for (int i = 0; i < size; i++) { - os.writeInt(i); + os.writeInt(i*j); } os.close(); String[] listAll = newDir.listAll(); @@ -654,10 +654,10 @@ csw.close(); CompoundFileDirectory csr = new CompoundFileDirectory(newDir, "d.cfs", newIOContext(random), false); for (int j = 0; j < 2; j++) { - IndexInput openInput = csr.openInput("seg" + j + "_foo.txt", newIOContext(random)); + IndexInput openInput = csr.openInput("seg_" + j + "_foo.txt", newIOContext(random)); assertEquals(size * 4, openInput.length()); for (int i = 0; i < size; i++) { - assertEquals(i, openInput.readInt()); + assertEquals(i*j, openInput.readInt()); } openInput.close(); Index: lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java (working copy) @@ -108,7 +108,7 @@ if (reader instanceof MultiReader) // MultiReader does not "own" the directory so it does // not write the changes to sis on commit: - sis.commit(dir); + sis.commit(dir, sis.codecFormat()); sis.read(dir); reader = openReader(); @@ -121,7 +121,7 @@ if (reader instanceof MultiReader) // MultiReader does not "own" the directory so it does // not write the changes to sis on commit: - sis.commit(dir); + sis.commit(dir, sis.codecFormat()); sis.read(dir); reader = openReader(); assertEquals( 1, reader.numDocs() ); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java (working copy) @@ -30,6 +30,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; @@ -988,15 +989,16 @@ public void testNoTermsIndex() throws Exception { // Some Codecs don't honor the ReaderTermsIndexDivisor, so skip the test if // they're picked. - HashSet illegalCodecs = new HashSet(); - illegalCodecs.add("PreFlex"); - illegalCodecs.add("SimpleText"); - illegalCodecs.add("Memory"); + assumeFalse("PreFlex codec does not support ReaderTermsIndexDivisor!", + "Lucene3x".equals(Codec.getDefault().getName())); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setReaderTermsIndexDivisor(-1); + // Don't proceed if picked Codec is in the list of illegal ones. - if (illegalCodecs.contains(conf.getCodecProvider().getFieldCodec("f"))) return; + final String format = _TestUtil.getPostingsFormat("f"); + assumeFalse("Format: " + format + " does not support ReaderTermsIndexDivisor!", + (format.equals("SimpleText") || format.equals("Memory"))); Directory dir = newDirectory(); IndexWriter w = new IndexWriter(dir, conf); @@ -1006,7 +1008,7 @@ IndexReader r = IndexReader.open(w, true).getSequentialSubReaders()[0]; try { r.termDocsEnum(null, "f", new BytesRef("val")); - fail("should have failed to seek since terms index was not loaded. Codec used " + conf.getCodecProvider().getFieldCodec("f")); + fail("should have failed to seek since terms index was not loaded."); } catch (IllegalStateException e) { // expected - we didn't load the term index } finally { Index: lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestRollingUpdates.java (working copy) @@ -19,7 +19,8 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.*; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.memory.MemoryPostingsFormat; import org.apache.lucene.store.*; import org.apache.lucene.util.*; import org.junit.Test; @@ -35,13 +36,12 @@ dir.setCheckIndexOnClose(false); // we use a custom codec provider final LineFileDocs docs = new LineFileDocs(random); - CodecProvider provider = CodecProvider.getDefault(); //provider.register(new MemoryCodec()); - if ( (!"PreFlex".equals(provider.getDefaultFieldCodec())) && random.nextBoolean()) { - provider.setFieldCodec("docid", "Memory"); + if ( (!"Lucene3x".equals(Codec.getDefault().getName())) && random.nextBoolean()) { + Codec.setDefault(_TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat())); } - final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(provider)); + final IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))); w.setInfoStream(VERBOSE ? System.out : null); final int SIZE = atLeast(TEST_NIGHTLY ? 100 : 20); int id = 0; Index: lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestBinaryTerms.java (working copy) @@ -23,7 +23,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TopDocs; @@ -37,7 +37,7 @@ public class TestBinaryTerms extends LuceneTestCase { public void testBinary() throws IOException { assumeFalse("PreFlex codec cannot work with binary terms!", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + Codec.getDefault().getName().equals("Lucene3x")); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir); Index: lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java (working copy) @@ -23,6 +23,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.util.BytesRef; import java.io.IOException; @@ -74,7 +75,7 @@ } public void testMerge() throws IOException { - SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, null, new FieldInfos(), newIOContext(random)); + SegmentMerger merger = new SegmentMerger(mergedDir, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL, mergedSegment, null, null, new FieldInfos(), Codec.getDefault(), newIOContext(random)); merger.add(reader1); merger.add(reader2); int docsMerged = merger.merge(); @@ -82,7 +83,7 @@ final FieldInfos fieldInfos = merger.fieldInfos(); //Should be able to open a new SegmentReader against the new directory SegmentReader mergedReader = SegmentReader.get(false, mergedDir, new SegmentInfo(mergedSegment, docsMerged, mergedDir, false, - merger.getSegmentCodecs(), fieldInfos), + merger.getCodec(), fieldInfos), true, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, newIOContext(random)); assertTrue(mergedReader != null); assertTrue(mergedReader.numDocs() == 2); @@ -145,7 +146,7 @@ w.close(); // Assert that SM fails if .del exists - SegmentMerger sm = new SegmentMerger(dir, 1, "a", null, null, null, newIOContext(random)); + SegmentMerger sm = new SegmentMerger(dir, 1, "a", null, null, null, Codec.getDefault(), newIOContext(random)); boolean doFail = false; try { sm.createCompoundFile("b1", w.segmentInfos.info(0), newIOContext(random)); Index: lucene/src/test/org/apache/lucene/index/TestAddIndexes.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (working copy) @@ -17,9 +17,10 @@ * limitations under the License. */ +import java.io.FileNotFoundException; import java.io.IOException; -import java.io.FileNotFoundException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import org.apache.lucene.analysis.MockAnalyzer; @@ -29,11 +30,22 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexWriterConfig.OpenMode; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.mocksep.MockSepCodec; -import org.apache.lucene.index.codecs.pulsing.PulsingCodec; -import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec; -import org.apache.lucene.index.codecs.standard.StandardCodec; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DefaultDocValuesFormat; +import org.apache.lucene.index.codecs.DefaultFieldsFormat; +import org.apache.lucene.index.codecs.DefaultSegmentInfosFormat; +import org.apache.lucene.index.codecs.DocValuesFormat; +import org.apache.lucene.index.codecs.FieldsFormat; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.SegmentInfosFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40Codec; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsBaseFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; +import org.apache.lucene.index.codecs.perfield.PerFieldPostingsFormat; +import org.apache.lucene.index.codecs.pulsing.Pulsing40PostingsFormat; +import org.apache.lucene.index.codecs.pulsing.PulsingPostingsFormat; +import org.apache.lucene.index.codecs.simpletext.SimpleTextPostingsFormat; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.store.AlreadyClosedException; @@ -975,30 +987,29 @@ } } - public void testSimpleCaseCustomCodecProvider() throws IOException { + public void testSimpleCaseCustomCodec() throws IOException { // main directory Directory dir = newDirectory(); // two auxiliary directories Directory aux = newDirectory(); Directory aux2 = newDirectory(); - CodecProvider provider = new MockCodecProvider(); + Codec codec = new CustomPerFieldCodec(); IndexWriter writer = null; writer = newWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodecProvider( - provider)); + new MockAnalyzer(random)).setOpenMode(OpenMode.CREATE).setCodec(codec)); // add 100 documents addDocs3(writer, 100); assertEquals(100, writer.maxDoc()); writer.commit(); writer.close(); - _TestUtil.checkIndex(dir, provider); + _TestUtil.checkIndex(dir); writer = newWriter( aux, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.CREATE). - setCodecProvider(provider). + setCodec(codec). setMaxBufferedDocs(10). setMergePolicy(newLogMergePolicy(false)) ); @@ -1012,7 +1023,7 @@ aux2, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.CREATE). - setCodecProvider(provider) + setCodec(codec) ); // add 40 documents in compound files addDocs2(writer, 50); @@ -1025,7 +1036,7 @@ dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). setOpenMode(OpenMode.APPEND). - setCodecProvider(provider) + setCodec(codec) ); assertEquals(100, writer.maxDoc()); writer.addIndexes(aux, aux2); @@ -1037,19 +1048,24 @@ aux2.close(); } - public static class MockCodecProvider extends CodecProvider { - public MockCodecProvider() { - StandardCodec standardCodec = new StandardCodec(); - SimpleTextCodec simpleTextCodec = new SimpleTextCodec(); - MockSepCodec mockSepCodec = new MockSepCodec(); - register(standardCodec); - register(mockSepCodec); - register(simpleTextCodec); - setFieldCodec("id", simpleTextCodec.name); - setFieldCodec("content", mockSepCodec.name); + private static final class CustomPerFieldCodec extends Lucene40Codec { + private final PostingsFormat simpleTextFormat = PostingsFormat.forName("SimpleText"); + private final PostingsFormat defaultFormat = PostingsFormat.forName("Lucene40"); + private final PostingsFormat mockSepFormat = PostingsFormat.forName("MockSep"); + + @Override + public PostingsFormat getPostingsFormatForField(String field) { + if (field.equals("id")) { + return simpleTextFormat; + } else if (field.equals("content")) { + return mockSepFormat; + } else { + return defaultFormat; + } } } + // LUCENE-2790: tests that the non CFS files were deleted by addIndexes public void testNonCFSLeftovers() throws Exception { Directory[] dirs = new Directory[2]; @@ -1066,16 +1082,19 @@ IndexReader[] readers = new IndexReader[] { IndexReader.open(dirs[0]), IndexReader.open(dirs[1]) }; - Directory dir = new RAMDirectory(); + Directory dir = new MockDirectoryWrapper(random, new RAMDirectory()); IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()); LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy(); lmp.setUseCompoundFile(true); lmp.setNoCFSRatio(1.0); // Force creation of CFS IndexWriter w3 = new IndexWriter(dir, conf); + w3.setInfoStream(VERBOSE ? System.out : null); w3.addIndexes(readers); w3.close(); - // we should now see segments_X, segments.gen,_Y.cfs,_Y.cfe, _Z.fnx - assertEquals("Only one compound segment should exist", 5, dir.listAll().length); + // we should now see segments_X, + // segments.gen,_Y.cfs,_Y.cfe, _Z.fnx + assertEquals("Only one compound segment should exist, but got: " + Arrays.toString(dir.listAll()), 5, dir.listAll().length); + dir.close(); } // LUCENE-3126: tests that if a non-CFS segment is copied, it is converted to @@ -1136,18 +1155,45 @@ src.close(); target.close(); } + + private static class UnRegisteredCodec extends Codec { + public UnRegisteredCodec() { + super("NotRegistered"); + } + + @Override + public PostingsFormat postingsFormat() { + return PostingsFormat.forName("Lucene40"); + } + + @Override + public DocValuesFormat docValuesFormat() { + return new DefaultDocValuesFormat(); + } + + @Override + public FieldsFormat fieldsFormat() { + return new DefaultFieldsFormat(); + } + + @Override + public SegmentInfosFormat segmentInfosFormat() { + return new DefaultSegmentInfosFormat(); + } + } /* * simple test that ensures we getting expected exceptions */ public void testAddIndexMissingCodec() throws IOException { - Directory toAdd = newDirectory(); + MockDirectoryWrapper toAdd = newDirectory(); + // Disable checkIndex, else we get an exception because + // of the unregistered codec: + toAdd.setCheckIndexOnClose(false); { IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); - CodecProvider provider = new CodecProvider(); - provider.register(new StandardCodec()); - conf.setCodecProvider(provider); + conf.setCodec(new UnRegisteredCodec()); IndexWriter w = new IndexWriter(toAdd, conf); Document doc = new Document(); FieldType customType = new FieldType(); @@ -1156,13 +1202,12 @@ w.addDocument(doc); w.close(); } + { Directory dir = newDirectory(); IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); - CodecProvider provider = new CodecProvider(); - provider.register(new PulsingCodec(1 + random.nextInt(20))); - conf.setCodecProvider(provider); + conf.setCodec(_TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1 + random.nextInt(20)))); IndexWriter w = new IndexWriter(dir, conf); try { w.addIndexes(toAdd); @@ -1177,27 +1222,11 @@ dir.close(); } - { - Directory dir = newDirectory(); - IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, - new MockAnalyzer(random)); - CodecProvider provider = new CodecProvider(); - provider.register(new PulsingCodec(1 + random.nextInt(20))); - conf.setCodecProvider(provider); - IndexWriter w = new IndexWriter(dir, conf); + try { IndexReader indexReader = IndexReader.open(toAdd); - try { - w.addIndexes(indexReader); - fail("no such codec"); - } catch (IllegalArgumentException ex) { - // expected - } - indexReader.close(); - w.close(); - IndexReader open = IndexReader.open(dir); - assertEquals(0, open.numDocs()); - open.close(); - dir.close(); + fail("no such codec"); + } catch (IllegalArgumentException ex) { + // expected } toAdd.close(); } Index: lucene/src/test/org/apache/lucene/index/TestTermsEnum.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestTermsEnum.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestTermsEnum.java (working copy) @@ -358,11 +358,7 @@ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)); /* - CoreCodecProvider cp = new CoreCodecProvider(); - cp.unregister(cp.lookup("Standard")); - cp.register(new StandardCodec(minTermsInBlock, maxTermsInBlock)); - cp.setDefaultFieldCodec("Standard"); - iwc.setCodecProvider(cp); + iwc.setCodec(new StandardCodec(minTermsInBlock, maxTermsInBlock)); */ final RandomIndexWriter w = new RandomIndexWriter(random, d, iwc); Index: lucene/src/test/org/apache/lucene/index/TestLongPostings.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestLongPostings.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestLongPostings.java (working copy) @@ -29,7 +29,6 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; Index: lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (working copy) @@ -568,7 +568,7 @@ "_0_1.s" + contentFieldIndex, "segments_2", "segments.gen", - "1.fnx"}; + "_1.fnx"}; String[] actual = dir.listAll(); Arrays.sort(expected); Index: lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestLazyProxSkipping.java (working copy) @@ -23,7 +23,6 @@ import org.apache.lucene.analysis.*; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; import org.apache.lucene.search.ScoreDoc; @@ -34,6 +33,7 @@ import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util._TestUtil; /** * Tests lazy skipping on the proximity file. @@ -131,8 +131,10 @@ } public void testLazySkipping() throws IOException { - assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("SimpleText")); - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec(this.field).equals("Memory")); + final String fieldFormat = _TestUtil.getPostingsFormat(this.field); + assumeFalse("This test cannot run with Memory codec", fieldFormat.equals("Memory")); + assumeFalse("This test cannot run with SimpleText codec", fieldFormat.equals("SimpleText")); + // test whether only the minimum amount of seeks() // are performed performTest(5); Index: lucene/src/test/org/apache/lucene/index/TestDocCount.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDocCount.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestDocCount.java (working copy) @@ -19,7 +19,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.StringField; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.Directory; import org.apache.lucene.util.FixedBitSet; @@ -32,7 +32,7 @@ public class TestDocCount extends LuceneTestCase { public void testSimple() throws Exception { assumeFalse("PreFlex codec does not support docCount statistic!", - "PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())); + "Lucene3x".equals(Codec.getDefault().getName())); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir); int numDocs = atLeast(100); Index: lucene/src/test/org/apache/lucene/index/TestIndexReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexReader.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestIndexReader.java (working copy) @@ -39,7 +39,7 @@ import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader.FieldOption; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.FieldCache; @@ -88,6 +88,7 @@ writer = new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode( OpenMode.APPEND).setMaxBufferedDocs(2)); + writer.setInfoStream(VERBOSE ? System.out : null); for(int i=0;i<7;i++) addDocumentWithFields(writer); writer.close(); @@ -947,7 +948,7 @@ writer.close(); SegmentInfos sis = new SegmentInfos(); - sis.read(d, CodecProvider.getDefault()); + sis.read(d); IndexReader r = IndexReader.open(d, false); IndexCommit c = r.getIndexCommit(); @@ -1231,7 +1232,7 @@ // LUCENE-1609: don't load terms index public void testNoTermsIndex() throws Throwable { Directory dir = newDirectory(); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(_TestUtil.alwaysCodec("Standard"))); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat()))); Document doc = new Document(); doc.add(newField("field", "a b c d e f g h i j k l m n o p q r s t u v w x y z", TextField.TYPE_UNSTORED)); doc.add(newField("number", "0 1 2 3 4 5 6 7 8 9", TextField.TYPE_UNSTORED)); @@ -1251,7 +1252,7 @@ writer = new IndexWriter( dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)). - setCodecProvider(_TestUtil.alwaysCodec("Standard")). + setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())). setMergePolicy(newLogMergePolicy(10)) ); writer.addDocument(doc); Index: lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestForTooMuchCloning.java (working copy) @@ -22,7 +22,6 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.TopDocs; @@ -36,6 +35,9 @@ // Make sure we don't clone IndexInputs too frequently // during merging: public void test() throws Exception { + // NOTE: if we see a fail on this test with "NestedPulsing" its because its + // reuse isnt perfect (but reasonable). see TestPulsingReuse.testNestedPulsing + // for more details final MockDirectoryWrapper dir = newDirectory(); final TieredMergePolicy tmp = new TieredMergePolicy(); tmp.setMaxMergeAtOnce(2); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -52,12 +52,17 @@ import org.apache.lucene.search.spans.SpanTermQuery; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.Directory; +import org.apache.lucene.store.FSDirectory; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockFactory; +import org.apache.lucene.store.LockObtainFailedException; import org.apache.lucene.store.MockDirectoryWrapper; +import org.apache.lucene.store.NativeFSLockFactory; import org.apache.lucene.store.NoLockFactory; import org.apache.lucene.store.RAMDirectory; +import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.store.SimpleFSLockFactory; import org.apache.lucene.store.SingleInstanceLockFactory; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase; @@ -1937,4 +1942,22 @@ assert(version3 > version2); d.close(); } + + public void testWhetherDeleteAllDeletesWriteLock() throws Exception { + Directory d = newFSDirectory(_TestUtil.getTempDir("TestIndexWriter.testWhetherDeleteAllDeletesWriteLock")); + // Must use SimpleFSLockFactory... NativeFSLockFactory + // somehow "knows" a lock is held against write.lock + // even if you remove that file: + d.setLockFactory(new SimpleFSLockFactory()); + RandomIndexWriter w1 = new RandomIndexWriter(random, d); + w1.deleteAll(); + try { + new RandomIndexWriter(random, d); + fail("should not be able to create another writer"); + } catch (LockObtainFailedException lofe) { + // expected + } + w1.close(); + d.close(); + } } Index: lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (working copy) @@ -25,6 +25,7 @@ import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -68,7 +69,7 @@ public void testSimpleSkip() throws IOException { Directory dir = new CountingRAMDirectory(new RAMDirectory()); - IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new PayloadAnalyzer()).setCodecProvider(_TestUtil.alwaysCodec("Standard")).setMergePolicy(newLogMergePolicy())); + IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new PayloadAnalyzer()).setCodec(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())).setMergePolicy(newLogMergePolicy())); Term term = new Term("test", "a"); for (int i = 0; i < 5000; i++) { Document d1 = new Document(); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (working copy) @@ -26,6 +26,7 @@ import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.similarities.DefaultSimilarityProvider; import org.apache.lucene.util.LuceneTestCase; @@ -71,6 +72,7 @@ assertEquals(ThreadAffinityDocumentsWriterThreadPool.class, conf.getIndexerThreadPool().getClass()); assertEquals(FlushByRamOrCountsPolicy.class, conf.getFlushPolicy().getClass()); assertEquals(IndexWriterConfig.DEFAULT_RAM_PER_THREAD_HARD_LIMIT_MB, conf.getRAMPerThreadHardLimitMB()); + assertEquals(Codec.getDefault(), conf.getCodec()); // Sanity check - validate that all getters are covered. Set getters = new HashSet(); getters.add("getAnalyzer"); @@ -88,7 +90,6 @@ getters.add("getMaxBufferedDocs"); getters.add("getIndexingChain"); getters.add("getMergedSegmentWarmer"); - getters.add("getCodecProvider"); getters.add("getMergePolicy"); getters.add("getMaxThreadStates"); getters.add("getReaderPooling"); @@ -96,6 +97,7 @@ getters.add("getReaderTermsIndexDivisor"); getters.add("getFlushPolicy"); getters.add("getRAMPerThreadHardLimitMB"); + getters.add("getCodec"); for (Method m : IndexWriterConfig.class.getDeclaredMethods()) { if (m.getDeclaringClass() == IndexWriterConfig.class && m.getName().startsWith("get")) { Index: lucene/src/test/org/apache/lucene/index/TestCodecs.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestCodecs.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestCodecs.java (working copy) @@ -26,14 +26,15 @@ import org.apache.lucene.index.FieldInfo.IndexOptions; import org.apache.lucene.document.FieldType; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldsConsumer; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.PostingsConsumer; import org.apache.lucene.index.codecs.TermStats; import org.apache.lucene.index.codecs.TermsConsumer; -import org.apache.lucene.index.codecs.mocksep.MockSepCodec; -import org.apache.lucene.index.codecs.preflex.PreFlexCodec; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xCodec; +import org.apache.lucene.index.codecs.lucene3x.Lucene3xPostingsFormat; +import org.apache.lucene.index.codecs.mocksep.MockSepPostingsFormat; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PhraseQuery; @@ -64,6 +65,7 @@ // goes to 1 before next one known to exist // - skipTo(term) // - skipTo(doc) + public class TestCodecs extends LuceneTestCase { private static String[] fieldNames = new String[] {"one", "two", "three", "four"}; @@ -255,9 +257,10 @@ final Directory dir = newDirectory(); FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone(); this.write(fieldInfos, dir, fields, true); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, clonedFieldInfos.buildSegmentCodecs(false), clonedFieldInfos); + Codec codec = Codec.getDefault(); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, codec, clonedFieldInfos); - final FieldsProducer reader = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); + final FieldsProducer reader = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); final FieldsEnum fieldsEnum = reader.iterator(); assertNotNull(fieldsEnum.next()); @@ -307,12 +310,13 @@ FieldInfos clonedFieldInfos = (FieldInfos) fieldInfos.clone(); this.write(fieldInfos, dir, fields, false); - final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, clonedFieldInfos.buildSegmentCodecs(false), clonedFieldInfos); + Codec codec = Codec.getDefault(); + final SegmentInfo si = new SegmentInfo(SEGMENT, 10000, dir, false, codec, clonedFieldInfos); if (VERBOSE) { System.out.println("TEST: now read postings"); } - final FieldsProducer terms = si.getSegmentCodecs().codec().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); + final FieldsProducer terms = codec.postingsFormat().fieldsProducer(new SegmentReadState(dir, si, fieldInfos, newIOContext(random), IndexReader.DEFAULT_TERMS_INDEX_DIVISOR)); final Verify[] threads = new Verify[NUM_TEST_THREADS-1]; for(int i=0;i files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, id, files); - BlockTermsReader.files(dir, segmentInfo, id, files); - FixedGapTermsIndexReader.files(dir, segmentInfo, id, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); - } - - @Override - public void getExtensions(Set extensions) { - getStandardExtensions(extensions); - DefaultDocValuesConsumer.getExtensions(extensions); - } - - public static void getStandardExtensions(Set extensions) { - extensions.add(FREQ_EXTENSION); - extensions.add(PROX_EXTENSION); - BlockTermsReader.getExtensions(extensions); - FixedGapTermsIndexReader.getIndexExtensions(extensions); - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); - } - } - public void testRandom() throws Exception { MockDirectoryWrapper dir = newDirectory(); @@ -252,13 +124,8 @@ // Sometimes swap in codec that impls ord(): if (random.nextInt(10) == 7) { // Make sure terms index has ords: - CoreCodecProvider cp = new CoreCodecProvider(); - cp.register(new StandardCodecWithOrds()); - cp.setDefaultFieldCodec("StandardOrds"); - - // So checkIndex on close works - dir.setCodecProvider(cp); - conf.setCodecProvider(cp); + Codec codec = _TestUtil.alwaysPostingsFormat(PostingsFormat.forName("Lucene40WithOrds")); + conf.setCodec(codec); } final RandomIndexWriter w = new RandomIndexWriter(random, dir, conf); @@ -354,14 +221,8 @@ // Sometimes swap in codec that impls ord(): if (random.nextInt(10) == 7) { - // Make sure terms index has ords: - CoreCodecProvider cp = new CoreCodecProvider(); - cp.register(new StandardCodecWithOrds()); - cp.setDefaultFieldCodec("StandardOrds"); - - // So checkIndex on close works - dir.setCodecProvider(cp); - conf.setCodecProvider(cp); + Codec codec = _TestUtil.alwaysPostingsFormat(PostingsFormat.forName("Lucene40WithOrds")); + conf.setCodec(codec); } final RandomIndexWriter w = new RandomIndexWriter(random, dir, conf); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java (working copy) @@ -31,7 +31,6 @@ import org.apache.lucene.document.FieldType; import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; @@ -897,8 +896,9 @@ } public void testIndexingThenDeleting() throws Exception { - assumeFalse("This test cannot run with Memory codec", CodecProvider.getDefault().getFieldCodec("field").equals("Memory")); - assumeFalse("This test cannot run with SimpleText codec", CodecProvider.getDefault().getFieldCodec("field").equals("SimpleText")); + final String fieldFormat = _TestUtil.getPostingsFormat("field"); + assumeFalse("This test cannot run with Memory codec", fieldFormat.equals("Memory")); + assumeFalse("This test cannot run with SimpleText codec", fieldFormat.equals("SimpleText")); final Random r = random; Directory dir = newDirectory(); // note this test explicitly disables payloads Index: lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestTermsEnum2.java (working copy) @@ -29,7 +29,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; import org.apache.lucene.index.TermsEnum.SeekStatus; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.AutomatonQuery; import org.apache.lucene.search.CheckHits; import org.apache.lucene.search.IndexSearcher; @@ -57,7 +57,7 @@ super.setUp(); // we generate aweful regexps: good for testing. // but for preflex codec, the test can be very slow, so use less iterations. - numIterations = CodecProvider.getDefault().getFieldCodec("field").equals("PreFlex") ? 10 * RANDOM_MULTIPLIER : atLeast(50); + numIterations = Codec.getDefault().getName().equals("Lucene3x") ? 10 * RANDOM_MULTIPLIER : atLeast(50); dir = newDirectory(); RandomIndexWriter writer = new RandomIndexWriter(random, dir, newIndexWriterConfig(TEST_VERSION_CURRENT, Index: lucene/src/test/org/apache/lucene/index/Test2BTerms.java =================================================================== --- lucene/src/test/org/apache/lucene/index/Test2BTerms.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/Test2BTerms.java (working copy) @@ -24,7 +24,8 @@ import org.apache.lucene.analysis.tokenattributes.*; import org.apache.lucene.document.*; import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; + import java.io.IOException; import java.util.ArrayList; import java.util.Collections; @@ -142,7 +143,7 @@ @Ignore("Takes ~4 hours to run on a fast machine!! And requires that you don't use PreFlex codec.") public void test2BTerms() throws IOException { - if ("PreFlex".equals(CodecProvider.getDefault().getDefaultFieldCodec())) { + if ("Lucene3x".equals(Codec.getDefault().getName())) { throw new RuntimeException("thist test cannot run with PreFlex codec"); } Index: lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java =================================================================== --- lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java (working copy) @@ -35,7 +35,8 @@ import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.PostingsFormat; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsBaseFormat; import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; @@ -51,13 +52,13 @@ public class Test10KPulsings extends LuceneTestCase { public void test10kPulsed() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); + Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1)); File f = _TestUtil.getTempDir("10kpulsed"); MockDirectoryWrapper dir = newFSDirectory(f); dir.setCheckIndexOnClose(false); // we do this ourselves explicitly RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document document = new Document(); FieldType ft = new FieldType(TextField.TYPE_STORED); @@ -101,13 +102,14 @@ */ public void test10kNotPulsed() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); + int freqCutoff = _TestUtil.nextInt(random, 1, 10); + Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(freqCutoff)); File f = _TestUtil.getTempDir("10knotpulsed"); MockDirectoryWrapper dir = newFSDirectory(f); dir.setCheckIndexOnClose(false); // we do this ourselves explicitly RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document document = new Document(); FieldType ft = new FieldType(TextField.TYPE_STORED); @@ -123,10 +125,7 @@ NumberFormat df = new DecimalFormat("00000", new DecimalFormatSymbols(Locale.ENGLISH)); - Codec codec = cp.lookup(cp.getFieldCodec("field")); - assertTrue(codec instanceof PulsingCodec); - PulsingCodec pulsing = (PulsingCodec) codec; - final int freq = pulsing.getFreqCutoff() + 1; + final int freq = freqCutoff + 1; for (int i = 0; i < 10050; i++) { StringBuilder sb = new StringBuilder(); Index: lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java =================================================================== --- lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java (working copy) @@ -17,10 +17,8 @@ * limitations under the License. */ -import java.io.IOException; import java.util.IdentityHashMap; import java.util.Map; -import java.util.Set; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -30,27 +28,10 @@ import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.SegmentWriteState; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.BlockTreeTermsReader; -import org.apache.lucene.index.codecs.BlockTreeTermsWriter; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; -import org.apache.lucene.index.codecs.DefaultDocValuesConsumer; -import org.apache.lucene.index.codecs.DefaultDocValuesProducer; -import org.apache.lucene.index.codecs.FieldsConsumer; -import org.apache.lucene.index.codecs.FieldsProducer; -import org.apache.lucene.index.codecs.PerDocConsumer; -import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.index.codecs.PostingsReaderBase; -import org.apache.lucene.index.codecs.PostingsWriterBase; -import org.apache.lucene.index.codecs.standard.StandardCodec; -import org.apache.lucene.index.codecs.standard.StandardPostingsReader; -import org.apache.lucene.index.codecs.standard.StandardPostingsWriter; +import org.apache.lucene.index.codecs.nestedpulsing.NestedPulsingPostingsFormat; import org.apache.lucene.store.Directory; import org.apache.lucene.store.MockDirectoryWrapper; import org.apache.lucene.util.LuceneTestCase; @@ -63,10 +44,10 @@ // TODO: this is a basic test. this thing is complicated, add more public void testSophisticatedReuse() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new PulsingCodec(1)); + Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1)); Directory dir = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document doc = new Document(); doc.add(new Field("foo", "a b b c c c d e f g g h i i j j k", TextField.TYPE_UNSTORED)); iw.addDocument(doc); @@ -101,11 +82,11 @@ /** tests reuse with Pulsing1(Pulsing2(Standard)) */ public void testNestedPulsing() throws Exception { // we always run this test with pulsing codec. - CodecProvider cp = _TestUtil.alwaysCodec(new NestedPulsing()); + Codec cp = _TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat()); MockDirectoryWrapper dir = newDirectory(); dir.setCheckIndexOnClose(false); // will do this ourselves, custom codec RandomIndexWriter iw = new RandomIndexWriter(random, dir, - newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodecProvider(cp)); + newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setCodec(cp)); Document doc = new Document(); doc.add(new Field("foo", "a b b c c c d e f g g g h i i j j k l l m m m", TextField.TYPE_UNSTORED)); // note: the reuse is imperfect, here we would have 4 enums (lost reuse when we get an enum for 'm') @@ -138,79 +119,7 @@ ir.close(); CheckIndex ci = new CheckIndex(dir); - ci.checkIndex(null, cp); + ci.checkIndex(null); dir.close(); } - - static class NestedPulsing extends Codec { - public NestedPulsing() { - super("NestedPulsing"); - } - - @Override - public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { - PostingsWriterBase docsWriter = new StandardPostingsWriter(state); - - PostingsWriterBase pulsingWriterInner = new PulsingPostingsWriter(2, docsWriter); - PostingsWriterBase pulsingWriter = new PulsingPostingsWriter(1, pulsingWriterInner); - - // Terms dict - boolean success = false; - try { - FieldsConsumer ret = new BlockTreeTermsWriter(state, pulsingWriter, - BlockTreeTermsWriter.DEFAULT_MIN_BLOCK_SIZE, BlockTreeTermsWriter.DEFAULT_MAX_BLOCK_SIZE); - success = true; - return ret; - } finally { - if (!success) { - pulsingWriter.close(); - } - } - } - - @Override - public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); - PostingsReaderBase pulsingReaderInner = new PulsingPostingsReader(docsReader); - PostingsReaderBase pulsingReader = new PulsingPostingsReader(pulsingReaderInner); - boolean success = false; - try { - FieldsProducer ret = new BlockTreeTermsReader( - state.dir, state.fieldInfos, state.segmentInfo.name, - pulsingReader, - state.context, - state.codecId, - state.termsIndexDivisor); - success = true; - return ret; - } finally { - if (!success) { - pulsingReader.close(); - } - } - } - - @Override - public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException { - return new DefaultDocValuesConsumer(state); - } - - @Override - public PerDocValues docsProducer(SegmentReadState state) throws IOException { - return new DefaultDocValuesProducer(state); - } - - @Override - public void files(Directory dir, SegmentInfo segmentInfo, int id, Set files) throws IOException { - StandardPostingsReader.files(dir, segmentInfo, id, files); - BlockTreeTermsReader.files(dir, segmentInfo, id, files); - DefaultDocValuesConsumer.files(dir, segmentInfo, id, files); - } - - @Override - public void getExtensions(Set extensions) { - StandardCodec.getStandardExtensions(extensions); - DefaultDocValuesConsumer.getExtensions(extensions); - } - } } Index: lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java =================================================================== --- lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java (working copy) @@ -27,7 +27,7 @@ public void testSimpleIntBlocks() throws Exception { Directory dir = newDirectory(); - IntStreamFactory f = new MockFixedIntBlockCodec(128).getIntFactory(); + IntStreamFactory f = new MockFixedIntBlockPostingsFormat(128).getIntFactory(); IntIndexOutput out = f.createOutput(dir, "test", newIOContext(random)); for(int i=0;i<11777;i++) { @@ -49,7 +49,7 @@ public void testEmptySimpleIntBlocks() throws Exception { Directory dir = newDirectory(); - IntStreamFactory f = new MockFixedIntBlockCodec(128).getIntFactory(); + IntStreamFactory f = new MockFixedIntBlockPostingsFormat(128).getIntFactory(); IntIndexOutput out = f.createOutput(dir, "test", newIOContext(random)); // write no ints Index: lucene/src/test/org/apache/lucene/index/codecs/preflex/TestTermInfosReaderIndex.java (deleted) =================================================================== Index: lucene/src/test/org/apache/lucene/index/codecs/preflex/TestSurrogates.java (deleted) =================================================================== Index: lucene/src/test/org/apache/lucene/index/TestPerFieldCodecSupport.java (deleted) =================================================================== Index: lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestConsistentFieldNumbers.java (working copy) @@ -178,7 +178,7 @@ FieldInfos fis1 = sis.info(0).getFieldInfos(); assertEquals("f1", fis1.fieldInfo(0).name); assertEquals("f2", fis1.fieldInfo(1).name); - assertTrue(dir.fileExists("1.fnx")); + assertTrue(dir.fileExists("_1.fnx")); } @@ -202,8 +202,8 @@ assertEquals("f1", fis2.fieldInfo(0).name); assertNull(fis2.fieldInfo(1)); assertEquals("f3", fis2.fieldInfo(2).name); - assertFalse(dir.fileExists("1.fnx")); - assertTrue(dir.fileExists("2.fnx")); + assertFalse(dir.fileExists("_1.fnx")); + assertTrue(dir.fileExists("_2.fnx")); } { @@ -231,9 +231,9 @@ assertEquals("f1", fis3.fieldInfo(0).name); assertEquals("f2", fis3.fieldInfo(1).name); assertEquals("f3", fis3.fieldInfo(2).name); - assertFalse(dir.fileExists("1.fnx")); - assertTrue(dir.fileExists("2.fnx")); - assertFalse(dir.fileExists("3.fnx")); + assertFalse(dir.fileExists("_1.fnx")); + assertTrue(dir.fileExists("_2.fnx")); + assertFalse(dir.fileExists("_3.fnx")); } { @@ -262,9 +262,9 @@ assertEquals("f1", fis1.fieldInfo(0).name); assertEquals("f2", fis1.fieldInfo(1).name); assertEquals("f3", fis1.fieldInfo(2).name); - assertFalse(dir.fileExists("1.fnx")); - assertTrue(dir.fileExists("2.fnx")); - assertFalse(dir.fileExists("3.fnx")); + assertFalse(dir.fileExists("_1.fnx")); + assertTrue(dir.fileExists("_2.fnx")); + assertFalse(dir.fileExists("_3.fnx")); dir.close(); } } Index: lucene/src/test/org/apache/lucene/index/TestReaderClosed.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestReaderClosed.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/index/TestReaderClosed.java (working copy) @@ -22,7 +22,6 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.StringField; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.store.AlreadyClosedException; Index: lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java =================================================================== --- lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java (revision 1197230) +++ lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java (working copy) @@ -40,7 +40,8 @@ import org.apache.lucene.index.Term; import org.apache.lucene.index.Terms; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.lucene40.Lucene40PostingsFormat; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.TermQuery; import org.apache.lucene.store.Directory; @@ -1013,10 +1014,11 @@ // file, up until a time limit public void testRealTerms() throws Exception { - final String defaultCodec = CodecProvider.getDefault().getDefaultFieldCodec(); - if (defaultCodec.equals("SimpleText") || defaultCodec.equals("Memory")) { + // TODO: is this necessary? we use the annotation... + final String defaultFormat = _TestUtil.getPostingsFormat("abracadabra"); + if (defaultFormat.equals("SimpleText") || defaultFormat.equals("Memory")) { // no - CodecProvider.getDefault().setDefaultFieldCodec("Standard"); + Codec.setDefault(_TestUtil.alwaysPostingsFormat(new Lucene40PostingsFormat())); } final LineFileDocs docs = new LineFileDocs(random); Index: lucene/src/java/org/apache/lucene/index/FieldInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInfos.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/FieldInfos.java (working copy) @@ -29,9 +29,6 @@ import java.util.Map.Entry; import org.apache.lucene.index.FieldInfo.IndexOptions; -import org.apache.lucene.index.SegmentCodecs; // Required for Java 1.5 javadocs -import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.values.ValueType; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -146,17 +143,6 @@ } /** - * Returns a new {@link FieldInfos} instance with this as the global field - * map - * - * @return a new {@link FieldInfos} instance with this as the global field - * map - */ - public FieldInfos newFieldInfos(SegmentCodecsBuilder segmentCodecsBuilder) { - return new FieldInfos(this, segmentCodecsBuilder); - } - - /** * Returns true iff the last committed version differs from the * current version, otherwise false * @@ -198,7 +184,6 @@ private final SortedMap byNumber = new TreeMap(); private final HashMap byName = new HashMap(); private final FieldNumberBiMap globalFieldNumbers; - private final SegmentCodecsBuilder segmentCodecsBuilder; // First used in 2.9; prior to 2.9 there was no format header public static final int FORMAT_START = -2; @@ -230,16 +215,15 @@ /** * Creates a new {@link FieldInfos} instance with a private - * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} and a default {@link SegmentCodecsBuilder} - * initialized with {@link CodecProvider#getDefault()}. + * {@link org.apache.lucene.index.FieldInfos.FieldNumberBiMap} *

* Note: this ctor should not be used during indexing use * {@link FieldInfos#FieldInfos(FieldInfos)} or - * {@link FieldInfos#FieldInfos(FieldNumberBiMap,org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder)} + * {@link FieldInfos#FieldInfos(FieldNumberBiMap)} * instead. */ public FieldInfos() { - this(new FieldNumberBiMap(), SegmentCodecsBuilder.create(CodecProvider.getDefault())); + this(new FieldNumberBiMap()); } /** @@ -249,7 +233,7 @@ * @see #isReadOnly() */ FieldInfos(FieldInfos other) { - this(other.globalFieldNumbers, other.segmentCodecsBuilder); + this(other.globalFieldNumbers); } /** @@ -257,9 +241,8 @@ * If the {@link FieldNumberBiMap} is null this instance will be read-only. * @see #isReadOnly() */ - FieldInfos(FieldNumberBiMap globalFieldNumbers, SegmentCodecsBuilder segmentCodecsBuilder) { + FieldInfos(FieldNumberBiMap globalFieldNumbers) { this.globalFieldNumbers = globalFieldNumbers; - this.segmentCodecsBuilder = segmentCodecsBuilder; } /** @@ -273,7 +256,7 @@ * @throws IOException */ public FieldInfos(Directory d, String name) throws IOException { - this((FieldNumberBiMap)null, null); // use null here to make this FIs Read-Only + this((FieldNumberBiMap)null); // use null here to make this FIs Read-Only final IndexInput input = d.openInput(name, IOContext.READONCE); try { read(input, name); @@ -309,7 +292,7 @@ */ @Override synchronized public Object clone() { - FieldInfos fis = new FieldInfos(globalFieldNumbers, segmentCodecsBuilder); + FieldInfos fis = new FieldInfos(globalFieldNumbers); fis.format = format; fis.hasFreq = hasFreq; fis.hasProx = hasProx; @@ -468,7 +451,6 @@ if (globalFieldNumbers == null) { throw new IllegalStateException("FieldInfos are read-only, create a new instance with a global field map to make modifications to FieldInfos"); } - assert segmentCodecsBuilder != null : "SegmentCodecsBuilder is set to null but FieldInfos is not read-only"; FieldInfo fi = fieldInfo(name); if (fi == null) { final int fieldNumber = nextFieldNumber(name, preferredFieldNumber); @@ -477,9 +459,6 @@ fi.update(isIndexed, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions); fi.setDocValues(docValues); } - if ((fi.isIndexed || fi.hasDocValues()) && fi.getCodecId() == FieldInfo.UNASSIGNED_CODEC_ID) { - segmentCodecsBuilder.tryAddAndSet(fi); - } version++; return fi; } @@ -569,22 +548,6 @@ } return false; } - - /** - * Builds the {@link SegmentCodecs} mapping for this {@link FieldInfos} instance. - * @param clearBuilder true iff the internal {@link SegmentCodecsBuilder} must be cleared otherwise false - */ - public SegmentCodecs buildSegmentCodecs(boolean clearBuilder) { - if (globalFieldNumbers == null) { - throw new IllegalStateException("FieldInfos are read-only no SegmentCodecs available"); - } - assert segmentCodecsBuilder != null; - final SegmentCodecs segmentCodecs = segmentCodecsBuilder.build(); - if (clearBuilder) { - segmentCodecsBuilder.clear(); - } - return segmentCodecs; - } public void write(Directory d, String name) throws IOException { IndexOutput output = d.createOutput(name, IOContext.READONCE); @@ -628,7 +591,6 @@ bits |= OMIT_POSITIONS; output.writeString(fi.name); output.writeInt(fi.number); - output.writeInt(fi.getCodecId()); output.writeByte(bits); final byte b; @@ -698,9 +660,7 @@ for (int i = 0; i < size; i++) { String name = input.readString(); - // if this is a previous format codec 0 will be preflex! final int fieldNumber = format <= FORMAT_FLEX? input.readInt():i; - final int codecId = format <= FORMAT_FLEX? input.readInt():0; byte bits = input.readByte(); boolean isIndexed = (bits & IS_INDEXED) != 0; boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0; @@ -781,8 +741,7 @@ throw new IllegalStateException("unhandled indexValues type " + b); } } - final FieldInfo addInternal = addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType); - addInternal.setCodecId(codecId); + addInternal(name, fieldNumber, isIndexed, storeTermVector, storePositionsWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValuesType); } if (input.getFilePointer() != input.length()) { @@ -804,7 +763,7 @@ if (isReadOnly()) { return this; } - final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null, null); + final FieldInfos roFis = new FieldInfos((FieldNumberBiMap)null); for (FieldInfo fieldInfo : this) { FieldInfo clone = (FieldInfo) (fieldInfo).clone(); roFis.putInternal(clone); @@ -814,5 +773,14 @@ } return roFis; } - + + public boolean anyDocValuesFields() { + for (FieldInfo fi : this) { + if (fi.hasDocValues()) { + return true; + } + } + + return false; + } } Index: lucene/src/java/org/apache/lucene/index/IndexFileNames.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexFileNames.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/IndexFileNames.java (working copy) @@ -19,8 +19,10 @@ import java.util.regex.Pattern; -import org.apache.lucene.index.codecs.Codec; // for javadocs +import org.apache.lucene.index.codecs.PostingsFormat; // for javadocs +// TODO: put all files under codec and remove all the static extensions here + /** * This class contains useful constants representing filenames and extensions * used by lucene, as well as convenience methods for querying whether a file @@ -31,7 +33,7 @@ * {@link #segmentFileName(String, String, String) segmentFileName}). * *

NOTE: extensions used by codecs are not - * listed here. You must interact with the {@link Codec} + * listed here. You must interact with the {@link PostingsFormat} * directly. * * @lucene.internal @@ -188,20 +190,20 @@ * NOTE: .<ext> is added to the result file name only if * ext is not empty. *

- * NOTE: _<name> is added to the result file name only if - * name is not empty. + * NOTE: _<segmentSuffix> is added to the result file name only if + * it's not the empty string *

* NOTE: all custom files should be named using this method, or * otherwise some structures may fail to handle them properly (such as if they * are added to compound files). */ - public static String segmentFileName(String segmentName, String name, String ext) { - if (ext.length() > 0 || name.length() > 0) { + public static String segmentFileName(String segmentName, String segmentSuffix, String ext) { + if (ext.length() > 0 || segmentSuffix.length() > 0) { assert !ext.startsWith("."); - StringBuilder sb = new StringBuilder(segmentName.length() + 2 + name.length() + ext.length()); + StringBuilder sb = new StringBuilder(segmentName.length() + 2 + segmentSuffix.length() + ext.length()); sb.append(segmentName); - if (name.length() > 0) { - sb.append('_').append(name); + if (segmentSuffix.length() > 0) { + sb.append('_').append(segmentSuffix); } if (ext.length() > 0) { sb.append('.').append(ext); @@ -212,11 +214,6 @@ } } - /** Sugar for passing "" + name instead */ - public static String segmentFileName(String segmentName, int name, String ext) { - return segmentFileName(segmentName, ""+name, ext); - } - /** * Returns true if the given filename ends with the given extension. One * should provide a pure extension, without '.'. Index: lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java (working copy) @@ -26,7 +26,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FlushInfo; @@ -152,7 +152,7 @@ } private final static boolean INFO_VERBOSE = false; final DocumentsWriter parent; - final CodecProvider codecProvider; + final Codec codec; final IndexWriter writer; final Directory directory; final DocState docState; @@ -183,7 +183,7 @@ this.fieldInfos = fieldInfos; this.writer = parent.indexWriter; this.infoStream = parent.infoStream; - this.codecProvider = this.writer.codecs; + this.codec = parent.codec; this.docState = new DocState(this); this.docState.similarityProvider = parent.indexWriter.getConfig() .getSimilarityProvider(); @@ -405,8 +405,8 @@ return numDocsInRAM; } - SegmentCodecs getCodec() { - return flushState.segmentCodecs; + Codec getCodec() { + return flushState.codec; } /** Reset after a flush */ @@ -443,7 +443,7 @@ assert deleteSlice == null : "all deletes must be applied in prepareFlush"; flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos, numDocsInRAM, writer.getConfig().getTermIndexInterval(), - fieldInfos.buildSegmentCodecs(true), pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed()))); + codec, pendingDeletes, new IOContext(new FlushInfo(numDocsInRAM, bytesUsed()))); final double startMBUsed = parent.flushControl.netBytes() / 1024. / 1024.; // Apply delete-by-docID now (delete-byDocID only // happens when an exception is hit processing that @@ -474,12 +474,12 @@ try { consumer.flush(flushState); pendingDeletes.terms.clear(); - final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.segmentCodecs, fieldInfos.asReadOnly()); + final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.codec, fieldInfos.asReadOnly()); if (infoStream != null) { message("new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.liveDocs.count())) + " deleted docs"); message("new segment has " + (newSegment.getHasVectors() ? "vectors" : "no vectors")); message("flushedFiles=" + newSegment.files()); - message("flushed codecs=" + newSegment.getSegmentCodecs()); + message("flushed codec=" + newSegment.getCodec()); } flushedDocCount += flushState.numDocs; @@ -556,9 +556,9 @@ bytesUsed.addAndGet(-(length *(INT_BLOCK_SIZE*RamUsageEstimator.NUM_BYTES_INT))); } - PerDocWriteState newPerDocWriteState(int codecId) { + PerDocWriteState newPerDocWriteState(String segmentSuffix) { assert segment != null; - return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, codecId, IOContext.DEFAULT); + return new PerDocWriteState(infoStream, directory, segment, fieldInfos, bytesUsed, segmentSuffix, IOContext.DEFAULT); } void setInfoStream(PrintStream infoStream) { Index: lucene/src/java/org/apache/lucene/index/IndexUpgrader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexUpgrader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/IndexUpgrader.java (working copy) @@ -114,12 +114,12 @@ } public void upgrade() throws IOException { - if (!IndexReader.indexExists(dir, iwc.getCodecProvider())) { + if (!IndexReader.indexExists(dir)) { throw new IndexNotFoundException(dir.toString()); } if (!deletePriorCommits) { - final Collection commits = DirectoryReader.listCommits(dir, iwc.getCodecProvider()); + final Collection commits = DirectoryReader.listCommits(dir); if (commits.size() > 1) { throw new IllegalArgumentException("This tool was invoked to not delete prior commit points, but the following commits were found: " + commits); } Index: lucene/src/java/org/apache/lucene/index/SegmentInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfo.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/SegmentInfo.java (working copy) @@ -28,7 +28,6 @@ import java.util.Set; import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; @@ -97,7 +96,7 @@ private FieldInfos fieldInfos; - private SegmentCodecs segmentCodecs; + private Codec codec; private Map diagnostics; @@ -116,7 +115,7 @@ private long fieldInfosVersion; public SegmentInfo(String name, int docCount, Directory dir, boolean isCompoundFile, - SegmentCodecs segmentCodecs, FieldInfos fieldInfos) { + Codec codec, FieldInfos fieldInfos) { this.name = name; this.docCount = docCount; this.dir = dir; @@ -124,7 +123,7 @@ this.isCompoundFile = isCompoundFile; this.docStoreOffset = -1; this.docStoreSegment = name; - this.segmentCodecs = segmentCodecs; + this.codec = codec; delCount = 0; version = Constants.LUCENE_MAIN_VERSION; this.fieldInfos = fieldInfos; @@ -156,7 +155,7 @@ } isCompoundFile = src.isCompoundFile; delCount = src.delCount; - segmentCodecs = src.segmentCodecs; + codec = src.codec; } void setDiagnostics(Map diagnostics) { @@ -177,7 +176,7 @@ * @param format format of the segments info file * @param input input handle to read segment info from */ - public SegmentInfo(Directory dir, int format, IndexInput input, CodecProvider codecs) throws IOException { + public SegmentInfo(Directory dir, int format, IndexInput input) throws IOException { this.dir = dir; if (format <= DefaultSegmentInfosWriter.FORMAT_3_1) { version = input.readString(); @@ -221,13 +220,13 @@ hasProx = input.readByte(); + // System.out.println(Thread.currentThread().getName() + ": si.read hasProx=" + hasProx + " seg=" + name); + // note: if the codec is not available: Codec.forName will throw an exception. if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { - segmentCodecs = new SegmentCodecs(codecs, input); + codec = Codec.forName(input.readString()); } else { - // codec ID on FieldInfo is 0 so it will simply use the first codec available - // TODO what todo if preflex is not available in the provider? register it or fail? - segmentCodecs = new SegmentCodecs(codecs, new Codec[] { codecs.lookup("PreFlex")}); + codec = Codec.forName("Lucene3x"); } diagnostics = input.readStringStringMap(); @@ -350,7 +349,7 @@ @Override public Object clone() { - final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, segmentCodecs, + final SegmentInfo si = new SegmentInfo(name, docCount, dir, isCompoundFile, codec, fieldInfos == null ? null : (FieldInfos) fieldInfos.clone()); si.docStoreOffset = docStoreOffset; si.docStoreSegment = docStoreSegment; @@ -573,7 +572,7 @@ output.writeByte((byte) (isCompoundFile ? YES : NO)); output.writeInt(delCount); output.writeByte((byte) (hasProx)); - segmentCodecs.write(output); + output.writeString(codec.getName()); output.writeStringStringMap(diagnostics); output.writeByte((byte) (hasVectors)); } @@ -583,16 +582,16 @@ } /** Can only be called once. */ - public void setSegmentCodecs(SegmentCodecs segmentCodecs) { - assert this.segmentCodecs == null; - if (segmentCodecs == null) { + public void setCodec(Codec codec) { + assert this.codec == null; + if (codec == null) { throw new IllegalArgumentException("segmentCodecs must be non-null"); } - this.segmentCodecs = segmentCodecs; + this.codec = codec; } - SegmentCodecs getSegmentCodecs() { - return segmentCodecs; + Codec getCodec() { + return codec; } private void addIfExists(Set files, String fileName) throws IOException { @@ -628,7 +627,7 @@ for(String ext : IndexFileNames.NON_STORE_INDEX_EXTENSIONS) { addIfExists(fileSet, IndexFileNames.segmentFileName(name, "", ext)); } - segmentCodecs.files(dir, this, fileSet); + codec.files(dir, this, fileSet); } if (docStoreOffset != -1) { Index: lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java =================================================================== --- lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java (working copy) @@ -24,8 +24,7 @@ import org.apache.lucene.index.Fields; import org.apache.lucene.index.FieldsEnum; import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.util.BytesRef; /** @@ -40,13 +39,11 @@ * performance {@link IndexDocValues} should be consumed per-segment just like * IndexReader. *

- * {@link IndexDocValues} are fully integrated into the {@link Codec} API. - * Custom implementations can be exposed on a per field basis via - * {@link CodecProvider}. + * {@link IndexDocValues} are fully integrated into the {@link DocValuesFormat} API. * * @see ValueType for limitations and default implementation documentation * @see IndexDocValuesField for adding values to the index - * @see Codec#docsConsumer(org.apache.lucene.index.PerDocWriteState) for + * @see DocValuesFormat#docsConsumer(org.apache.lucene.index.PerDocWriteState) for * customization * @lucene.experimental */ Index: lucene/src/java/org/apache/lucene/index/values/ValueType.java =================================================================== --- lucene/src/java/org/apache/lucene/index/values/ValueType.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/values/ValueType.java (working copy) @@ -17,7 +17,7 @@ * limitations under the License. */ -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.util.BytesRef; @@ -27,7 +27,7 @@ * ValueType specifies the {@link IndexDocValues} type for a * certain field. A ValueType only defines the data type for a field * while the actual implementation used to encode and decode the values depends - * on the the {@link Codec#docsConsumer} and {@link Codec#docsProducer} methods. + * on the the {@link DocValuesFormat#docsConsumer} and {@link DocValuesFormat#docsProducer} methods. * * @lucene.experimental */ Index: lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java =================================================================== --- lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/values/PackedIntValues.java (working copy) @@ -156,7 +156,7 @@ protected PackedIntsReader(Directory dir, String id, int numDocs, IOContext context) throws IOException { datIn = dir.openInput( - IndexFileNames.segmentFileName(id, "", Writer.DATA_EXTENSION), + IndexFileNames.segmentFileName(id, Bytes.DV_SEGMENT_SUFFIX, Writer.DATA_EXTENSION), context); this.numDocs = numDocs; boolean success = false; Index: lucene/src/java/org/apache/lucene/index/values/Bytes.java =================================================================== --- lucene/src/java/org/apache/lucene/index/values/Bytes.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/values/Bytes.java (working copy) @@ -61,6 +61,9 @@ * @lucene.experimental */ public final class Bytes { + + static final String DV_SEGMENT_SUFFIX = "dv"; + // TODO - add bulk copy where possible private Bytes() { /* don't instantiate! */ } @@ -244,7 +247,7 @@ if (datOut == null) { boolean success = false; try { - datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", + datOut = dir.createOutput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, DATA_EXTENSION), context); CodecUtil.writeHeader(datOut, codecName, version); success = true; @@ -269,7 +272,7 @@ boolean success = false; try { if (idxOut == null) { - idxOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", + idxOut = dir.createOutput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, INDEX_EXTENSION), context); CodecUtil.writeHeader(idxOut, codecName, version); } @@ -307,10 +310,10 @@ @Override public void files(Collection files) throws IOException { assert datOut != null; - files.add(IndexFileNames.segmentFileName(id, "", DATA_EXTENSION)); + files.add(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, DATA_EXTENSION)); if (idxOut != null) { // called after flush - so this must be initialized // if needed or present - final String idxFile = IndexFileNames.segmentFileName(id, "", + final String idxFile = IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, INDEX_EXTENSION); files.add(idxFile); } @@ -334,11 +337,11 @@ IndexInput indexIn = null; boolean success = false; try { - dataIn = dir.openInput(IndexFileNames.segmentFileName(id, "", + dataIn = dir.openInput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, Writer.DATA_EXTENSION), context); version = CodecUtil.checkHeader(dataIn, codecName, maxVersion, maxVersion); if (doIndex) { - indexIn = dir.openInput(IndexFileNames.segmentFileName(id, "", + indexIn = dir.openInput(IndexFileNames.segmentFileName(id, DV_SEGMENT_SUFFIX, Writer.INDEX_EXTENSION), context); final int version2 = CodecUtil.checkHeader(indexIn, codecName, maxVersion, maxVersion); @@ -494,8 +497,7 @@ protected void releaseResources() { hash.close(); - bytesUsed - .addAndGet((-docToEntry.length) * RamUsageEstimator.NUM_BYTES_INT); + bytesUsed.addAndGet((-docToEntry.length) * RamUsageEstimator.NUM_BYTES_INT); docToEntry = null; } Index: lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java (working copy) @@ -19,7 +19,7 @@ import java.io.IOException; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.FieldsWriter; import org.apache.lucene.store.IOContext; import org.apache.lucene.util.ArrayUtil; @@ -35,12 +35,12 @@ int freeCount; final DocumentsWriterPerThread.DocState docState; - final CodecProvider codecProvider; + final Codec codec; public StoredFieldsWriter(DocumentsWriterPerThread docWriter) { this.docWriter = docWriter; this.docState = docWriter.docState; - this.codecProvider = docWriter.codecProvider; + this.codec = docWriter.codec; } private int numStoredFields; @@ -81,7 +81,7 @@ private synchronized void initFieldsWriter(IOContext context) throws IOException { if (fieldsWriter == null) { - fieldsWriter = codecProvider.fieldsWriter(docWriter.directory, docWriter.getSegment(), context); + fieldsWriter = codec.fieldsFormat().fieldsWriter(docWriter.directory, docWriter.getSegment(), context); lastDocID = 0; } } Index: lucene/src/java/org/apache/lucene/index/SegmentReadState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentReadState.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/SegmentReadState.java (working copy) @@ -35,11 +35,11 @@ // that must do so), then it should negate this value to // get the app's terms divisor: public int termsIndexDivisor; - public final int codecId; + public final String segmentSuffix; public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, IOContext context, int termsIndexDivisor) { - this(dir, info, fieldInfos, context, termsIndexDivisor, -1); + this(dir, info, fieldInfos, context, termsIndexDivisor, ""); } public SegmentReadState(Directory dir, @@ -47,12 +47,22 @@ FieldInfos fieldInfos, IOContext context, int termsIndexDivisor, - int codecId) { + String segmentSuffix) { this.dir = dir; this.segmentInfo = info; this.fieldInfos = fieldInfos; this.context = context; this.termsIndexDivisor = termsIndexDivisor; - this.codecId = codecId; + this.segmentSuffix = segmentSuffix; } -} \ No newline at end of file + + public SegmentReadState(SegmentReadState other, + String newSegmentSuffix) { + this.dir = other.dir; + this.segmentInfo = other.segmentInfo; + this.fieldInfos = other.fieldInfos; + this.context = other.context; + this.termsIndexDivisor = other.termsIndexDivisor; + this.segmentSuffix = newSegmentSuffix; + } +} Index: lucene/src/java/org/apache/lucene/index/SegmentInfos.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentInfos.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/SegmentInfos.java (working copy) @@ -32,10 +32,11 @@ import java.util.Set; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.index.codecs.SegmentInfosReader; import org.apache.lucene.index.codecs.SegmentInfosWriter; +import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; @@ -83,8 +84,6 @@ // there was an IOException that had interrupted a commit public Map userData = Collections.emptyMap(); // Opaque Map that user can specify during IndexWriter.commit - - private CodecProvider codecs; private int format; @@ -95,20 +94,14 @@ private transient List cachedUnmodifiableList; private transient Set cachedUnmodifiableSet; + private Codec codecFormat; + /** * If non-null, information about loading segments_N files * will be printed here. @see #setInfoStream. */ private static PrintStream infoStream = null; - public SegmentInfos() { - this(CodecProvider.getDefault()); - } - - public SegmentInfos(CodecProvider codecs) { - this.codecs = codecs; - } - public void setFormat(int format) { this.format = format; } @@ -197,7 +190,7 @@ * since this file might belong to more than one segment (global map) and * could otherwise easily be confused with a per-segment file. */ - return IndexFileNames.segmentFileName(""+ version, "", IndexFileNames.GLOBAL_FIELD_NUM_MAP_EXTENSION); + return IndexFileNames.segmentFileName("_"+ version, "", IndexFileNames.GLOBAL_FIELD_NUM_MAP_EXTENSION); } /** @@ -241,9 +234,7 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public final void read(Directory directory, String segmentFileName, - CodecProvider codecs) throws CorruptIndexException, IOException { - this.codecs = codecs; + public final void read(Directory directory, String segmentFileName) throws CorruptIndexException, IOException { boolean success = false; // Clear any previous segments: @@ -253,12 +244,40 @@ lastGeneration = generation; + // TODO: scary to have default impl reopen the file... but to make it a bit more flexible, + // maybe we could use a plain indexinput here... could default impl rewind/wrap with checksumII, + // and any checksumming is then up to implementation? + ChecksumIndexInput input = null; try { - SegmentInfosReader infosReader = codecs.getSegmentInfosReader(); - infosReader.read(directory, segmentFileName, codecs, this, IOContext.READ); + input = new ChecksumIndexInput(directory.openInput(segmentFileName, IOContext.READ)); + final int format = input.readInt(); + setFormat(format); + + // check that it is a format we can understand + if (format > DefaultSegmentInfosWriter.FORMAT_MINIMUM) + throw new IndexFormatTooOldException(segmentFileName, format, + DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); + if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT) + throw new IndexFormatTooNewException(segmentFileName, format, + DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); + + if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { + codecFormat = Codec.forName(input.readString()); + } else { + codecFormat = Codec.forName("Lucene3x"); + } + SegmentInfosReader infosReader = codecFormat.segmentInfosFormat().getSegmentInfosReader(); + infosReader.read(directory, segmentFileName, input, this, IOContext.READ); + final long checksumNow = input.getChecksum(); + final long checksumThen = input.readLong(); + if (checksumNow != checksumThen) + throw new CorruptIndexException("checksum mismatch in segments file"); success = true; } finally { + if (input != null) { + input.close(); + } if (!success) { // Clear any segment infos we had loaded so we // have a clean slate on retry: @@ -267,25 +286,14 @@ } } - /** - * This version of read uses the retry logic (for lock-less - * commits) to find the right segments file to load. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ public final void read(Directory directory) throws CorruptIndexException, IOException { - read(directory, CodecProvider.getDefault()); - } - - public final void read(Directory directory, final CodecProvider codecs) throws CorruptIndexException, IOException { generation = lastGeneration = -1; - this.codecs = codecs; new FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - read(directory, segmentFileName, codecs); + read(directory, segmentFileName); return null; } }.run(); @@ -297,7 +305,7 @@ // before finishCommit is called IndexOutput pendingSegnOutput; - private void write(Directory directory) throws IOException { + private void write(Directory directory, Codec codec) throws IOException { String segmentFileName = getNextSegmentFileName(); final String globalFieldMapFile; @@ -322,8 +330,8 @@ boolean success = false; try { - SegmentInfosWriter infosWriter = codecs.getSegmentInfosWriter(); - segnOutput = infosWriter.writeInfos(directory, segmentFileName, this, IOContext.DEFAULT); + SegmentInfosWriter infosWriter = codec.segmentInfosFormat().getSegmentInfosWriter(); + segnOutput = infosWriter.writeInfos(directory, segmentFileName, codec.getName(), this, IOContext.DEFAULT); infosWriter.prepareCommit(segnOutput); pendingSegnOutput = segnOutput; success = true; @@ -380,7 +388,7 @@ sis.cachedUnmodifiableList = null; sis.cachedUnmodifiableSet = null; for(final SegmentInfo info : this) { - assert info.getSegmentCodecs() != null; + assert info.getCodec() != null; // dont directly access segments, use add method!!! sis.add((SegmentInfo) info.clone()); } @@ -409,7 +417,7 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static long readCurrentVersion(Directory directory, final CodecProvider codecs) + public static long readCurrentVersion(Directory directory) throws CorruptIndexException, IOException { // Fully read the segments file: this ensures that it's @@ -417,8 +425,8 @@ // IndexWriter.prepareCommit has been called (but not // yet commit), then the reader will still see itself as // current: - SegmentInfos sis = new SegmentInfos(codecs); - sis.read(directory, codecs); + SegmentInfos sis = new SegmentInfos(); + sis.read(directory); return sis.version; } @@ -427,10 +435,10 @@ * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static Map readCurrentUserData(Directory directory, CodecProvider codecs) + public static Map readCurrentUserData(Directory directory) throws CorruptIndexException, IOException { - SegmentInfos sis = new SegmentInfos(codecs); - sis.read(directory, codecs); + SegmentInfos sis = new SegmentInfos(); + sis.read(directory); return sis.getUserData(); } @@ -808,10 +816,10 @@ * method if changes have been made to this {@link SegmentInfos} instance *

**/ - final void prepareCommit(Directory dir) throws IOException { + final void prepareCommit(Directory dir, Codec codec) throws IOException { if (pendingSegnOutput != null) throw new IllegalStateException("prepareCommit was already called"); - write(dir); + write(dir, codec); } private final long writeGlobalFieldMap(FieldNumberBiMap map, Directory dir, String name) throws IOException { @@ -882,12 +890,12 @@ return files; } - final void finishCommit(Directory dir) throws IOException { + final void finishCommit(Directory dir, Codec codec) throws IOException { if (pendingSegnOutput == null) throw new IllegalStateException("prepareCommit was not called"); boolean success = false; try { - SegmentInfosWriter infosWriter = codecs.getSegmentInfosWriter(); + SegmentInfosWriter infosWriter = codec.segmentInfosFormat().getSegmentInfosWriter(); infosWriter.finishCommit(pendingSegnOutput); pendingSegnOutput = null; success = true; @@ -958,9 +966,9 @@ * method if changes have been made to this {@link SegmentInfos} instance *

**/ - final void commit(Directory dir) throws IOException { - prepareCommit(dir); - finishCommit(dir); + final void commit(Directory dir, Codec codec) throws IOException { + prepareCommit(dir, codec); + finishCommit(dir, codec); } public String toString(Directory directory) { @@ -1106,7 +1114,7 @@ if (cloneChildren) { final List list = new ArrayList(size()); for(final SegmentInfo info : this) { - assert info.getSegmentCodecs() != null; + assert info.getCodec() != null; list.add((SegmentInfo) info.clone()); } return list; @@ -1120,6 +1128,14 @@ this.addAll(infos); } + /** + * Returns the codec used to decode this SegmentInfos from disk + * @lucene.internal + */ + Codec codecFormat() { + return codecFormat; + } + /** Returns an unmodifiable {@link Iterator} of contained segments in order. */ // @Override (comment out until Java 6) public Iterator iterator() { Index: lucene/src/java/org/apache/lucene/index/CheckIndex.java =================================================================== --- lucene/src/java/org/apache/lucene/index/CheckIndex.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/CheckIndex.java (working copy) @@ -25,7 +25,7 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.IndexInput; import org.apache.lucene.document.Document; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import java.io.File; import java.io.IOException; @@ -143,8 +143,8 @@ /** Name of the segment. */ public String name; - /** CodecInfo used to read this segment. */ - public SegmentCodecs codec; + /** Codec used to read this segment. */ + public Codec codec; /** Document count (does not take deletions into account). */ public int docCount; @@ -322,10 +322,6 @@ public Status checkIndex() throws IOException { return checkIndex(null); } - - public Status checkIndex(List onlySegments) throws IOException { - return checkIndex(onlySegments, CodecProvider.getDefault()); - } /** Returns a {@link Status} instance detailing * the state of the index. @@ -339,13 +335,13 @@ *

WARNING: make sure * you only call this when the index is not opened by any * writer. */ - public Status checkIndex(List onlySegments, CodecProvider codecs) throws IOException { + public Status checkIndex(List onlySegments) throws IOException { NumberFormat nf = NumberFormat.getInstance(); - SegmentInfos sis = new SegmentInfos(codecs); + SegmentInfos sis = new SegmentInfos(); Status result = new Status(); result.dir = dir; try { - sis.read(dir, codecs); + sis.read(dir); } catch (Throwable t) { msg("ERROR: could not read any segments file in directory"); result.missingSegments = true; @@ -377,6 +373,7 @@ final int numSegments = sis.size(); final String segmentsFileName = sis.getCurrentSegmentFileName(); + // note: we only read the format byte (required preamble) here! IndexInput input = null; try { input = dir.openInput(segmentsFileName, IOContext.DEFAULT); @@ -489,7 +486,7 @@ SegmentReader reader = null; try { - final SegmentCodecs codec = info.getSegmentCodecs(); + final Codec codec = info.getCodec(); msg(" codec=" + codec); segInfoStat.codec = codec; msg(" compound=" + info.getUseCompoundFile()); @@ -1182,11 +1179,11 @@ * *

WARNING: Make sure you only call this when the * index is not opened by any writer. */ - public void fixIndex(Status result) throws IOException { + public void fixIndex(Status result, Codec codec) throws IOException { if (result.partial) throw new IllegalArgumentException("can only fix an index that was fully checked (this status checked a subset of segments)"); result.newSegments.changed(); - result.newSegments.commit(result.dir); + result.newSegments.commit(result.dir, codec); } private static boolean assertsOn; @@ -1236,6 +1233,7 @@ public static void main(String[] args) throws IOException, InterruptedException { boolean doFix = false; + Codec codec = Codec.getDefault(); // only used when fixing boolean verbose = false; List onlySegments = new ArrayList(); String indexPath = null; @@ -1244,6 +1242,13 @@ if (args[i].equals("-fix")) { doFix = true; i++; + } else if (args[i].equals("-codec")) { + if (i == args.length-1) { + System.out.println("ERROR: missing name for -codec option"); + System.exit(1); + } + codec = Codec.forName(args[i+1]); + i+=2; } else if (args[i].equals("-verbose")) { verbose = true; i++; @@ -1269,6 +1274,7 @@ System.out.println("\nUsage: java org.apache.lucene.index.CheckIndex pathToIndex [-fix] [-segment X] [-segment Y]\n" + "\n" + " -fix: actually write a new segments_N file, removing any problematic segments\n" + + " -codec X: when fixing, codec to write the new segments_N file with\n" + " -verbose: print additional details\n" + " -segment X: only check the specified segments. This can be specified multiple\n" + " times, to check more than one segment, eg '-segment _2 -segment _a'.\n" + @@ -1329,7 +1335,7 @@ System.out.println(" " + (5-s) + "..."); } System.out.println("Writing..."); - checker.fixIndex(result); + checker.fixIndex(result, codec); System.out.println("OK"); System.out.println("Wrote new segments file \"" + result.newSegments.getCurrentSegmentFileName() + "\""); } Index: lucene/src/java/org/apache/lucene/index/PerDocWriteState.java =================================================================== --- lucene/src/java/org/apache/lucene/index/PerDocWriteState.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/PerDocWriteState.java (working copy) @@ -34,41 +34,37 @@ public final String segmentName; public final FieldInfos fieldInfos; public final Counter bytesUsed; - public final SegmentCodecs segmentCodecs; - public final int codecId; + public final String segmentSuffix; public final IOContext context; - PerDocWriteState(PrintStream infoStream, Directory directory, + public PerDocWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos, Counter bytesUsed, - int codecId, IOContext context) { + String segmentSuffix, IOContext context) { this.infoStream = infoStream; this.directory = directory; this.segmentName = segmentName; this.fieldInfos = fieldInfos; - this.segmentCodecs = fieldInfos.buildSegmentCodecs(false); - this.codecId = codecId; + this.segmentSuffix = segmentSuffix; this.bytesUsed = bytesUsed; this.context = context; } - PerDocWriteState(SegmentWriteState state) { + public PerDocWriteState(SegmentWriteState state) { infoStream = state.infoStream; directory = state.directory; - segmentCodecs = state.segmentCodecs; segmentName = state.segmentName; fieldInfos = state.fieldInfos; - codecId = state.codecId; + segmentSuffix = state.segmentSuffix; bytesUsed = Counter.newCounter(); context = state.context; } - PerDocWriteState(PerDocWriteState state, int codecId) { + public PerDocWriteState(PerDocWriteState state, String segmentSuffix) { this.infoStream = state.infoStream; this.directory = state.directory; this.segmentName = state.segmentName; this.fieldInfos = state.fieldInfos; - this.segmentCodecs = state.segmentCodecs; - this.codecId = codecId; + this.segmentSuffix = segmentSuffix; this.bytesUsed = state.bytesUsed; this.context = state.context; } Index: lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThreadPool.java (working copy) @@ -20,8 +20,6 @@ import java.util.concurrent.locks.ReentrantLock; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; -import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.SetOnce; /** @@ -128,7 +126,6 @@ private final ThreadState[] perThreads; private volatile int numThreadStatesActive; - private CodecProvider codecProvider; private FieldNumberBiMap globalFieldMap; private final SetOnce documentsWriter = new SetOnce(); @@ -148,11 +145,9 @@ public void initialize(DocumentsWriter documentsWriter, FieldNumberBiMap globalFieldMap, IndexWriterConfig config) { this.documentsWriter.set(documentsWriter); // thread pool is bound to DW - final CodecProvider codecs = config.getCodecProvider(); - this.codecProvider = codecs; this.globalFieldMap = globalFieldMap; for (int i = 0; i < perThreads.length; i++) { - final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)); + final FieldInfos infos = new FieldInfos(globalFieldMap); perThreads[i] = new ThreadState(new DocumentsWriterPerThread(documentsWriter.directory, documentsWriter, infos, documentsWriter.chain)); } } @@ -240,7 +235,7 @@ assert threadState.isHeldByCurrentThread(); final DocumentsWriterPerThread dwpt = threadState.perThread; if (!closed) { - final FieldInfos infos = globalFieldMap.newFieldInfos(SegmentCodecsBuilder.create(codecProvider)); + final FieldInfos infos = new FieldInfos(globalFieldMap); final DocumentsWriterPerThread newDwpt = new DocumentsWriterPerThread(dwpt, infos); newDwpt.initialize(); threadState.resetWriter(newDwpt); Index: lucene/src/java/org/apache/lucene/index/DirectoryReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DirectoryReader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/DirectoryReader.java (working copy) @@ -33,7 +33,6 @@ import org.apache.lucene.store.IOContext; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; @@ -45,8 +44,6 @@ class DirectoryReader extends IndexReader implements Cloneable { protected Directory directory; protected boolean readOnly; - - protected CodecProvider codecs; IndexWriter writer; @@ -78,15 +75,13 @@ // } static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, - final int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException { - final CodecProvider codecProvider = codecs == null ? CodecProvider.getDefault() - : codecs; + final int termInfosIndexDivisor) throws CorruptIndexException, IOException { return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - SegmentInfos infos = new SegmentInfos(codecProvider); - infos.read(directory, segmentFileName, codecProvider); - return new DirectoryReader(directory, infos, deletionPolicy, readOnly, termInfosIndexDivisor, codecProvider); + SegmentInfos infos = new SegmentInfos(); + infos.read(directory, segmentFileName); + return new DirectoryReader(directory, infos, deletionPolicy, readOnly, termInfosIndexDivisor); } }.run(commit); } @@ -97,17 +92,12 @@ // } /** Construct reading the named set of readers. */ - DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws IOException { + DirectoryReader(Directory directory, SegmentInfos sis, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws IOException { this.directory = directory; this.readOnly = readOnly; this.segmentInfos = sis; this.deletionPolicy = deletionPolicy; this.termInfosIndexDivisor = termInfosIndexDivisor; - if (codecs == null) { - this.codecs = CodecProvider.getDefault(); - } else { - this.codecs = codecs; - } readerFinishedListeners = new MapBackedSet(new ConcurrentHashMap()); applyAllDeletes = false; @@ -141,17 +131,12 @@ } // Used by near real-time search - DirectoryReader(IndexWriter writer, SegmentInfos infos, CodecProvider codecs, boolean applyAllDeletes) throws IOException { + DirectoryReader(IndexWriter writer, SegmentInfos infos, boolean applyAllDeletes) throws IOException { this.directory = writer.getDirectory(); this.readOnly = true; this.applyAllDeletes = applyAllDeletes; // saved for reopen this.termInfosIndexDivisor = writer.getConfig().getReaderTermsIndexDivisor(); - if (codecs == null) { - this.codecs = CodecProvider.getDefault(); - } else { - this.codecs = codecs; - } readerFinishedListeners = writer.getReaderFinishedListeners(); // IndexWriter synchronizes externally before calling @@ -200,8 +185,7 @@ /** This constructor is only used for {@link #doOpenIfChanged()} */ DirectoryReader(Directory directory, SegmentInfos infos, SegmentReader[] oldReaders, - boolean readOnly, boolean doClone, int termInfosIndexDivisor, CodecProvider codecs, - Collection readerFinishedListeners) throws IOException { + boolean readOnly, boolean doClone, int termInfosIndexDivisor, Collection readerFinishedListeners) throws IOException { this.directory = directory; this.readOnly = readOnly; this.segmentInfos = infos; @@ -209,13 +193,6 @@ this.readerFinishedListeners = readerFinishedListeners; applyAllDeletes = false; - if (codecs == null) { - this.codecs = CodecProvider.getDefault(); - } else { - this.codecs = codecs; - } - - // we put the old SegmentReaders in a map, that allows us // to lookup a reader using its segment name Map segmentReaders = new HashMap(); @@ -347,7 +324,7 @@ starts[subReaders.length] = maxDoc; if (!readOnly) { - maxIndexVersion = SegmentInfos.readCurrentVersion(directory, codecs); + maxIndexVersion = SegmentInfos.readCurrentVersion(directory); } } @@ -498,15 +475,15 @@ return (IndexReader) new SegmentInfos.FindSegmentsFile(directory) { @Override protected Object doBody(String segmentFileName) throws CorruptIndexException, IOException { - final SegmentInfos infos = new SegmentInfos(codecs); - infos.read(directory, segmentFileName, codecs); + final SegmentInfos infos = new SegmentInfos(); + infos.read(directory, segmentFileName); return doOpenIfChanged(infos, false, openReadOnly); } }.run(commit); } private synchronized DirectoryReader doOpenIfChanged(SegmentInfos infos, boolean doClone, boolean openReadOnly) throws CorruptIndexException, IOException { - return new DirectoryReader(directory, infos, subReaders, openReadOnly, doClone, termInfosIndexDivisor, codecs, readerFinishedListeners); + return new DirectoryReader(directory, infos, subReaders, openReadOnly, doClone, termInfosIndexDivisor, readerFinishedListeners); } /** Version number when this IndexReader was opened. */ @@ -712,7 +689,7 @@ // we have to check whether index has changed since this reader was opened. // if so, this reader is no longer valid for deletion - if (SegmentInfos.readCurrentVersion(directory, codecs) > maxIndexVersion) { + if (SegmentInfos.readCurrentVersion(directory) > maxIndexVersion) { stale = true; this.writeLock.release(); this.writeLock = null; @@ -743,7 +720,7 @@ // KeepOnlyLastCommitDeleter: IndexFileDeleter deleter = new IndexFileDeleter(directory, deletionPolicy == null ? new KeepOnlyLastCommitDeletionPolicy() : deletionPolicy, - segmentInfos, null, codecs, null); + segmentInfos, null, null); segmentInfos.updateGeneration(deleter.getLastSegmentInfos()); segmentInfos.changed(); @@ -764,7 +741,7 @@ // Sync all files we just wrote directory.sync(segmentInfos.files(directory, false)); - segmentInfos.commit(directory); + segmentInfos.commit(directory, segmentInfos.codecFormat()); success = true; } finally { @@ -842,7 +819,7 @@ ensureOpen(); if (writer == null || writer.isClosed()) { // we loaded SegmentInfos from the directory - return SegmentInfos.readCurrentVersion(directory, codecs) == segmentInfos.getVersion(); + return SegmentInfos.readCurrentVersion(directory) == segmentInfos.getVersion(); } else { return writer.nrtIsCurrent(segmentInfos); } @@ -925,17 +902,12 @@ /** @see org.apache.lucene.index.IndexReader#listCommits */ public static List listCommits(Directory dir) throws IOException { - return listCommits(dir, CodecProvider.getDefault()); - } - - /** @see org.apache.lucene.index.IndexReader#listCommits */ - public static List listCommits(Directory dir, CodecProvider codecs) throws IOException { final String[] files = dir.listAll(); List commits = new ArrayList(); - SegmentInfos latest = new SegmentInfos(codecs); - latest.read(dir, codecs); + SegmentInfos latest = new SegmentInfos(); + latest.read(dir); final long currentGen = latest.getGeneration(); commits.add(new ReaderCommit(latest, dir)); @@ -948,11 +920,11 @@ !fileName.equals(IndexFileNames.SEGMENTS_GEN) && SegmentInfos.generationFromSegmentsFileName(fileName) < currentGen) { - SegmentInfos sis = new SegmentInfos(codecs); + SegmentInfos sis = new SegmentInfos(); try { // IOException allowed to throw there, in case // segments_N is corrupt - sis.read(dir, fileName, codecs); + sis.read(dir, fileName); } catch (FileNotFoundException fnfe) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth Index: lucene/src/java/org/apache/lucene/index/SegmentMerger.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentMerger.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/SegmentMerger.java (working copy) @@ -73,7 +73,7 @@ private IOContext context; - SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, IOContext context) { + SegmentMerger(Directory dir, int termIndexInterval, String name, MergePolicy.OneMerge merge, PayloadProcessorProvider payloadProcessorProvider, FieldInfos fieldInfos, Codec codec, IOContext context) { this.payloadProcessorProvider = payloadProcessorProvider; directory = dir; segment = name; @@ -89,6 +89,7 @@ }; } this.termIndexInterval = termIndexInterval; + this.codec = codec; this.context = context; } @@ -254,12 +255,11 @@ fieldInfos.addOrUpdate(reader.getFieldNames(FieldOption.DOC_VALUES), false); } } - final SegmentCodecs codecInfo = fieldInfos.buildSegmentCodecs(false); int docCount = 0; setMatchingSegmentReaders(); - final FieldsWriter fieldsWriter = codecInfo.provider.fieldsWriter(directory, segment, context); + final FieldsWriter fieldsWriter = codec.fieldsFormat().fieldsWriter(directory, segment, context); try { int idx = 0; for (MergeState.IndexReaderAndLiveDocs reader : readers) { @@ -293,7 +293,7 @@ // entering the index. See LUCENE-1282 for // details. throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption"); - segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo, null, context); + segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codec, null, context); return docCount; } @@ -493,9 +493,9 @@ } } - SegmentCodecs getSegmentCodecs() { + Codec getCodec() { assert segmentWriteState != null; - return segmentWriteState.segmentCodecs; + return segmentWriteState.codec; } private final void mergeTerms() throws CorruptIndexException, IOException { @@ -566,8 +566,8 @@ mergeState.dirPayloadProcessor[i] = payloadProcessorProvider.getDirProcessor(reader.reader.directory()); } } - codec = segmentWriteState.segmentCodecs.codec(); - final FieldsConsumer consumer = codec.fieldsConsumer(segmentWriteState); + + final FieldsConsumer consumer = codec.postingsFormat().fieldsConsumer(segmentWriteState); boolean success = false; try { consumer.merge(mergeState, @@ -584,7 +584,7 @@ } private void mergePerDoc() throws IOException { - final PerDocConsumer docsConsumer = codec + final PerDocConsumer docsConsumer = codec.docValuesFormat() .docsConsumer(new PerDocWriteState(segmentWriteState)); // TODO: remove this check when 3.x indexes are no longer supported // (3.x indexes don't have docvalues) Index: lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/DocFieldProcessor.java (working copy) @@ -25,7 +25,7 @@ import java.util.Map; import org.apache.lucene.index.DocumentsWriterPerThread.DocState; -import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.DocValuesFormat; import org.apache.lucene.index.codecs.DocValuesConsumer; import org.apache.lucene.index.codecs.PerDocConsumer; import org.apache.lucene.index.values.PerDocFieldValues; @@ -320,14 +320,13 @@ docValuesConsumerAndDocID.docID = docState.docID; return docValuesConsumerAndDocID.docValuesConsumer; } - PerDocConsumer perDocConsumer = perDocConsumers.get(fieldInfo.getCodecId()); + + PerDocConsumer perDocConsumer = perDocConsumers.get(0); if (perDocConsumer == null) { - PerDocWriteState perDocWriteState = docState.docWriter.newPerDocWriteState(fieldInfo.getCodecId()); - SegmentCodecs codecs = perDocWriteState.segmentCodecs; - assert codecs.codecs.length > fieldInfo.getCodecId(); - Codec codec = codecs.codecs[fieldInfo.getCodecId()]; - perDocConsumer = codec.docsConsumer(perDocWriteState); - perDocConsumers.put(Integer.valueOf(fieldInfo.getCodecId()), perDocConsumer); + PerDocWriteState perDocWriteState = docState.docWriter.newPerDocWriteState(""); + DocValuesFormat dvFormat = docState.docWriter.codec.docValuesFormat(); + perDocConsumer = dvFormat.docsConsumer(perDocWriteState); + perDocConsumers.put(0, perDocConsumer); } boolean success = false; DocValuesConsumer docValuesConsumer = null; Index: lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/FreqProxTermsWriter.java (working copy) @@ -57,7 +57,7 @@ // Sort by field name CollectionUtil.quickSort(allFields); - final FieldsConsumer consumer = state.segmentCodecs.codec().fieldsConsumer(state); + final FieldsConsumer consumer = state.codec.postingsFormat().fieldsConsumer(state); boolean success = false; Index: lucene/src/java/org/apache/lucene/index/SegmentCodecs.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/FieldInfo.java =================================================================== --- lucene/src/java/org/apache/lucene/index/FieldInfo.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/FieldInfo.java (working copy) @@ -21,7 +21,6 @@ /** @lucene.experimental */ public final class FieldInfo { - public static final int UNASSIGNED_CODEC_ID = -1; public final String name; public final int number; @@ -38,7 +37,6 @@ public IndexOptions indexOptions; public boolean storePayloads; // whether this field stores payloads together with term positions - private int codecId = UNASSIGNED_CODEC_ID; // set inside SegmentCodecs#build() during segment flush - this is used to identify the codec used to write this field /** * Controls how much information is stored in the postings lists. @@ -77,21 +75,11 @@ } assert indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS || !storePayloads; } - - void setCodecId(int codecId) { - assert this.codecId == UNASSIGNED_CODEC_ID : "CodecId can only be set once."; - this.codecId = codecId; - } - - public int getCodecId() { - return codecId; - } @Override public Object clone() { FieldInfo clone = new FieldInfo(name, isIndexed, number, storeTermVector, storePositionWithTermVector, storeOffsetWithTermVector, omitNorms, storePayloads, indexOptions, docValues); - clone.codecId = this.codecId; return clone; } Index: lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexReader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -28,8 +28,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.DocumentStoredFieldVisitor; -import org.apache.lucene.index.codecs.Codec; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.index.values.IndexDocValues; import org.apache.lucene.search.FieldCache; // javadocs @@ -312,7 +311,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory) throws CorruptIndexException, IOException { - return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(directory, null, null, true, DEFAULT_TERMS_INDEX_DIVISOR); } /** Returns an IndexReader reading the index in the given @@ -326,9 +325,9 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(directory, null, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } - + /** * Open a near real time IndexReader from the {@link org.apache.lucene.index.IndexWriter}. * @@ -363,7 +362,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(commit.getDirectory(), null, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } /** Expert: returns an IndexReader reading the index in @@ -381,7 +380,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(directory, deletionPolicy, null, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } /** Expert: returns an IndexReader reading the index in @@ -409,7 +408,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, null); + return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor); } /** Expert: returns an IndexReader reading the index in @@ -429,7 +428,7 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR, null); + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, DEFAULT_TERMS_INDEX_DIVISOR); } /** Expert: returns an IndexReader reading the index in @@ -462,80 +461,13 @@ * @throws IOException if there is a low-level IO error */ public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, null); + return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor); } - /** Expert: returns an IndexReader reading the index in - * the given Directory, with a custom {@link - * IndexDeletionPolicy}, and specified {@link CodecProvider}. - * You should pass readOnly=true, since it gives much - * better concurrent performance, unless you intend to do - * write operations (delete documents or change norms) - * with the reader. - * @param directory the index directory - * @param deletionPolicy a custom deletion policy (only used - * if you use this reader to perform deletes or to set - * norms); see {@link IndexWriter} for details. - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader - * @param termInfosIndexDivisor Subsamples which indexed - * terms are loaded into RAM. This has the same effect as {@link - * IndexWriterConfig#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1. Set this - * to -1 to skip loading the terms index entirely. - * @param codecs CodecProvider to use when opening index - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static IndexReader open(final Directory directory, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException { - return open(directory, deletionPolicy, null, readOnly, termInfosIndexDivisor, codecs); + private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor) throws CorruptIndexException, IOException { + return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor); } - /** Expert: returns an IndexReader reading the index in - * the given Directory, using a specific commit and with - * a custom {@link IndexDeletionPolicy} and specified - * {@link CodecProvider}. You should pass readOnly=true, since - * it gives much better concurrent performance, unless - * you intend to do write operations (delete documents or - * change norms) with the reader. - - * @param commit the specific {@link IndexCommit} to open; - * see {@link IndexReader#listCommits} to list all commits - * in a directory - * @param deletionPolicy a custom deletion policy (only used - * if you use this reader to perform deletes or to set - * norms); see {@link IndexWriter} for details. - * @param readOnly true if no changes (deletions, norms) will be made with this IndexReader - * @param termInfosIndexDivisor Subsamples which indexed - * terms are loaded into RAM. This has the same effect as {@link - * IndexWriterConfig#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1. Set this - * to -1 to skip loading the terms index entirely. - * @param codecs CodecProvider to use when opening index - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static IndexReader open(final IndexCommit commit, IndexDeletionPolicy deletionPolicy, boolean readOnly, int termInfosIndexDivisor, CodecProvider codecs) throws CorruptIndexException, IOException { - return open(commit.getDirectory(), deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs); - } - - private static IndexReader open(final Directory directory, final IndexDeletionPolicy deletionPolicy, final IndexCommit commit, final boolean readOnly, int termInfosIndexDivisor, - CodecProvider codecs) throws CorruptIndexException, IOException { - if (codecs == null) { - codecs = CodecProvider.getDefault(); - } - return DirectoryReader.open(directory, deletionPolicy, commit, readOnly, termInfosIndexDivisor, codecs); - } - /** * If the index has changed since the provided reader was * opened, open and return a new reader; else, return @@ -767,7 +699,7 @@ } }.run()).longValue(); } - + /** * Reads version number from segments files. The version number is * initialized with a timestamp and then increased by one for each change of @@ -779,25 +711,10 @@ * @throws IOException if there is a low-level IO error */ public static long getCurrentVersion(Directory directory) throws CorruptIndexException, IOException { - return getCurrentVersion(directory, CodecProvider.getDefault()); + return SegmentInfos.readCurrentVersion(directory); } /** - * Reads version number from segments files. The version number is - * initialized with a timestamp and then increased by one for each change of - * the index. - * - * @param directory where the index resides. - * @param codecs the {@link CodecProvider} holding all {@link Codec}s required to open the index - * @return version number. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - */ - public static long getCurrentVersion(Directory directory, CodecProvider codecs) throws CorruptIndexException, IOException { - return SegmentInfos.readCurrentVersion(directory, codecs); - } - - /** * Reads commitUserData, previously passed to {@link * IndexWriter#commit(Map)}, from current index * segments file. This will return null if {@link @@ -811,29 +728,9 @@ * * @see #getCommitUserData() */ - public static Map getCommitUserData(Directory directory) throws CorruptIndexException, IOException { - return getCommitUserData(directory, CodecProvider.getDefault()); + public static Map getCommitUserData(Directory directory) throws CorruptIndexException, IOException { + return SegmentInfos.readCurrentUserData(directory); } - - - /** - * Reads commitUserData, previously passed to {@link - * IndexWriter#commit(Map)}, from current index - * segments file. This will return null if {@link - * IndexWriter#commit(Map)} has never been called for - * this index. - * - * @param directory where the index resides. - * @param codecs the {@link CodecProvider} provider holding all {@link Codec}s required to open the index - * @return commit userData. - * @throws CorruptIndexException if the index is corrupt - * @throws IOException if there is a low-level IO error - * - * @see #getCommitUserData() - */ - public static Map getCommitUserData(Directory directory, CodecProvider codecs) throws CorruptIndexException, IOException { - return SegmentInfos.readCurrentUserData(directory, codecs); - } /** * Version number when this IndexReader was opened. Not @@ -985,22 +882,6 @@ } } - /** - * Returns true if an index exists at the specified directory. - * @param directory the directory to check for an index - * @param codecProvider provides a CodecProvider in case the index uses non-core codecs - * @return true if an index exists; false otherwise - * @throws IOException if there is a problem with accessing the index - */ - public static boolean indexExists(Directory directory, CodecProvider codecProvider) throws IOException { - try { - new SegmentInfos().read(directory, codecProvider); - return true; - } catch (IOException ioe) { - return false; - } - } - /** Returns the number of documents in this index. */ public abstract int numDocs(); Index: lucene/src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -31,6 +31,7 @@ import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.DocumentsWriterPerThreadPool.ThreadState; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.Query; import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.store.AlreadyClosedException; @@ -125,8 +126,11 @@ final DocumentsWriterPerThreadPool perThreadPool; final FlushPolicy flushPolicy; final DocumentsWriterFlushControl flushControl; - DocumentsWriter(IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers, + + final Codec codec; + DocumentsWriter(Codec codec, IndexWriterConfig config, Directory directory, IndexWriter writer, FieldNumberBiMap globalFieldNumbers, BufferedDeletesStream bufferedDeletesStream) throws IOException { + this.codec = codec; this.directory = directory; this.indexWriter = writer; this.similarityProvider = config.getSimilarityProvider(); Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -39,8 +39,7 @@ import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor; -import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.Query; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.CompoundFileDirectory; @@ -375,7 +374,7 @@ // just like we do when loading segments_N synchronized(this) { maybeApplyDeletes(applyAllDeletes); - r = new DirectoryReader(this, segmentInfos, codecs, applyAllDeletes); + r = new DirectoryReader(this, segmentInfos, applyAllDeletes); if (infoStream != null) { message("return reader version=" + r.getVersion() + " reader=" + r); } @@ -802,7 +801,7 @@ infoStream.println("IW " + messageID + " [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message); } - CodecProvider codecs; + final Codec codec; // for writing new segments /** * Constructs a new IndexWriter per the settings given in conf. @@ -837,7 +836,7 @@ mergePolicy = conf.getMergePolicy(); mergePolicy.setIndexWriter(this); mergeScheduler = conf.getMergeScheduler(); - codecs = conf.getCodecProvider(); + codec = conf.getCodec(); bufferedDeletesStream = new BufferedDeletesStream(messageID); bufferedDeletesStream.setInfoStream(infoStream); @@ -862,7 +861,7 @@ // If index is too old, reading the segments will throw // IndexFormatTooOldException. - segmentInfos = new SegmentInfos(codecs); + segmentInfos = new SegmentInfos(); try { if (create) { // Try to read first. This is to allow create @@ -870,7 +869,7 @@ // searching. In this case we write the next // segments_N file with no segments: try { - segmentInfos.read(directory, codecs); + segmentInfos.read(directory); segmentInfos.clear(); } catch (IOException e) { // Likely this means it's a fresh directory @@ -881,7 +880,7 @@ changeCount++; segmentInfos.changed(); } else { - segmentInfos.read(directory, codecs); + segmentInfos.read(directory); IndexCommit commit = conf.getIndexCommit(); if (commit != null) { @@ -892,8 +891,8 @@ // points. if (commit.getDirectory() != directory) throw new IllegalArgumentException("IndexCommit's directory doesn't match my directory"); - SegmentInfos oldInfos = new SegmentInfos(codecs); - oldInfos.read(directory, commit.getSegmentsFileName(), codecs); + SegmentInfos oldInfos = new SegmentInfos(); + oldInfos.read(directory, commit.getSegmentsFileName()); segmentInfos.replace(oldInfos); changeCount++; segmentInfos.changed(); @@ -906,7 +905,7 @@ // start with previous field numbers, but new FieldInfos globalFieldNumberMap = segmentInfos.getOrLoadGlobalFieldNumberMap(directory); - docWriter = new DocumentsWriter(config, directory, this, globalFieldNumberMap, bufferedDeletesStream); + docWriter = new DocumentsWriter(codec, config, directory, this, globalFieldNumberMap, bufferedDeletesStream); docWriter.setInfoStream(infoStream); // Default deleter (for backwards compatibility) is @@ -914,8 +913,7 @@ synchronized(this) { deleter = new IndexFileDeleter(directory, conf.getIndexDeletionPolicy(), - segmentInfos, infoStream, - codecs, this); + segmentInfos, infoStream, this); } if (deleter.startingCommitDeleted) { @@ -2149,6 +2147,7 @@ */ public synchronized void deleteAll() throws IOException { ensureOpen(); + boolean success = false; try { // Abort any running merges @@ -2170,10 +2169,11 @@ // Mark that the index has changed ++changeCount; segmentInfos.changed(); + success = true; } catch (OutOfMemoryError oom) { handleOOM(oom, "deleteAll"); } finally { - if (infoStream != null) { + if (!success && infoStream != null) { message("hit exception during deleteAll"); } } @@ -2476,8 +2476,8 @@ if (infoStream != null) { message("addIndexes: process directory " + dir); } - SegmentInfos sis = new SegmentInfos(codecs); // read infos from dir - sis.read(dir, codecs); + SegmentInfos sis = new SegmentInfos(); // read infos from dir + sis.read(dir); final Set dsFilesCopied = new HashSet(); final Map dsNames = new HashMap(); for (SegmentInfo info : sis) { @@ -2567,7 +2567,7 @@ // abortable so that IW.close(false) is able to stop it SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, null, payloadProcessorProvider, - globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs)), context); + new FieldInfos(globalFieldNumberMap), codec, context); for (IndexReader reader : readers) // add new indexes merger.add(reader); @@ -2575,7 +2575,7 @@ final FieldInfos fieldInfos = merger.fieldInfos(); SegmentInfo info = new SegmentInfo(mergedName, docCount, directory, - false, merger.getSegmentCodecs(), + false, merger.getCodec(), fieldInfos); setDiagnostics(info, "addIndexes(IndexReader...)"); @@ -2591,7 +2591,7 @@ // Now create the compound file if needed if (useCompoundFile) { - merger.createCompoundFile(mergedName + ".cfs", info, context); + merger.createCompoundFile(IndexFileNames.segmentFileName(mergedName, "", IndexFileNames.COMPOUND_FILE_EXTENSION), info, context); // delete new non cfs files directly: they were never // registered with IFD @@ -2916,7 +2916,7 @@ try { if (infoStream != null) message("commit: pendingCommit != null"); - pendingCommit.finishCommit(directory); + pendingCommit.finishCommit(directory, codec); if (infoStream != null) message("commit: wrote segments file \"" + pendingCommit.getCurrentSegmentFileName() + "\""); lastCommitChangeCount = pendingCommitChangeCount; @@ -3459,7 +3459,7 @@ // Bind a new segment name here so even with // ConcurrentMergePolicy we keep deterministic segment // names. - merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, globalFieldNumberMap.newFieldInfos(SegmentCodecsBuilder.create(codecs))); + merge.info = new SegmentInfo(newSegmentName(), 0, directory, false, null, new FieldInfos(globalFieldNumberMap)); // Lock order: IW -> BD final BufferedDeletesStream.ApplyDeletesResult result = bufferedDeletesStream.applyDeletes(readerPool, merge.segments); @@ -3633,7 +3633,7 @@ IOContext context = new IOContext(merge.getMergeInfo()); SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge, - payloadProcessorProvider, merge.info.getFieldInfos(), context); + payloadProcessorProvider, merge.info.getFieldInfos(), codec, context); if (infoStream != null) { message("merging " + merge.segString(directory) + " mergeVectors=" + merge.info.getFieldInfos().hasVectors()); @@ -3679,10 +3679,10 @@ mergedDocCount = merge.info.docCount = merger.merge(); // Record which codec was used to write the segment - merge.info.setSegmentCodecs(merger.getSegmentCodecs()); + merge.info.setCodec(merger.getCodec()); if (infoStream != null) { - message("merge segmentCodecs=" + merger.getSegmentCodecs()); + message("merge codecs=" + merger.getCodec()); message("merge store matchedCount=" + merger.getMatchedSubReaderCount() + " vs " + merge.readers.size()); } anyNonBulkMerges |= merger.getAnyNonBulkMerges(); @@ -3975,7 +3975,7 @@ // Exception here means nothing is prepared // (this method unwinds everything it did on // an exception) - toSync.prepareCommit(directory); + toSync.prepareCommit(directory, codec); pendingCommitSet = true; pendingCommit = toSync; Index: lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/IndexFileDeleter.java (working copy) @@ -18,7 +18,6 @@ */ import java.io.FileNotFoundException; -import java.io.FilenameFilter; import java.io.IOException; import java.io.PrintStream; import java.util.ArrayList; @@ -29,7 +28,6 @@ import java.util.List; import java.util.Map; -import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.store.NoSuchDirectoryException; import org.apache.lucene.util.CollectionUtil; @@ -122,8 +120,6 @@ infoStream.println("IFD [" + new Date() + "; " + Thread.currentThread().getName() + "]: " + message); } - private final FilenameFilter indexFilenameFilter; - // called only from assert private boolean locked() { return writer == null || Thread.holdsLock(writer); @@ -138,7 +134,7 @@ * @throws IOException if there is a low-level IO error */ public IndexFileDeleter(Directory directory, IndexDeletionPolicy policy, SegmentInfos segmentInfos, - PrintStream infoStream, CodecProvider codecs, IndexWriter writer) throws CorruptIndexException, IOException { + PrintStream infoStream, IndexWriter writer) throws CorruptIndexException, IOException { this.infoStream = infoStream; this.writer = writer; @@ -154,7 +150,6 @@ // First pass: walk the files and initialize our ref // counts: long currentGen = segmentInfos.getGeneration(); - indexFilenameFilter = new IndexFileNameFilter(codecs); CommitPoint currentCommitPoint = null; String[] files = null; @@ -167,7 +162,7 @@ for (String fileName : files) { - if ((indexFilenameFilter.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) { + if ((IndexFileNameFilter.INSTANCE.accept(null, fileName)) && !fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) { // Add this file to refCounts with initial count 0: getRefCount(fileName); @@ -180,9 +175,9 @@ if (infoStream != null) { message("init: load commit \"" + fileName + "\""); } - SegmentInfos sis = new SegmentInfos(codecs); + SegmentInfos sis = new SegmentInfos(); try { - sis.read(directory, fileName, codecs); + sis.read(directory, fileName); } catch (FileNotFoundException e) { // LUCENE-948: on NFS (and maybe others), if // you have writers switching back and forth @@ -253,9 +248,9 @@ // listing was stale (eg when index accessed via NFS // client with stale directory listing cache). So we // try now to explicitly open this commit point: - SegmentInfos sis = new SegmentInfos(codecs); + SegmentInfos sis = new SegmentInfos(); try { - sis.read(directory, currentSegmentsFile, codecs); + sis.read(directory, currentSegmentsFile); } catch (IOException e) { throw new CorruptIndexException("failed to locate current segments_N file"); } @@ -373,7 +368,7 @@ for(int i=0;iThis does not accept *-write.lock files. + * * @lucene.internal */ public class IndexFileNameFilter implements FilenameFilter { - private final HashSet extensions; - - public IndexFileNameFilter(CodecProvider codecs) { - extensions = new HashSet(); - for (String ext : IndexFileNames.INDEX_EXTENSIONS) { - extensions.add(ext); - } - if (codecs != null) { - for(String ext : codecs.getAllExtensions()) { - extensions.add(ext); - } - } + public static final FilenameFilter INSTANCE = new IndexFileNameFilter(); + + private IndexFileNameFilter() { } + // Approximate match for files that seem to be Lucene + // index files. This can easily over-match, ie if some + // app names a file _foo_bar.go: + private final Pattern luceneFilePattern = Pattern.compile("^_[a-z0-9]+(_[a-z0-9]+)?\\.[a-z0-9]+$"); + /* (non-Javadoc) * @see java.io.FilenameFilter#accept(java.io.File, java.lang.String) */ public boolean accept(File dir, String name) { - int i = name.lastIndexOf('.'); - if (i != -1) { - String extension = name.substring(1+i); - if (extensions.contains(extension)) { - return true; - } else if (extension.startsWith("f") && - extension.matches("f\\d+")) { - return true; - } else if (extension.startsWith("s") && - extension.matches("s\\d+")) { - return true; - } + if (name.lastIndexOf('.') != -1) { + // Has an extension + return luceneFilePattern.matcher(name).matches(); } else { - if (name.startsWith(IndexFileNames.SEGMENTS)) return true; + // No extension -- only segments_N file; + return name.startsWith(IndexFileNames.SEGMENTS); } - return false; } } Index: lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java (working copy) @@ -71,7 +71,7 @@ public BlockTermsWriter(TermsIndexWriterBase termsIndexWriter, SegmentWriteState state, PostingsWriterBase postingsWriter) throws IOException { - final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_EXTENSION); this.termsIndexWriter = termsIndexWriter; out = state.directory.createOutput(termsFileName, state.context); boolean success = false; Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java (working copy) @@ -56,12 +56,13 @@ public static final int FORMAT_MINIMUM = FORMAT_DIAGNOSTICS; @Override - public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos, IOContext context) + public IndexOutput writeInfos(Directory dir, String segmentFileName, String codecID, SegmentInfos infos, IOContext context) throws IOException { IndexOutput out = createOutput(dir, segmentFileName, new IOContext(new FlushInfo(infos.size(), infos.totalDocCount()))); boolean success = false; try { out.writeInt(FORMAT_CURRENT); // write FORMAT + out.writeString(codecID); // write codecID out.writeLong(infos.version); out.writeInt(infos.counter); // write counter out.writeLong(infos.getGlobalFieldMapVersion()); Index: lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/DocValuesReaderBase.java (working copy) @@ -65,19 +65,19 @@ // Only opens files... doesn't actually load any values protected TreeMap load(FieldInfos fieldInfos, - String segment, int docCount, Directory dir, int codecId, IOContext context) + String segment, int docCount, Directory dir, IOContext context) throws IOException { TreeMap values = new TreeMap(); boolean success = false; try { for (FieldInfo fieldInfo : fieldInfos) { - if (codecId == fieldInfo.getCodecId() && fieldInfo.hasDocValues()) { + if (fieldInfo.hasDocValues()) { final String field = fieldInfo.name; // TODO can we have a compound file per segment and codec for // docvalues? final String id = DefaultDocValuesConsumer.docValuesId(segment, - codecId, fieldInfo.number); + fieldInfo.number); values.put(field, loadDocValues(docCount, dir, id, fieldInfo.getDocValues(), context)); } Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesConsumer.java (working copy) @@ -25,7 +25,6 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; -import org.apache.lucene.index.codecs.DocValuesWriterBase; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; @@ -34,42 +33,45 @@ * @lucene.experimental */ public class DefaultDocValuesConsumer extends DocValuesWriterBase { - private final Directory directory; + private final Directory mainDirectory; + private Directory directory; + + final static String DOC_VALUES_SEGMENT_SUFFIX = "dv"; public DefaultDocValuesConsumer(PerDocWriteState state) throws IOException { super(state); + mainDirectory = state.directory; //TODO maybe we should enable a global CFS that all codecs can pull on demand to further reduce the number of files? - this.directory = new CompoundFileDirectory(state.directory, - IndexFileNames.segmentFileName(state.segmentName, state.codecId, - IndexFileNames.COMPOUND_FILE_EXTENSION), state.context, true); } @Override - protected Directory getDirectory() { + protected Directory getDirectory() throws IOException { + // lazy init + if (directory == null) { + directory = new CompoundFileDirectory(mainDirectory, + IndexFileNames.segmentFileName(segmentName, DOC_VALUES_SEGMENT_SUFFIX, + IndexFileNames.COMPOUND_FILE_EXTENSION), context, true); + } return directory; } @Override public void close() throws IOException { - this.directory.close(); + if (directory != null) { + directory.close(); + } } - @SuppressWarnings("fallthrough") - public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, Set files) throws IOException { + public static void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException { FieldInfos fieldInfos = segmentInfo.getFieldInfos(); for (FieldInfo fieldInfo : fieldInfos) { - if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); - assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); - assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, codecId, IndexFileNames.COMPOUND_FILE_EXTENSION)); - return; + if (fieldInfo.hasDocValues()) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION)); + assert dir.fileExists(IndexFileNames.segmentFileName(segmentInfo.name, DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_EXTENSION)); + break; } } } - - public static void getExtensions(Set extensions) { - extensions.add(IndexFileNames.COMPOUND_FILE_ENTRIES_EXTENSION); - extensions.add(IndexFileNames.COMPOUND_FILE_EXTENSION); - } } Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsWriter.java (working copy) @@ -61,6 +61,10 @@ // when removing support for old versions, leave the last supported version here static final int FORMAT_MINIMUM = FORMAT_LUCENE_3_0_NO_COMPRESSED_FIELDS; + // TODO: remove from IndexFileNames + public static final String FIELDS_EXTENSION = IndexFileNames.FIELDS_EXTENSION; + public static final String FIELDS_INDEX_EXTENSION = IndexFileNames.FIELDS_INDEX_EXTENSION; + // If null - we were supplied with streams, if notnull - we manage them ourselves private Directory directory; private String segment; @@ -73,8 +77,8 @@ boolean success = false; try { - fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); - indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION), context); + fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION), context); + indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION), context); fieldsStream.writeInt(FORMAT_CURRENT); indexStream.writeInt(FORMAT_CURRENT); @@ -129,11 +133,11 @@ } catch (IOException ignored) { } try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION)); + directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_EXTENSION)); } catch (IOException ignored) { } try { - directory.deleteFile(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); + directory.deleteFile(IndexFileNames.segmentFileName(segment, "", FIELDS_INDEX_EXTENSION)); } catch (IOException ignored) { } } Index: lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/FieldsProducer.java (working copy) @@ -34,23 +34,4 @@ public abstract class FieldsProducer extends Fields implements Closeable { public abstract void close() throws IOException; - - public static final FieldsProducer EMPTY = new FieldsProducer() { - - @Override - public Terms terms(String field) throws IOException { - return null; - } - - @Override - public FieldsEnum iterator() throws IOException { - return FieldsEnum.EMPTY; - } - - @Override - public void close() throws IOException { - - } - }; - } Index: lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/DocValuesWriterBase.java (working copy) @@ -33,19 +33,19 @@ * @lucene.experimental */ public abstract class DocValuesWriterBase extends PerDocConsumer { - private final String segmentName; - private final int codecId; + protected final String segmentName; + protected final String segmentSuffix; private final Counter bytesUsed; - private final IOContext context; + protected final IOContext context; protected DocValuesWriterBase(PerDocWriteState state) { this.segmentName = state.segmentName; - this.codecId = state.codecId; + this.segmentSuffix = state.segmentSuffix; this.bytesUsed = state.bytesUsed; this.context = state.context; } - protected abstract Directory getDirectory(); + protected abstract Directory getDirectory() throws IOException; @Override public void close() throws IOException { @@ -54,12 +54,12 @@ @Override public DocValuesConsumer addValuesField(FieldInfo field) throws IOException { return Writer.create(field.getDocValues(), - docValuesId(segmentName, codecId, field.number), + docValuesId(segmentName, field.number), getDirectory(), getComparator(), bytesUsed, context); } - public static String docValuesId(String segmentsName, int codecID, int fieldId) { - return segmentsName + "_" + codecID + "-" + fieldId; + public static String docValuesId(String segmentsName, int fieldId) { + return segmentsName + "_" + fieldId; } Index: lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java (working copy) @@ -68,14 +68,14 @@ // start of the field info data protected long dirOffset; - public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, int codecId, IOContext context) + public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, String segmentSuffix, IOContext context) throws IOException { this.termComp = termComp; assert indexDivisor == -1 || indexDivisor > 0; - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); + in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); boolean success = false; @@ -387,20 +387,11 @@ } } - public static void files(Directory dir, SegmentInfo info, int id, Collection files) { - files.add(IndexFileNames.segmentFileName(info.name, id, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + public static void files(Directory dir, SegmentInfo info, String segmentSuffix, Collection files) { + files.add(IndexFileNames.segmentFileName(info.name, segmentSuffix, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); } - public static void getIndexExtensions(Collection extensions) { - extensions.add(FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION); - } - @Override - public void getExtensions(Collection extensions) { - getIndexExtensions(extensions); - } - - @Override public void close() throws IOException { if (in != null && !indexLoaded) { in.close(); Index: lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java (working copy) @@ -47,7 +47,7 @@ final static BytesRef PAYLOAD = new BytesRef(" payload "); public SimpleTextFieldsWriter(SegmentWriteState state) throws IOException { - final String fileName = SimpleTextCodec.getPostingsFileName(state.segmentName, state.codecId); + final String fileName = SimpleTextPostingsFormat.getPostingsFileName(state.segmentName, state.segmentSuffix); out = state.directory.createOutput(fileName, state.context); } Index: lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (working copy) @@ -60,7 +60,7 @@ final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD; public SimpleTextFieldsReader(SegmentReadState state) throws IOException { - in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, state.codecId), state.context); + in = state.dir.openInput(SimpleTextPostingsFormat.getPostingsFileName(state.segmentInfo.name, state.segmentSuffix), state.context); fieldInfos = state.fieldInfos; } Index: lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextCodec.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListWriter.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/standard/DefaultSkipListReader.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/standard/package.html (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java (working copy) @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; @@ -33,9 +34,9 @@ * Read {@link SegmentInfos} data from a directory. * @param directory directory to read from * @param segmentsFileName name of the "segments_N" file - * @param codecs current codecs + * @param header input of "segments_N" file after reading preamble * @param infos empty instance to be populated with data * @throws IOException */ - public abstract void read(Directory directory, String segmentsFileName, CodecProvider codecs, SegmentInfos infos, IOContext context) throws IOException; + public abstract void read(Directory directory, String segmentsFileName, ChecksumIndexInput header, SegmentInfos infos, IOContext context) throws IOException; } Index: lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (working copy) @@ -36,6 +36,9 @@ * time. * @lucene.experimental */ +// TODO: find a better name; this defines the API that the +// terms dict impls use to talk to a postings impl. +// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer public abstract class PostingsReaderBase implements Closeable { public abstract void init(IndexInput termsIn) throws IOException; Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermEnum.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/TermBuffer.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReaderIndex.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermPositions.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/SegmentTermDocs.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfo.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/preflex/package.html (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java (working copy) @@ -56,7 +56,7 @@ private final FieldInfos fieldInfos; // unread public FixedGapTermsIndexWriter(SegmentWriteState state) throws IOException { - final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); + final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_INDEX_EXTENSION); termIndexInterval = state.termIndexInterval; out = state.directory.createOutput(indexFileName, state.context); boolean success = false; Index: lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/PerDocValues.java (working copy) @@ -28,7 +28,7 @@ * {@link PerDocConsumer} counterpart. *

* The {@link PerDocValues} API is accessible through the - * {@link Codec} - API providing per field consumers and producers for inverted + * {@link PostingsFormat} - API providing per field consumers and producers for inverted * data (terms, postings) as well as per-document data. * * @lucene.experimental Index: lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/PostingsWriterBase.java (working copy) @@ -27,6 +27,9 @@ * @lucene.experimental */ +// TODO: find a better name; this defines the API that the +// terms dict impls use to talk to a postings impl. +// TermsDict + PostingsReader/WriterBase == PostingsConsumer/Producer public abstract class PostingsWriterBase extends PostingsConsumer implements Closeable { public abstract void start(IndexOutput termsOut) throws IOException; Index: lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java (working copy) @@ -42,13 +42,13 @@ * phase commit" operations as described above. * @throws IOException */ - public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, SegmentInfos infos, IOContext context) throws IOException; + public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, String codecID, SegmentInfos infos, IOContext context) throws IOException; /** * First phase of the two-phase commit - ensure that all output can be * successfully written out. * @param out an instance of {@link IndexOutput} returned from a previous - * call to {@link #writeInfos(Directory, String, SegmentInfos, IOContext)}. + * call to {@link #writeInfos(Directory, String, String, SegmentInfos, IOContext)}. * @throws IOException */ public abstract void prepareCommit(IndexOutput out) throws IOException; @@ -57,7 +57,7 @@ * Second phase of the two-phase commit. In this step the output should be * finalized and closed. * @param out an instance of {@link IndexOutput} returned from a previous - * call to {@link #writeInfos(Directory, String, SegmentInfos, IOContext)}. + * call to {@link #writeInfos(Directory, String, String, SegmentInfos, IOContext)}. * @throws IOException */ public abstract void finishCommit(IndexOutput out) throws IOException; Index: lucene/src/java/org/apache/lucene/index/codecs/TermsIndexReaderBase.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/TermsIndexReaderBase.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/TermsIndexReaderBase.java (working copy) @@ -45,8 +45,6 @@ public abstract void close() throws IOException; - public abstract void getExtensions(Collection extensions); - public abstract boolean supportsOrd(); public abstract int getDivisor(); Index: lucene/src/java/org/apache/lucene/index/codecs/CoreCodecProvider.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java (working copy) @@ -57,9 +57,9 @@ protected long dirOffset; final String segment; - public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, int codecId, IOContext context) + public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, String segmentSuffix, IOContext context) throws IOException { - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true)); + in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), new IOContext(context, true)); this.segment = segment; boolean success = false; assert indexDivisor == -1 || indexDivisor > 0; @@ -215,20 +215,11 @@ } } - public static void files(Directory dir, SegmentInfo info, int id, Collection files) { - files.add(IndexFileNames.segmentFileName(info.name, id, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + public static void files(Directory dir, SegmentInfo info, String segmentSuffix, Collection files) { + files.add(IndexFileNames.segmentFileName(info.name, segmentSuffix, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); } - public static void getIndexExtensions(Collection extensions) { - extensions.add(VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION); - } - @Override - public void getExtensions(Collection extensions) { - getIndexExtensions(extensions); - } - - @Override public void close() throws IOException { if (in != null && !indexLoaded) { in.close(); Property changes on: lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java ___________________________________________________________________ Modified: svn:mergeinfo Reverse-merged /lucene/dev/trunk/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java:r1196666-1197230 Merged /lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java:r1188713-1197363 Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java (working copy) @@ -60,23 +60,23 @@ int maxSkipLevels; int skipMinimum; - public SepPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext context, IntStreamFactory intFactory, int codecId) throws IOException { + public SepPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext context, IntStreamFactory intFactory, String segmentSuffix) throws IOException { boolean success = false; try { - final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.DOC_EXTENSION); + final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.DOC_EXTENSION); docIn = intFactory.openInput(dir, docFileName, context); - skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.SKIP_EXTENSION), context); + skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.SKIP_EXTENSION), context); if (segmentInfo.getFieldInfos().hasFreq()) { - freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.FREQ_EXTENSION), context); + freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.FREQ_EXTENSION), context); } else { freqIn = null; } if (segmentInfo.getHasProx()) { - posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.POS_EXTENSION), context); - payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.PAYLOAD_EXTENSION), context); + posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.POS_EXTENSION), context); + payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.PAYLOAD_EXTENSION), context); } else { posIn = null; payloadIn = null; @@ -89,17 +89,17 @@ } } - public static void files(SegmentInfo segmentInfo, int codecId, Collection files) throws IOException { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.DOC_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.SKIP_EXTENSION)); + public static void files(SegmentInfo segmentInfo, String segmentSuffix, Collection files) throws IOException { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.DOC_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.SKIP_EXTENSION)); if (segmentInfo.getFieldInfos().hasFreq()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.FREQ_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.FREQ_EXTENSION)); } if (segmentInfo.getHasProx()) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.POS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriter.PAYLOAD_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.POS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, SepPostingsWriter.PAYLOAD_EXTENSION)); } } Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesProducer.java (working copy) @@ -39,7 +39,7 @@ * {@link IndexDocValues} instances for this segment and codec. */ public SepDocValuesProducer(SegmentReadState state) throws IOException { - docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, state.dir, state.codecId, state.context); + docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, state.dir, state.context); } @Override Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepDocValuesConsumer.java (working copy) @@ -47,12 +47,12 @@ } @SuppressWarnings("fallthrough") - public static void files(Directory dir, SegmentInfo segmentInfo, int codecId, + public static void files(Directory dir, SegmentInfo segmentInfo, Set files) throws IOException { FieldInfos fieldInfos = segmentInfo.getFieldInfos(); for (FieldInfo fieldInfo : fieldInfos) { - if (fieldInfo.getCodecId() == codecId && fieldInfo.hasDocValues()) { - String filename = docValuesId(segmentInfo.name, codecId, fieldInfo.number); + if (fieldInfo.hasDocValues()) { + String filename = docValuesId(segmentInfo.name, fieldInfo.number); switch (fieldInfo.getDocValues()) { case BYTES_FIXED_DEREF: case BYTES_VAR_DEREF: @@ -83,9 +83,4 @@ } } } - - public static void getExtensions(Set extensions) { - extensions.add(Writer.DATA_EXTENSION); - extensions.add(Writer.INDEX_EXTENSION); - } } Index: lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriter.java (working copy) @@ -116,27 +116,27 @@ try { this.skipInterval = skipInterval; this.skipMinimum = skipInterval; /* set to the same for now */ - final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, DOC_EXTENSION); + final String docFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, DOC_EXTENSION); docOut = factory.createOutput(state.directory, docFileName, state.context); docIndex = docOut.index(); if (state.fieldInfos.hasFreq()) { - final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, FREQ_EXTENSION); + final String frqFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, FREQ_EXTENSION); freqOut = factory.createOutput(state.directory, frqFileName, state.context); freqIndex = freqOut.index(); } if (state.fieldInfos.hasProx()) { - final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, POS_EXTENSION); + final String posFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, POS_EXTENSION); posOut = factory.createOutput(state.directory, posFileName, state.context); posIndex = posOut.index(); // TODO: -- only if at least one field stores payloads? - final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, PAYLOAD_EXTENSION); + final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, PAYLOAD_EXTENSION); payloadOut = state.directory.createOutput(payloadFileName, state.context); } - final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SKIP_EXTENSION); + final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, SKIP_EXTENSION); skipOut = state.directory.createOutput(skipFileName, state.context); totalNumDocs = state.numDocs; @@ -391,12 +391,4 @@ public void close() throws IOException { IOUtils.close(docOut, skipOut, freqOut, posOut, payloadOut); } - - public static void getExtensions(Set extensions) { - extensions.add(DOC_EXTENSION); - extensions.add(FREQ_EXTENSION); - extensions.add(SKIP_EXTENSION); - extensions.add(POS_EXTENSION); - extensions.add(PAYLOAD_EXTENSION); - } } Index: lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java (working copy) @@ -110,13 +110,13 @@ public BlockTreeTermsReader(Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext ioContext, - int codecId, int indexDivisor) + String segmentSuffix, int indexDivisor) throws IOException { this.postingsReader = postingsReader; this.segment = segment; - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTreeTermsWriter.TERMS_EXTENSION), + in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTreeTermsWriter.TERMS_EXTENSION), ioContext); boolean success = false; @@ -125,7 +125,7 @@ try { readHeader(in); if (indexDivisor != -1) { - indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION), + indexIn = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION), ioContext); readIndexHeader(indexIn); } @@ -206,16 +206,11 @@ } } - public static void files(Directory dir, SegmentInfo segmentInfo, int codecID, Collection files) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, BlockTreeTermsWriter.TERMS_EXTENSION)); - files.add(IndexFileNames.segmentFileName(segmentInfo.name, codecID, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION)); + public static void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTreeTermsWriter.TERMS_EXTENSION)); + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTreeTermsWriter.TERMS_INDEX_EXTENSION)); } - public static void getExtensions(Collection extensions) { - extensions.add(BlockTreeTermsWriter.TERMS_EXTENSION); - extensions.add(BlockTreeTermsWriter.TERMS_INDEX_EXTENSION); - } - @Override public FieldsEnum iterator() { return new TermFieldsEnum(); Index: lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java (working copy) @@ -158,7 +158,7 @@ // in the extremes. public VariableGapTermsIndexWriter(SegmentWriteState state, IndexTermSelector policy) throws IOException { - final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); + final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_INDEX_EXTENSION); out = state.directory.createOutput(indexFileName, state.context); boolean success = false; try { Index: lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsWriter.java (working copy) @@ -145,7 +145,7 @@ throw new IllegalArgumentException("maxItemsInBlock must be at least 2*(minItemsInBlock-1); got maxItemsInBlock=" + maxItemsInBlock + " minItemsInBlock=" + minItemsInBlock); } - final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); + final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_EXTENSION); out = state.directory.createOutput(termsFileName, state.context); boolean success = false; IndexOutput indexOut = null; @@ -157,7 +157,7 @@ //DEBUG = state.segmentName.equals("_4a"); - final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); + final String termsIndexFileName = IndexFileNames.segmentFileName(state.segmentName, state.segmentSuffix, TERMS_INDEX_EXTENSION); indexOut = state.directory.createOutput(termsIndexFileName, state.context); writeIndexHeader(indexOut); Index: lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java (working copy) @@ -109,14 +109,14 @@ // private String segment; public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext context, - int termsCacheSize, int codecId) + int termsCacheSize, String segmentSuffix) throws IOException { this.postingsReader = postingsReader; termsCache = new DoubleBarrelLRUCache(termsCacheSize); // this.segment = segment; - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTermsWriter.TERMS_EXTENSION), + in = dir.openInput(IndexFileNames.segmentFileName(segment, segmentSuffix, BlockTermsWriter.TERMS_EXTENSION), context); boolean success = false; @@ -194,14 +194,10 @@ } } - public static void files(Directory dir, SegmentInfo segmentInfo, int id, Collection files) { - files.add(IndexFileNames.segmentFileName(segmentInfo.name, id, BlockTermsWriter.TERMS_EXTENSION)); + public static void files(Directory dir, SegmentInfo segmentInfo, String segmentSuffix, Collection files) { + files.add(IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, BlockTermsWriter.TERMS_EXTENSION)); } - public static void getExtensions(Collection extensions) { - extensions.add(BlockTermsWriter.TERMS_EXTENSION); - } - @Override public FieldsEnum iterator() { return new TermFieldsEnum(); Index: lucene/src/java/org/apache/lucene/index/codecs/CodecProvider.java (deleted) =================================================================== Index: lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/PerDocConsumer.java (working copy) @@ -29,7 +29,7 @@ * this convert field values into a Codec specific format during indexing. *

* The {@link PerDocConsumer} API is accessible through the - * {@link Codec} - API providing per field consumers and producers for inverted + * {@link PostingsFormat} - API providing per field consumers and producers for inverted * data (terms, postings) as well as per-document data. * * @lucene.experimental Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java (working copy) @@ -19,17 +19,14 @@ import java.io.IOException; -import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFormatTooOldException; -import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IOContext; -import org.apache.lucene.store.IndexInput; /** * Default implementation of {@link SegmentInfosReader}. @@ -37,90 +34,60 @@ */ public class DefaultSegmentInfosReader extends SegmentInfosReader { + // TODO: shove all backwards code to preflex! + // this is a little tricky, because of IR.commit(), two options: + // 1. PreFlex writes 4.x SIS format, but reads both 3.x and 4.x + // (and maybe RW always only writes the 3.x one? for that to work well, + // we have to move .fnx file to codec too, not too bad but more work). + // or we just have crappier RW testing like today. + // 2. PreFlex writes 3.x SIS format, and only reads 3.x + // (in this case we have to move .fnx file to codec as well) @Override - public void read(Directory directory, String segmentsFileName, CodecProvider codecs, - SegmentInfos infos, IOContext context) throws IOException { - IndexInput input = null; - try { - input = openInput(directory, segmentsFileName, context); - final int format = input.readInt(); - infos.setFormat(format); - - // check that it is a format we can understand - if (format > DefaultSegmentInfosWriter.FORMAT_MINIMUM) - throw new IndexFormatTooOldException(segmentsFileName, format, - DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); - if (format < DefaultSegmentInfosWriter.FORMAT_CURRENT) - throw new IndexFormatTooNewException(segmentsFileName, format, - DefaultSegmentInfosWriter.FORMAT_MINIMUM, DefaultSegmentInfosWriter.FORMAT_CURRENT); - - infos.version = input.readLong(); // read version - infos.counter = input.readInt(); // read counter - if (infos.getFormat() <= DefaultSegmentInfosWriter.FORMAT_4_0) { - infos.setGlobalFieldMapVersion(input.readLong()); - } - for (int i = input.readInt(); i > 0; i--) { // read segmentInfos - SegmentInfo si = new SegmentInfo(directory, format, input, codecs); - if (si.getVersion() == null) { - // Could be a 3.0 - try to open the doc stores - if it fails, it's a - // 2.x segment, and an IndexFormatTooOldException will be thrown, - // which is what we want. - Directory dir = directory; - if (si.getDocStoreOffset() != -1) { - if (si.getDocStoreIsCompoundFile()) { - dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName( - si.getDocStoreSegment(), "", - IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false); - } - } else if (si.getUseCompoundFile()) { - dir = new CompoundFileDirectory(dir,IndexFileNames.segmentFileName( - si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); + public void read(Directory directory, String segmentsFileName, ChecksumIndexInput input, SegmentInfos infos, IOContext context) throws IOException { + infos.version = input.readLong(); // read version + infos.counter = input.readInt(); // read counter + final int format = infos.getFormat(); + if (format <= DefaultSegmentInfosWriter.FORMAT_4_0) { + infos.setGlobalFieldMapVersion(input.readLong()); + } + for (int i = input.readInt(); i > 0; i--) { // read segmentInfos + SegmentInfo si = new SegmentInfo(directory, format, input); + if (si.getVersion() == null) { + // Could be a 3.0 - try to open the doc stores - if it fails, it's a + // 2.x segment, and an IndexFormatTooOldException will be thrown, + // which is what we want. + Directory dir = directory; + if (si.getDocStoreOffset() != -1) { + if (si.getDocStoreIsCompoundFile()) { + dir = new CompoundFileDirectory(dir, IndexFileNames.segmentFileName( + si.getDocStoreSegment(), "", + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context, false); } + } else if (si.getUseCompoundFile()) { + dir = new CompoundFileDirectory(dir,IndexFileNames.segmentFileName( + si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context, false); + } - try { - DefaultFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment()); - } finally { - // If we opened the directory, close it - if (dir != directory) dir.close(); - } - - // Above call succeeded, so it's a 3.0 segment. Upgrade it so the next - // time the segment is read, its version won't be null and we won't - // need to open FieldsReader every time for each such segment. - si.setVersion("3.0"); - } else if (si.getVersion().equals("2.x")) { - // If it's a 3x index touched by 3.1+ code, then segments record their - // version, whether they are 2.x ones or not. We detect that and throw - // appropriate exception. - throw new IndexFormatTooOldException(si.name, si.getVersion()); + try { + DefaultFieldsReader.checkCodeVersion(dir, si.getDocStoreSegment()); + } finally { + // If we opened the directory, close it + if (dir != directory) dir.close(); } - infos.add(si); + + // Above call succeeded, so it's a 3.0 segment. Upgrade it so the next + // time the segment is read, its version won't be null and we won't + // need to open FieldsReader every time for each such segment. + si.setVersion("3.0"); + } else if (si.getVersion().equals("2.x")) { + // If it's a 3x index touched by 3.1+ code, then segments record their + // version, whether they are 2.x ones or not. We detect that and throw + // appropriate exception. + throw new IndexFormatTooOldException(si.name, si.getVersion()); } - - infos.userData = input.readStringStringMap(); - finalizeInput(input); - - } finally { - if (input != null) { - input.close(); - } + infos.add(si); } - + + infos.userData = input.readStringStringMap(); } - - public IndexInput openInput(Directory dir, String segmentsFileName, IOContext context) throws IOException { - IndexInput in = dir.openInput(segmentsFileName, context); - return new ChecksumIndexInput(in); - - } - - public void finalizeInput(IndexInput input) throws IOException, CorruptIndexException { - ChecksumIndexInput cksumInput = (ChecksumIndexInput)input; - final long checksumNow = cksumInput.getChecksum(); - final long checksumThen = cksumInput.readLong(); - if (checksumNow != checksumThen) - throw new CorruptIndexException("checksum mismatch in segments file"); - - } - } Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java (working copy) @@ -26,7 +26,6 @@ import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentReadState; -import org.apache.lucene.index.codecs.DocValuesReaderBase; import org.apache.lucene.index.values.IndexDocValues; import org.apache.lucene.store.CompoundFileDirectory; import org.apache.lucene.store.Directory; @@ -37,7 +36,7 @@ * @lucene.experimental */ public class DefaultDocValuesProducer extends DocValuesReaderBase { - protected final TreeMap docValues; + protected final TreeMap docValues; private final Directory cfs; /** @@ -45,10 +44,16 @@ * {@link IndexDocValues} instances for this segment and codec. */ public DefaultDocValuesProducer(SegmentReadState state) throws IOException { - cfs = new CompoundFileDirectory(state.dir, - IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, IndexFileNames.COMPOUND_FILE_EXTENSION), - state.context, false); - docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.codecId, state.context); + if (state.fieldInfos.anyDocValuesFields()) { + cfs = new CompoundFileDirectory(state.dir, + IndexFileNames.segmentFileName(state.segmentInfo.name, + DefaultDocValuesConsumer.DOC_VALUES_SEGMENT_SUFFIX, IndexFileNames.COMPOUND_FILE_EXTENSION), + state.context, false); + docValues = load(state.fieldInfos, state.segmentInfo.name, state.segmentInfo.docCount, cfs, state.context); + } else { + cfs = null; + docValues = new TreeMap(); + } } @Override @@ -58,8 +63,12 @@ @Override protected void closeInternal(Collection closeables) throws IOException { - final ArrayList list = new ArrayList(closeables); - list.add(cfs); - IOUtils.close(list); + if (cfs != null) { + final ArrayList list = new ArrayList(closeables); + list.add(cfs); + IOUtils.close(list); + } else { + IOUtils.close(closeables); + } } } Index: lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java =================================================================== --- lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/codecs/DefaultFieldsReader.java (working copy) @@ -82,7 +82,7 @@ /** Verifies that the code version which wrote the segment is supported. */ public static void checkCodeVersion(Directory dir, String segment) throws IOException { - final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); + final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_INDEX_EXTENSION); IndexInput idxStream = dir.openInput(indexStreamFN, IOContext.DEFAULT); try { @@ -121,8 +121,8 @@ try { fieldInfos = fn; - cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); - final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); + cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_EXTENSION), context); + final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", DefaultFieldsWriter.FIELDS_INDEX_EXTENSION); cloneableIndexStream = d.openInput(indexStreamFN, context); format = cloneableIndexStream.readInt(); Index: lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java =================================================================== --- lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java (working copy) @@ -21,6 +21,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.index.codecs.Codec; +import org.apache.lucene.index.codecs.PostingsFormat; import org.apache.lucene.index.codecs.FieldsProducer; import org.apache.lucene.index.codecs.FieldsReader; import org.apache.lucene.index.codecs.PerDocValues; @@ -68,7 +69,7 @@ } segment = si.name; - final SegmentCodecs segmentCodecs = si.getSegmentCodecs(); + final Codec codec = si.getCodec(); this.context = context; this.dir = dir; @@ -85,12 +86,12 @@ fieldInfos = si.getFieldInfos(); this.termsIndexDivisor = termsIndexDivisor; - final Codec codec = segmentCodecs.codec(); + final PostingsFormat format = codec.postingsFormat(); final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, context, termsIndexDivisor); // Ask codec for its Fields - fields = codec.fieldsProducer(segmentReadState); + fields = format.fieldsProducer(segmentReadState); assert fields != null; - perDocProducer = codec.docsProducer(segmentReadState); + perDocProducer = codec.docValuesFormat().docsProducer(segmentReadState); success = true; } finally { if (!success) { @@ -165,7 +166,7 @@ } final String storesSegment = si.getDocStoreSegment(); - fieldsReaderOrig = si.getSegmentCodecs().provider.fieldsReader(storeDir, storesSegment, fieldInfos, context, + fieldsReaderOrig = si.getCodec().fieldsFormat().fieldsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount); // Verify two sources of "maxDoc" agree: Index: lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java (working copy) @@ -20,7 +20,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.index.DocumentsWriterPerThread.IndexingChain; import org.apache.lucene.index.IndexWriter.IndexReaderWarmer; -import org.apache.lucene.index.codecs.CodecProvider; +import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.similarities.SimilarityProvider; import org.apache.lucene.util.Version; @@ -121,7 +121,7 @@ private volatile int maxBufferedDocs; private volatile IndexingChain indexingChain; private volatile IndexReaderWarmer mergedSegmentWarmer; - private volatile CodecProvider codecProvider; + private volatile Codec codec; private volatile MergePolicy mergePolicy; private volatile DocumentsWriterPerThreadPool indexerThreadPool; private volatile boolean readerPooling; @@ -158,7 +158,7 @@ maxBufferedDocs = DEFAULT_MAX_BUFFERED_DOCS; indexingChain = DocumentsWriterPerThread.defaultIndexingChain; mergedSegmentWarmer = null; - codecProvider = CodecProvider.getDefault(); + codec = Codec.getDefault(); if (matchVersion.onOrAfter(Version.LUCENE_32)) { mergePolicy = new TieredMergePolicy(); } else { @@ -521,17 +521,17 @@ return this; } - /** Set the CodecProvider. See {@link CodecProvider}. + /** Set the Codec. See {@link Codec}. * *

Only takes effect when IndexWriter is first created. */ - public IndexWriterConfig setCodecProvider(CodecProvider codecProvider) { - this.codecProvider = codecProvider; + public IndexWriterConfig setCodec(Codec codec) { + this.codec = codec; return this; } - /** Returns the current merged segment warmer. See {@link IndexReaderWarmer}. */ - public CodecProvider getCodecProvider() { - return codecProvider; + /** Returns the current Codec. See {@link Codec}. */ + public Codec getCodec() { + return codec; } @@ -694,7 +694,7 @@ sb.append("ramBufferSizeMB=").append(ramBufferSizeMB).append("\n"); sb.append("maxBufferedDocs=").append(maxBufferedDocs).append("\n"); sb.append("mergedSegmentWarmer=").append(mergedSegmentWarmer).append("\n"); - sb.append("codecProvider=").append(codecProvider).append("\n"); + sb.append("codec=").append(codec).append("\n"); sb.append("mergePolicy=").append(mergePolicy).append("\n"); sb.append("indexerThreadPool=").append(indexerThreadPool).append("\n"); sb.append("readerPooling=").append(readerPooling).append("\n"); Index: lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java =================================================================== --- lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/store/CompoundFileDirectory.java (working copy) @@ -106,7 +106,9 @@ numEntries); for (int i = 0; i < numEntries; i++) { final FileEntry fileEntry = new FileEntry(); - mapping.put(input.readString(), fileEntry); + final String id = input.readString(); + assert !mapping.containsKey(id): "id=" + id + " was written multiple times in the CFS"; + mapping.put(id, fileEntry); fileEntry.offset = input.readLong(); fileEntry.length = input.readLong(); } @@ -170,6 +172,9 @@ entry = new FileEntry(); entry.offset = offset; + + assert !entries.containsKey(id); + entries.put(id, entry); } @@ -271,7 +276,7 @@ public long fileLength(String name) throws IOException { ensureOpen(); if (this.writer != null) { - return writer.fileLenght(name); + return writer.fileLength(name); } FileEntry e = entries.get(IndexFileNames.stripSegmentName(name)); if (e == null) @@ -323,4 +328,9 @@ } }; } + + @Override + public String toString() { + return "CompoundFileDirectory(file=\"" + fileName + "\" in dir=" + directory + ")"; + } } Index: lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/store/CompoundFileWriter.java (working copy) @@ -22,9 +22,11 @@ import java.io.IOException; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.LinkedList; import java.util.Map; import java.util.Queue; +import java.util.Set; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.lucene.index.IndexFileNames; @@ -85,6 +87,7 @@ private final Directory directory; private final Map entries = new HashMap(); + private final Set seenIDs = new HashSet(); // all entries that are written to a sep. file but not yet moved into CFS private final Queue pendingEntries = new LinkedList(); private boolean closed = false; @@ -238,6 +241,9 @@ final FileEntry entry = new FileEntry(); entry.file = name; entries.put(name, entry); + final String id = IndexFileNames.stripSegmentName(name); + assert !seenIDs.contains(id): "file=\"" + name + "\" maps to id=\"" + id + "\", which was already written"; + seenIDs.add(id); final DirectCFSIndexOutput out; if (outputTaken.compareAndSet(false, true)) { out = new DirectCFSIndexOutput(dataOut, entry, false); @@ -284,7 +290,7 @@ } } - long fileLenght(String name) throws IOException { + long fileLength(String name) throws IOException { FileEntry fileEntry = entries.get(name); if (fileEntry == null) { throw new FileNotFoundException(name + " does not exist"); Index: lucene/src/java/org/apache/lucene/util/CodecUtil.java =================================================================== --- lucene/src/java/org/apache/lucene/util/CodecUtil.java (revision 1197230) +++ lucene/src/java/org/apache/lucene/util/CodecUtil.java (working copy) @@ -35,7 +35,7 @@ private final static int CODEC_MAGIC = 0x3fd76c17; - public static DataOutput writeHeader(DataOutput out, String codec, int version) + public static void writeHeader(DataOutput out, String codec, int version) throws IOException { BytesRef bytes = new BytesRef(codec); if (bytes.length != codec.length() || bytes.length >= 128) { @@ -44,8 +44,6 @@ out.writeInt(CODEC_MAGIC); out.writeString(codec); out.writeInt(version); - - return out; } public static int headerLength(String codec) { Index: lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/analysis/CollationTestBase.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/analysis/MockTokenFilter.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/analysis/MockVariableLengthPayloadFilter.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/analysis/VocabularyAssert.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/analysis/MockFixedLengthPayloadFilter.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/search/RandomSimilarityProvider.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/search/CachingWrapperFilterHelper.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/search/CheckHits.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/MockIndexInput.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/DocHelper.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/RandomCodecProvider.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/ThreadedIndexingAndSearchingTestCase.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/index/MockRandomMergePolicy.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/store/MockIndexInputWrapper.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/store/MockIndexOutputWrapper.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/store/MockLockFactoryWrapper.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/store/_TestHelper.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/europarl.lines.txt.gz (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/LineFileDocs.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/LuceneTestCaseRunner.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/ThrottledIndexOutput.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitDividingSelector.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/_TestIgnoredException.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/automaton/DaciukMihovAutomatonBuilder.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/automaton/AutomatonTestUtil.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/SmartRandom.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/LuceneJUnitResultFormatter.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/ThreeLongs.java (deleted) =================================================================== Index: lucene/src/test-framework/org/apache/lucene/util/_TestUtil.java (deleted) =================================================================== Index: lucene/src/test-framework/overview.html (deleted) =================================================================== Index: lucene/build.xml =================================================================== --- lucene/build.xml (revision 1197230) +++ lucene/build.xml (working copy) @@ -30,15 +30,15 @@ - + + - + - @@ -556,11 +556,11 @@ - + Property changes on: dev-tools/idea/lucene/contrib ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/branches/lucene2621/dev-tools/idea/lucene/contrib:r1188713-1197363 Index: dev-tools/eclipse/dot.classpath =================================================================== --- dev-tools/eclipse/dot.classpath (revision 1197230) +++ dev-tools/eclipse/dot.classpath (working copy) @@ -1,7 +1,9 @@ + + - + @@ -13,6 +15,7 @@ +