Property changes on: . ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk:r1040463 Property changes on: solr ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/solr:r1040463 Index: solr/src/test/org/apache/solr/core/TestConfig.java =================================================================== --- solr/src/test/org/apache/solr/core/TestConfig.java (revision 1040477) +++ solr/src/test/org/apache/solr/core/TestConfig.java (working copy) @@ -133,7 +133,7 @@ ExposeWriterHandler duh = new ExposeWriterHandler(); IndexWriter writer = duh.getWriter(); - int interval = writer.getTermIndexInterval(); + int interval = writer.getConfig().getTermIndexInterval(); assertEquals(256, interval); duh.close(); } Index: solr/src/test/org/apache/solr/core/TestPropInjectDefaults.java =================================================================== --- solr/src/test/org/apache/solr/core/TestPropInjectDefaults.java (revision 1040477) +++ solr/src/test/org/apache/solr/core/TestPropInjectDefaults.java (working copy) @@ -48,7 +48,7 @@ public void testMergePolicyDefaults() throws Exception { ExposeWriterHandler uh = new ExposeWriterHandler(); IndexWriter writer = uh.getWriter(); - LogByteSizeMergePolicy mp = (LogByteSizeMergePolicy)writer.getMergePolicy(); + LogByteSizeMergePolicy mp = (LogByteSizeMergePolicy)writer.getConfig().getMergePolicy(); assertEquals(32.0, mp.getMaxMergeMB()); uh.close(); } @@ -57,7 +57,7 @@ public void testPropsDefaults() throws Exception { ExposeWriterHandler uh = new ExposeWriterHandler(); IndexWriter writer = uh.getWriter(); - ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler)writer.getMergeScheduler(); + ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler)writer.getConfig().getMergeScheduler(); assertEquals(4, cms.getMaxThreadCount()); uh.close(); } Index: solr/src/test/org/apache/solr/core/TestPropInject.java =================================================================== --- solr/src/test/org/apache/solr/core/TestPropInject.java (revision 1040477) +++ solr/src/test/org/apache/solr/core/TestPropInject.java (working copy) @@ -31,7 +31,7 @@ public void testMergePolicy() throws Exception { ExposeWriterHandler uh = new ExposeWriterHandler(); IndexWriter writer = uh.getWriter(); - LogByteSizeMergePolicy mp = (LogByteSizeMergePolicy)writer.getMergePolicy(); + LogByteSizeMergePolicy mp = (LogByteSizeMergePolicy)writer.getConfig().getMergePolicy(); assertEquals(64.0, mp.getMaxMergeMB()); uh.close(); } @@ -39,7 +39,7 @@ public void testProps() throws Exception { ExposeWriterHandler uh = new ExposeWriterHandler(); IndexWriter writer = uh.getWriter(); - ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler)writer.getMergeScheduler(); + ConcurrentMergeScheduler cms = (ConcurrentMergeScheduler)writer.getConfig().getMergeScheduler(); assertEquals(2, cms.getMaxThreadCount()); uh.close(); } Index: solr/src/test/org/apache/solr/core/TestArbitraryIndexDir.java =================================================================== --- solr/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (revision 1040477) +++ solr/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (working copy) @@ -27,11 +27,8 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexWriter.MaxFieldLength; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.queryParser.ParseException; -import org.apache.lucene.queryParser.QueryParser; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.Query; import org.apache.lucene.store.Directory; import org.apache.lucene.store.FSDirectory; import org.apache.lucene.util.Version; @@ -101,7 +98,11 @@ //add a doc in the new index dir Directory dir = FSDirectory.open(newDir); - IndexWriter iw = new IndexWriter(dir, new StandardAnalyzer(Version.LUCENE_24), new MaxFieldLength(1000)); + IndexWriter iw = new IndexWriter( + dir, + new IndexWriterConfig(TEST_VERSION_CURRENT, new StandardAnalyzer(TEST_VERSION_CURRENT)). + setMaxFieldLength(1000) + ); Document doc = new Document(); doc.add(new Field("id", "2", Field.Store.YES, Field.Index.ANALYZED)); doc.add(new Field("name", "name2", Field.Store.YES, Field.Index.ANALYZED)); Index: solr/src/test/org/apache/solr/core/TestLegacyMergeSchedulerPolicyConfig.java =================================================================== --- solr/src/test/org/apache/solr/core/TestLegacyMergeSchedulerPolicyConfig.java (revision 1040477) +++ solr/src/test/org/apache/solr/core/TestLegacyMergeSchedulerPolicyConfig.java (working copy) @@ -20,8 +20,8 @@ public void testLegacy() throws Exception { ExposeWriterHandler duh = new ExposeWriterHandler(); IndexWriter writer = duh.getWriter(); - assertTrue(writer.getMergePolicy().getClass().getName().equals(LogDocMergePolicy.class.getName())); - assertTrue(writer.getMergeScheduler().getClass().getName().equals(SerialMergeScheduler.class.getName())); + assertTrue(writer.getConfig().getMergePolicy().getClass().getName().equals(LogDocMergePolicy.class.getName())); + assertTrue(writer.getConfig().getMergeScheduler().getClass().getName().equals(SerialMergeScheduler.class.getName())); duh.close(); } Index: solr/src/test/org/apache/solr/search/TestSort.java =================================================================== --- solr/src/test/org/apache/solr/search/TestSort.java (revision 1040477) +++ solr/src/test/org/apache/solr/search/TestSort.java (working copy) @@ -22,6 +22,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.search.*; import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; @@ -59,7 +60,12 @@ Field f2 = new Field("f2","0", Field.Store.NO, Field.Index.NOT_ANALYZED_NO_NORMS); for (int iterCnt = 0; iterCnt convert(String origQuery) { Collection result = new HashSet(); - WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(); + WhitespaceAnalyzer analyzer = new WhitespaceAnalyzer(Version.LUCENE_31); TokenStream ts = analyzer.tokenStream("", new StringReader(origQuery)); // TODO: support custom attributes CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); Index: solr/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java =================================================================== --- solr/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java (revision 1040477) +++ solr/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java (working copy) @@ -24,6 +24,7 @@ import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.search.spell.JaroWinklerDistance; import org.apache.lucene.search.spell.SpellChecker; import org.apache.lucene.search.spell.StringDistance; @@ -283,7 +284,11 @@ File indexDir = new File(TEMP_DIR, "spellingIdx" + new Date().getTime()); //create a standalone index File altIndexDir = new File(TEMP_DIR, "alternateIdx" + new Date().getTime()); - IndexWriter iw = new IndexWriter(FSDirectory.open(altIndexDir), new WhitespaceAnalyzer(), IndexWriter.MaxFieldLength.LIMITED); + IndexWriter iw = new IndexWriter( + FSDirectory.open(altIndexDir), + new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)). + setMaxFieldLength(IndexWriterConfig.UNLIMITED_FIELD_LENGTH) + ); for (int i = 0; i < ALT_DOCS.length; i++) { Document doc = new Document(); doc.add(new Field("title", ALT_DOCS[i], Field.Store.YES, Field.Index.ANALYZED)); Index: solr/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java =================================================================== --- solr/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java (revision 1040477) +++ solr/src/test/org/apache/solr/spelling/SpellingQueryConverterTest.java (working copy) @@ -21,13 +21,9 @@ import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.util.LuceneTestCase; import org.apache.solr.common.util.NamedList; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; import org.junit.Test; -import org.junit.Assert; import java.util.Collection; -import java.util.ArrayList; /** @@ -42,7 +38,7 @@ public void test() throws Exception { SpellingQueryConverter converter = new SpellingQueryConverter(); converter.init(new NamedList()); - converter.setAnalyzer(new WhitespaceAnalyzer()); + converter.setAnalyzer(new WhitespaceAnalyzer(TEST_VERSION_CURRENT)); Collection tokens = converter.convert("field:foo"); assertTrue("tokens is null and it shouldn't be", tokens != null); assertTrue("tokens Size: " + tokens.size() + " is not: " + 1, tokens.size() == 1); @@ -52,7 +48,7 @@ public void testSpecialChars() { SpellingQueryConverter converter = new SpellingQueryConverter(); converter.init(new NamedList()); - converter.setAnalyzer(new WhitespaceAnalyzer()); + converter.setAnalyzer(new WhitespaceAnalyzer(TEST_VERSION_CURRENT)); String original = "field_with_underscore:value_with_underscore"; Collection tokens = converter.convert(original); assertTrue("tokens is null and it shouldn't be", tokens != null); @@ -98,7 +94,7 @@ public void testUnicode() { SpellingQueryConverter converter = new SpellingQueryConverter(); converter.init(new NamedList()); - converter.setAnalyzer(new WhitespaceAnalyzer()); + converter.setAnalyzer(new WhitespaceAnalyzer(TEST_VERSION_CURRENT)); // chinese text value Collection tokens = converter.convert("text_field:我购买了道具和服装。"); @@ -118,7 +114,7 @@ public void testMultipleClauses() { SpellingQueryConverter converter = new SpellingQueryConverter(); converter.init(new NamedList()); - converter.setAnalyzer(new WhitespaceAnalyzer()); + converter.setAnalyzer(new WhitespaceAnalyzer(TEST_VERSION_CURRENT)); // two field:value pairs should give two tokens Collection tokens = converter.convert("买text_field:我购买了道具和服装。 field2:bar"); Index: solr/src/java/org/apache/solr/schema/TrieDateField.java =================================================================== --- solr/src/java/org/apache/solr/schema/TrieDateField.java (revision 1040477) +++ solr/src/java/org/apache/solr/schema/TrieDateField.java (working copy) @@ -174,7 +174,7 @@ Field f; if (stored) { - f = new Field(field.getName(), arr, Field.Store.YES); + f = new Field(field.getName(), arr); if (indexed) f.setTokenStream(ts); } else { f = new Field(field.getName(), ts); Index: solr/src/java/org/apache/solr/schema/TrieField.java =================================================================== --- solr/src/java/org/apache/solr/schema/TrieField.java (revision 1040477) +++ solr/src/java/org/apache/solr/schema/TrieField.java (working copy) @@ -444,7 +444,7 @@ Field f; if (stored) { - f = new Field(field.getName(), arr, Field.Store.YES); + f = new Field(field.getName(), arr); if (indexed) f.setTokenStream(ts); } else { f = new Field(field.getName(), ts); Index: solr/src/java/org/apache/solr/schema/BinaryField.java =================================================================== --- solr/src/java/org/apache/solr/schema/BinaryField.java (revision 1040477) +++ solr/src/java/org/apache/solr/schema/BinaryField.java (working copy) @@ -79,8 +79,7 @@ len = buf.length; } - Field f = new Field(field.getName(), buf, offset, len, - getFieldStore(field, null)); + Field f = new Field(field.getName(), buf, offset, len); f.setBoost(boost); return f; } Index: solr/src/java/org/apache/solr/update/SolrIndexWriter.java =================================================================== --- solr/src/java/org/apache/solr/update/SolrIndexWriter.java (revision 1040477) +++ solr/src/java/org/apache/solr/update/SolrIndexWriter.java (working copy) @@ -20,11 +20,8 @@ import org.apache.lucene.index.*; import org.apache.lucene.store.*; import org.apache.solr.common.SolrException; -import org.apache.solr.common.util.NamedList; import org.apache.solr.core.DirectoryFactory; -import org.apache.solr.core.StandardDirectoryFactory; import org.apache.solr.schema.IndexSchema; -import org.apache.solr.util.SolrPluginUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,73 +42,12 @@ * @since solr 0.9 */ - public class SolrIndexWriter extends IndexWriter { private static Logger log = LoggerFactory.getLogger(SolrIndexWriter.class); String name; - IndexSchema schema; - private PrintStream infoStream; - private void init(String name, IndexSchema schema, SolrIndexConfig config) throws IOException { - log.debug("Opened Writer " + name); - this.name = name; - this.schema = schema; - setSimilarity(schema.getSimilarity()); - // setUseCompoundFile(false); - - if (config != null) { - //only set maxBufferedDocs - if (config.maxBufferedDocs != -1) { - setMaxBufferedDocs(config.maxBufferedDocs); - } - if (config.ramBufferSizeMB != -1) { - setRAMBufferSizeMB(config.ramBufferSizeMB); - } - if (config.termIndexInterval != -1) { - setTermIndexInterval(config.termIndexInterval); - - } - if (config.maxMergeDocs != -1) setMaxMergeDocs(config.maxMergeDocs); - if (config.maxFieldLength != -1) setMaxFieldLength(config.maxFieldLength); - String className = config.mergePolicyInfo == null ? SolrIndexConfig.DEFAULT_MERGE_POLICY_CLASSNAME: config.mergePolicyInfo.className; - MergePolicy policy = null; - try { - policy = (MergePolicy) schema.getResourceLoader().newInstance(className, null, new Class[]{IndexWriter.class}, new Object[] { this }); - } catch (Exception e) { - policy = (MergePolicy) schema.getResourceLoader().newInstance(className); - } - if(config.mergePolicyInfo != null) SolrPluginUtils.invokeSetters(policy,config.mergePolicyInfo.initArgs); - setMergePolicy(policy); - - if (getMergePolicy() instanceof LogMergePolicy) { - setUseCompoundFile(config.useCompoundFile); - if (config.mergeFactor != -1) { setMergeFactor(config.mergeFactor); } - } else { - log.warn("Use of compound file format or mergefactor cannot be configured if merge policy is not an instance " + - "of LogMergePolicy. The configured policy's defaults will be used."); - } - - className = config.mergeSchedulerInfo == null ? SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME: config.mergeSchedulerInfo.className; - MergeScheduler scheduler = (MergeScheduler) schema.getResourceLoader().newInstance(className); - if(config.mergeSchedulerInfo != null) SolrPluginUtils.invokeSetters(scheduler,config.mergeSchedulerInfo.initArgs); - setMergeScheduler(scheduler); - - String infoStreamFile = config.infoStreamFile; - if (infoStreamFile != null) { - File f = new File(infoStreamFile); - File parent = f.getParentFile(); - if (parent != null) parent.mkdirs(); - FileOutputStream fos = new FileOutputStream(f, true); - infoStream = new TimeLoggingPrintStream(fos, true); - setInfoStream(infoStream); - } - //if (config.commitLockTimeout != -1) setWriteLockTimeout(config.commitLockTimeout); - } - - } - public static Directory getDirectory(String path, DirectoryFactory directoryFactory, SolrIndexConfig config) throws IOException { Directory d = directoryFactory.open(path); @@ -135,7 +71,7 @@ } else if ("none".equals(lockType)) { // Recipe for disaster log.error("CONFIGURATION WARNING: locks are disabled on " + path); - d.setLockFactory(new NoLockFactory()); + d.setLockFactory(NoLockFactory.getNoLockFactory()); } else { throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unrecognized lockType: " + rawLockType); @@ -143,62 +79,29 @@ return d; } - /** @deprecated remove when getDirectory(String,SolrIndexConfig) is gone */ - @Deprecated - private static DirectoryFactory LEGACY_DIR_FACTORY - = new StandardDirectoryFactory(); - static { - LEGACY_DIR_FACTORY.init(new NamedList()); - } + public SolrIndexWriter(String name, String path, DirectoryFactory dirFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy) throws IOException { + super( + getDirectory(path, dirFactory, config), + config.toIndexWriterConfig(schema). + setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND). + setIndexDeletionPolicy(delPolicy) + ); + log.debug("Opened Writer " + name); + this.name = name; - /** - * @deprecated use getDirectory(String path, DirectoryFactory directoryFactory, SolrIndexConfig config) - */ - @Deprecated - public static Directory getDirectory(String path, SolrIndexConfig config) throws IOException { - log.warn("SolrIndexWriter is using LEGACY_DIR_FACTORY which means deprecated code is likely in use and SolrIndexWriter is ignoring any custom DirectoryFactory."); - return getDirectory(path, LEGACY_DIR_FACTORY, config); + String infoStreamFile = config.infoStreamFile; + if (infoStreamFile != null) { + File f = new File(infoStreamFile); + File parent = f.getParentFile(); + if (parent != null) parent.mkdirs(); + FileOutputStream fos = new FileOutputStream(f, true); + infoStream = new TimeLoggingPrintStream(fos, true); + setInfoStream(infoStream); + } } - - /** - * - */ - public SolrIndexWriter(String name, String path, DirectoryFactory dirFactory, boolean create, IndexSchema schema) throws IOException { - super(getDirectory(path, dirFactory, null), schema.getAnalyzer(), create, MaxFieldLength.LIMITED); - init(name, schema, null); - } - @Deprecated - public SolrIndexWriter(String name, String path, DirectoryFactory dirFactory, boolean create, IndexSchema schema, SolrIndexConfig config) throws IOException { - super(getDirectory(path, dirFactory, null), schema.getAnalyzer(), create, MaxFieldLength.LIMITED); - init(name, schema, config); - } - - /** - * @deprecated - */ - @Deprecated - public SolrIndexWriter(String name, String path, boolean create, IndexSchema schema) throws IOException { - super(getDirectory(path, null), schema.getAnalyzer(), create, MaxFieldLength.LIMITED); - init(name, schema, null); - } /** - * @deprecated - */ - @Deprecated - public SolrIndexWriter(String name, String path, boolean create, IndexSchema schema, SolrIndexConfig config) throws IOException { - super(getDirectory(path, config), schema.getAnalyzer(), create, MaxFieldLength.LIMITED); - init(name, schema, config); - } - - public SolrIndexWriter(String name, String path, DirectoryFactory dirFactory, boolean create, IndexSchema schema, SolrIndexConfig config, IndexDeletionPolicy delPolicy) throws IOException { - super(getDirectory(path, dirFactory, config), schema.getAnalyzer(), create, delPolicy, new MaxFieldLength(IndexWriter.DEFAULT_MAX_FIELD_LENGTH)); - init(name, schema, config); - } - - - /** * use DocumentBuilder now... * private final void addField(Document doc, String name, String val) { * SchemaField ftype = schema.getField(name); @@ -280,5 +183,4 @@ super.println(x); } } - } Index: solr/src/java/org/apache/solr/update/SolrIndexConfig.java =================================================================== --- solr/src/java/org/apache/solr/update/SolrIndexConfig.java (revision 1040477) +++ solr/src/java/org/apache/solr/update/SolrIndexConfig.java (working copy) @@ -17,11 +17,12 @@ package org.apache.solr.update; +import org.apache.lucene.index.*; +import org.apache.lucene.util.Version; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.PluginInfo; -import org.apache.lucene.index.LogByteSizeMergePolicy; -import org.apache.lucene.index.ConcurrentMergeScheduler; -import org.apache.lucene.index.IndexWriter; +import org.apache.solr.schema.IndexSchema; +import org.apache.solr.util.SolrPluginUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -45,8 +46,8 @@ public static final String DEFAULT_MERGE_SCHEDULER_CLASSNAME = ConcurrentMergeScheduler.class.getName(); static final SolrIndexConfig defaultDefaults = new SolrIndexConfig(); - private SolrIndexConfig() { + luceneVersion = Version.LUCENE_31; useCompoundFile = true; maxBufferedDocs = -1; maxMergeDocs = -1; @@ -56,10 +57,12 @@ writeLockTimeout = -1; commitLockTimeout = -1; lockType = null; - termIndexInterval = IndexWriter.DEFAULT_TERM_INDEX_INTERVAL; + termIndexInterval = IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL; mergePolicyInfo = null; mergeSchedulerInfo = null; } + + public final Version luceneVersion; public final boolean useCompoundFile; public final int maxBufferedDocs; @@ -83,6 +86,9 @@ prefix = defaultsName; if (def == null) def = defaultDefaults; + + luceneVersion = solrConfig.luceneMatchVersion; + useCompoundFile=solrConfig.getBool(prefix+"/useCompoundFile", def.useCompoundFile); maxBufferedDocs=solrConfig.getInt(prefix+"/maxBufferedDocs",def.maxBufferedDocs); maxMergeDocs=solrConfig.getInt(prefix+"/maxMergeDocs",def.maxMergeDocs); @@ -129,11 +135,74 @@ infoStreamFile= solrConfig.get(prefix + "/infoStream/@file", null); log.info("IndexWriter infoStream debug log is enabled: " + infoStreamFile); } - } private PluginInfo getPluginInfo(String path, SolrConfig solrConfig, PluginInfo def) { List l = solrConfig.readPluginInfos(path, false, true); return l.isEmpty() ? def : l.get(0); } + + public IndexWriterConfig toIndexWriterConfig(IndexSchema schema) { + IndexWriterConfig iwc = new IndexWriterConfig(luceneVersion, schema.getAnalyzer()); + if (maxBufferedDocs != -1) + iwc.setMaxBufferedDocs(maxBufferedDocs); + + if (ramBufferSizeMB != -1) + iwc.setRAMBufferSizeMB(ramBufferSizeMB); + + if (termIndexInterval != -1) + iwc.setTermIndexInterval(termIndexInterval); + + if (maxFieldLength != -1) + iwc.setMaxFieldLength(maxFieldLength); + + if (writeLockTimeout != -1) + iwc.setWriteLockTimeout(writeLockTimeout); + + iwc.setSimilarity(schema.getSimilarity()); + iwc.setMergePolicy(buildMergePolicy(schema)); + iwc.setMergeScheduler(buildMergeScheduler(schema)); + + return iwc; + } + + private MergePolicy buildMergePolicy(IndexSchema schema) { + MergePolicy policy; + String mpClassName = mergePolicyInfo == null ? SolrIndexConfig.DEFAULT_MERGE_POLICY_CLASSNAME : mergePolicyInfo.className; + + try { + policy = (MergePolicy) schema.getResourceLoader().newInstance(mpClassName, null, new Class[]{IndexWriter.class}, new Object[]{this}); + } catch (Exception e) { + policy = (MergePolicy) schema.getResourceLoader().newInstance(mpClassName); + } + + if (mergePolicyInfo != null) + SolrPluginUtils.invokeSetters(policy, mergePolicyInfo.initArgs); + + if (policy instanceof LogMergePolicy) { + LogMergePolicy logMergePolicy = (LogMergePolicy) policy; + + if (maxMergeDocs != -1) + logMergePolicy.setMaxMergeDocs(maxMergeDocs); + + logMergePolicy.setUseCompoundFile(useCompoundFile); + + if (mergeFactor != -1) + logMergePolicy.setMergeFactor(mergeFactor); + } else { + log.warn("Use of compound file format or mergefactor cannot be configured if merge policy is not an instance of LogMergePolicy. The configured policy's defaults will be used."); + } + + return policy; + } + + private MergeScheduler buildMergeScheduler(IndexSchema schema) { + String msClassName = mergeSchedulerInfo == null ? SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME : mergeSchedulerInfo.className; + MergeScheduler scheduler = (MergeScheduler) schema.getResourceLoader().newInstance(msClassName); + + if (mergeSchedulerInfo != null) + SolrPluginUtils.invokeSetters(scheduler, mergeSchedulerInfo.initArgs); + + return scheduler; + } } Index: solr/src/java/org/apache/solr/analysis/GreekLowerCaseFilterFactory.java =================================================================== --- solr/src/java/org/apache/solr/analysis/GreekLowerCaseFilterFactory.java (revision 1040477) +++ solr/src/java/org/apache/solr/analysis/GreekLowerCaseFilterFactory.java (working copy) @@ -41,7 +41,7 @@ } public GreekLowerCaseFilter create(TokenStream in) { - return new GreekLowerCaseFilter(in); + return new GreekLowerCaseFilter(luceneMatchVersion, in); } } Index: solr/src/java/org/apache/solr/search/SolrIndexSearcher.java =================================================================== --- solr/src/java/org/apache/solr/search/SolrIndexSearcher.java (revision 1040477) +++ solr/src/java/org/apache/solr/search/SolrIndexSearcher.java (working copy) @@ -147,7 +147,7 @@ if (r.directory() instanceof FSDirectory) { FSDirectory fsDirectory = (FSDirectory) r.directory(); - indexDir = fsDirectory.getFile().getAbsolutePath(); + indexDir = fsDirectory.getDirectory().getAbsolutePath(); } this.closeReader = closeReader; Index: solr/src/java/org/apache/solr/core/SolrDeletionPolicy.java =================================================================== --- solr/src/java/org/apache/solr/core/SolrDeletionPolicy.java (revision 1040477) +++ solr/src/java/org/apache/solr/core/SolrDeletionPolicy.java (working copy) @@ -81,7 +81,7 @@ if (dir instanceof FSDirectory) { FSDirectory fsd = (FSDirectory) dir; - sb.append("dir=").append(fsd.getFile()); + sb.append("dir=").append(fsd.getDirectory()); } else { sb.append("dir=").append(dir); } @@ -183,7 +183,7 @@ // be the same, regardless of the Directory instance. if (dir instanceof FSDirectory) { FSDirectory fsd = (FSDirectory) dir; - File fdir = fsd.getFile(); + File fdir = fsd.getDirectory(); sb.append(fdir.getPath()); } else { sb.append(dir); Index: solr/src/java/org/apache/solr/handler/component/SpellCheckComponent.java =================================================================== --- solr/src/java/org/apache/solr/handler/component/SpellCheckComponent.java (revision 1040477) +++ solr/src/java/org/apache/solr/handler/component/SpellCheckComponent.java (working copy) @@ -640,7 +640,7 @@ IndexSchema schema = core.getSchema(); String fieldTypeName = (String) initParams.get("queryAnalyzerFieldType"); FieldType fieldType = schema.getFieldTypes().get(fieldTypeName); - Analyzer analyzer = fieldType == null ? new WhitespaceAnalyzer() + Analyzer analyzer = fieldType == null ? new WhitespaceAnalyzer(core.getSolrConfig().luceneMatchVersion) : fieldType.getQueryAnalyzer(); //TODO: There's got to be a better way! Where's Spring when you need it? queryConverter.setAnalyzer(analyzer); Index: solr/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java =================================================================== --- solr/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java (revision 1040477) +++ solr/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java (working copy) @@ -19,20 +19,18 @@ import java.io.IOException; import java.io.InputStreamReader; import java.util.List; + +import org.apache.lucene.index.*; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.IndexWriter; import org.apache.lucene.search.spell.PlainTextDictionary; import org.apache.lucene.store.RAMDirectory; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.SolrCore; -import org.apache.solr.core.SolrResourceLoader; import org.apache.solr.schema.FieldType; -import org.apache.solr.schema.IndexSchema; import org.apache.solr.util.HighFrequencyDictionary; import org.apache.solr.search.SolrIndexSearcher; @@ -60,7 +58,7 @@ public void build(SolrCore core, SolrIndexSearcher searcher) { try { - loadExternalFileDictionary(core.getSchema(), core.getResourceLoader()); + loadExternalFileDictionary(core); spellChecker.clearIndex(); spellChecker.indexDictionary(dictionary); } catch (IOException e) { @@ -77,23 +75,29 @@ } @SuppressWarnings("unchecked") - private void loadExternalFileDictionary(IndexSchema schema, SolrResourceLoader loader) { + private void loadExternalFileDictionary(SolrCore core) { try { // Get the field's analyzer - if (fieldTypeName != null - && schema.getFieldTypeNoEx(fieldTypeName) != null) { - FieldType fieldType = schema.getFieldTypes() - .get(fieldTypeName); + if (fieldTypeName != null && core.getSchema().getFieldTypeNoEx(fieldTypeName) != null) { + FieldType fieldType = core.getSchema().getFieldTypes().get(fieldTypeName); // Do index-time analysis using the given fieldType's analyzer RAMDirectory ramDir = new RAMDirectory(); - IndexWriter writer = new IndexWriter(ramDir, fieldType.getAnalyzer(), - true, IndexWriter.MaxFieldLength.UNLIMITED); - writer.setMergeFactor(300); - writer.setMaxBufferedDocs(150); - List lines = loader.getLines(sourceLocation, characterEncoding); + LogMergePolicy mp = new LogByteSizeMergePolicy(); + mp.setMergeFactor(300); + IndexWriter writer = new IndexWriter( + ramDir, + new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, fieldType.getAnalyzer()). + setMaxBufferedDocs(150). + setMergePolicy(mp). + setMaxFieldLength(IndexWriterConfig.UNLIMITED_FIELD_LENGTH). + setOpenMode(IndexWriterConfig.OpenMode.CREATE) + ); + + List lines = core.getResourceLoader().getLines(sourceLocation, characterEncoding); + for (String s : lines) { Document d = new Document(); d.add(new Field(WORD_FIELD_NAME, s, Field.Store.NO, Field.Index.ANALYZED)); @@ -107,9 +111,9 @@ } else { // check if character encoding is defined if (characterEncoding == null) { - dictionary = new PlainTextDictionary(loader.openResource(sourceLocation)); + dictionary = new PlainTextDictionary(core.getResourceLoader().openResource(sourceLocation)); } else { - dictionary = new PlainTextDictionary(new InputStreamReader(loader.openResource(sourceLocation), characterEncoding)); + dictionary = new PlainTextDictionary(new InputStreamReader(core.getResourceLoader().openResource(sourceLocation), characterEncoding)); } } Index: solr/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java =================================================================== --- solr/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java (revision 1040477) +++ solr/src/java/org/apache/solr/spelling/AbstractLuceneSpellChecker.java (working copy) @@ -149,7 +149,7 @@ } if (analyzer == null) { log.info("Using WhitespaceAnalzyer for dictionary: " + name); - analyzer = new WhitespaceAnalyzer(); + analyzer = new WhitespaceAnalyzer(core.getSolrConfig().luceneMatchVersion); } return name; } Property changes on: lucene ___________________________________________________________________ Modified: svn:mergeinfo Merged /lucene/dev/trunk/lucene:r1040463 Index: lucene/src/java/org/apache/lucene/util/Version.java =================================================================== --- lucene/src/java/org/apache/lucene/util/Version.java (revision 1040477) +++ lucene/src/java/org/apache/lucene/util/Version.java (working copy) @@ -29,22 +29,40 @@ */ public enum Version { - /** Match settings and bugs in Lucene's 2.0 release. */ + /** Match settings and bugs in Lucene's 2.0 release. + * @deprecated (3.1) Use latest + */ + @Deprecated LUCENE_20, - /** Match settings and bugs in Lucene's 2.1 release. */ + /** Match settings and bugs in Lucene's 2.1 release. + * @deprecated (3.1) Use latest + */ + @Deprecated LUCENE_21, - /** Match settings and bugs in Lucene's 2.2 release. */ + /** Match settings and bugs in Lucene's 2.2 release. + * @deprecated (3.1) Use latest + */ + @Deprecated LUCENE_22, - /** Match settings and bugs in Lucene's 2.3 release. */ + /** Match settings and bugs in Lucene's 2.3 release. + * @deprecated (3.1) Use latest + */ + @Deprecated LUCENE_23, - /** Match settings and bugs in Lucene's 2.4 release. */ + /** Match settings and bugs in Lucene's 2.4 release. + * @deprecated (3.1) Use latest + */ + @Deprecated LUCENE_24, - /** Match settings and bugs in Lucene's 2.9 release. */ + /** Match settings and bugs in Lucene's 2.9 release. + * @deprecated (3.1) Use latest + */ + @Deprecated LUCENE_29, /** Match settings and bugs in Lucene's 3.0 release. */ Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java (working copy) @@ -71,7 +71,6 @@ * List of typical german stopwords. * @deprecated use {@link #getDefaultStopSet()} instead */ - //TODO make this private in 3.1, remove in 4.0 @Deprecated public final static String[] GERMAN_STOP_WORDS = { "einer", "eine", "eines", "einem", "einen", @@ -100,7 +99,7 @@ } private static class DefaultSetHolder { - /** @deprecated remove in Lucene 4.0 */ + /** @deprecated remove in Lucene 5.0 */ @Deprecated private static final Set DEFAULT_SET_30 = CharArraySet.unmodifiableSet(new CharArraySet( Version.LUCENE_CURRENT, Arrays.asList(GERMAN_STOP_WORDS), false)); Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemmer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemmer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemmer.java (working copy) @@ -26,7 +26,7 @@ * (French stemming algorithm) for details *

* @deprecated Use {@link org.tartarus.snowball.ext.FrenchStemmer} instead, - * which has the same functionality. This filter will be removed in Lucene 4.0 + * which has the same functionality. This filter will be removed in Lucene 5.0 */ @Deprecated public class FrenchStemmer { Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java (working copy) @@ -43,7 +43,7 @@ * @see KeywordMarkerFilter * @deprecated Use {@link SnowballFilter} with * {@link org.tartarus.snowball.ext.FrenchStemmer} instead, which has the - * same functionality. This filter will be removed in Lucene 4.0 + * same functionality. This filter will be removed in Lucene 5.0 */ @Deprecated public final class FrenchStemFilter extends TokenFilter { Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (working copy) @@ -71,7 +71,6 @@ * Extended list of typical French stopwords. * @deprecated use {@link #getDefaultStopSet()} instead */ - // TODO make this private in 3.1, remove in 4.0 @Deprecated public final static String[] FRENCH_STOP_WORDS = { "a", "afin", "ai", "ainsi", "après", "attendu", "au", "aujourd", "auquel", "aussi", @@ -104,7 +103,6 @@ /** * Contains words that should be indexed but not stemmed. */ - //TODO make this final in 3.0 private Set excltable = Collections.emptySet(); /** @@ -116,7 +114,7 @@ } private static class DefaultSetHolder { - /** @deprecated remove this in Lucene 4.0 */ + /** @deprecated remove this in Lucene 5.0 */ @Deprecated static final Set DEFAULT_STOP_SET_30 = CharArraySet .unmodifiableSet(new CharArraySet(Version.LUCENE_CURRENT, Arrays.asList(FRENCH_STOP_WORDS), @@ -250,7 +248,7 @@ return new TokenStreamComponents(source, result); } else { final Tokenizer source = new StandardTokenizer(matchVersion, reader); - TokenStream result = new StandardFilter(source); + TokenStream result = new StandardFilter(matchVersion, source); result = new StopFilter(matchVersion, result, stopwords); if(!excltable.isEmpty()) result = new KeywordMarkerFilter(result, excltable); Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemmer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemmer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemmer.java (working copy) @@ -27,7 +27,7 @@ * algorithm in Martin Porter's snowball project. *

* @deprecated Use {@link org.tartarus.snowball.ext.DutchStemmer} instead, - * which has the same functionality. This filter will be removed in Lucene 4.0 + * which has the same functionality. This filter will be removed in Lucene 5.0 */ @Deprecated public class DutchStemmer { Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java (working copy) @@ -45,7 +45,7 @@ * @see KeywordMarkerFilter * @deprecated Use {@link SnowballFilter} with * {@link org.tartarus.snowball.ext.DutchStemmer} instead, which has the - * same functionality. This filter will be removed in Lucene 4.0 + * same functionality. This filter will be removed in Lucene 5.0 */ @Deprecated public final class DutchStemFilter extends TokenFilter { Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (working copy) @@ -257,7 +257,7 @@ return new TokenStreamComponents(source, result); } else { final Tokenizer source = new StandardTokenizer(matchVersion, aReader); - TokenStream result = new StandardFilter(source); + TokenStream result = new StandardFilter(matchVersion, source); result = new StopFilter(matchVersion, result, stoptable); if (!excltable.isEmpty()) result = new KeywordMarkerFilter(result, excltable); Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java (working copy) @@ -41,7 +41,7 @@ * *

* @deprecated Use the language-specific analyzer in contrib/analyzers instead. - * This analyzer will be removed in Lucene 4.0 + * This analyzer will be removed in Lucene 5.0 */ @Deprecated public final class SnowballAnalyzer extends Analyzer { Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java (working copy) @@ -38,7 +38,7 @@ * {@link CharTokenizer#normalize(int)} for details. * * @deprecated Use {@link StandardTokenizer} instead, which has the same functionality. - * This filter will be removed in Lucene 4.0 + * This filter will be removed in Lucene 5.0 */ @Deprecated public class RussianLetterTokenizer extends CharTokenizer Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (working copy) @@ -56,7 +56,7 @@ { /** * List of typical Russian stopwords. (for backwards compatibility) - * @deprecated Remove this for LUCENE 4.0 + * @deprecated Remove this for LUCENE 5.0 */ @Deprecated private static final String[] RUSSIAN_STOP_WORDS_30 = { @@ -76,7 +76,7 @@ public final static String DEFAULT_STOPWORD_FILE = "russian_stop.txt"; private static class DefaultSetHolder { - /** @deprecated remove this for Lucene 4.0 */ + /** @deprecated remove this for Lucene 5.0 */ @Deprecated static final Set DEFAULT_STOP_SET_30 = CharArraySet .unmodifiableSet(new CharArraySet(Version.LUCENE_CURRENT, Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseAnalyzer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseAnalyzer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseAnalyzer.java (working copy) @@ -28,7 +28,7 @@ * An {@link Analyzer} that tokenizes text with {@link ChineseTokenizer} and * filters with {@link ChineseFilter} * @deprecated Use {@link StandardAnalyzer} instead, which has the same functionality. - * This analyzer will be removed in Lucene 4.0 + * This analyzer will be removed in Lucene 5.0 */ @Deprecated public final class ChineseAnalyzer extends ReusableAnalyzerBase { Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseFilter.java (working copy) @@ -43,7 +43,7 @@ * * @version 1.0 * @deprecated Use {@link StopFilter} instead, which has the same functionality. - * This filter will be removed in Lucene 4.0 + * This filter will be removed in Lucene 5.0 */ @Deprecated public final class ChineseFilter extends TokenFilter { Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java (working copy) @@ -54,7 +54,7 @@ *

* @version 1.0 * @deprecated Use {@link StandardTokenizer} instead, which has the same functionality. - * This filter will be removed in Lucene 4.0 + * This filter will be removed in Lucene 5.0 */ @Deprecated public final class ChineseTokenizer extends Tokenizer { Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/en/EnglishAnalyzer.java (working copy) @@ -104,7 +104,7 @@ protected TokenStreamComponents createComponents(String fieldName, Reader reader) { final Tokenizer source = new StandardTokenizer(matchVersion, reader); - TokenStream result = new StandardFilter(source); + TokenStream result = new StandardFilter(matchVersion, source); // prior to this we get the classic behavior, standardfilter does it for us. if (matchVersion.onOrAfter(Version.LUCENE_31)) result = new EnglishPossessiveFilter(result); Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java (working copy) @@ -42,8 +42,6 @@ * useful for searching and some double-byte interpunctions. * @deprecated use {@link #getDefaultStopSet()} instead */ - // TODO make this final in 3.1 - - // this might be revised and merged with StopFilter stop words too @Deprecated public final static String[] STOP_WORDS = { "a", "and", "are", "as", "at", "be", Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java (revision 1040477) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java (working copy) @@ -60,7 +60,6 @@ * List of typical stopwords. * @deprecated use {@link #getDefaultStopSet()} instead */ - // TODO make this private in 3.1 @Deprecated public final static String[] CZECH_STOP_WORDS = { "a","s","k","o","i","u","v","z","dnes","cz","t\u00edmto","bude\u0161","budem",