Index: solr/src/test/org/apache/solr/core/TestArbitraryIndexDir.java =================================================================== --- solr/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (revision 1059811) +++ solr/src/test/org/apache/solr/core/TestArbitraryIndexDir.java (working copy) @@ -98,9 +98,7 @@ Directory dir = newFSDirectory(newDir); IndexWriter iw = new IndexWriter( dir, - new IndexWriterConfig(TEST_VERSION_CURRENT, new StandardAnalyzer(TEST_VERSION_CURRENT)). - setMaxFieldLength(1000) - ); + new IndexWriterConfig(TEST_VERSION_CURRENT, new StandardAnalyzer(TEST_VERSION_CURRENT))); Document doc = new Document(); doc.add(new Field("id", "2", Field.Store.YES, Field.Index.ANALYZED)); doc.add(new Field("name", "name2", Field.Store.YES, Field.Index.ANALYZED)); Index: solr/src/test/org/apache/solr/search/TestSort.java =================================================================== --- solr/src/test/org/apache/solr/search/TestSort.java (revision 1059811) +++ solr/src/test/org/apache/solr/search/TestSort.java (working copy) @@ -33,7 +33,9 @@ import java.util.*; public class TestSort extends AbstractSolrTestCase { + @Override public String getSchemaFile() { return null; } + @Override public String getSolrConfigFile() { return null; } Random r = random; @@ -49,6 +51,7 @@ String val; String val2; + @Override public String toString() { return "{id=" +doc + " val1="+val + " val2="+val2 + "}"; } @@ -63,9 +66,7 @@ IndexWriter iw = new IndexWriter( dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new SimpleAnalyzer(TEST_VERSION_CURRENT)). - setOpenMode(IndexWriterConfig.OpenMode.CREATE). - setMaxFieldLength(IndexWriterConfig.UNLIMITED_FIELD_LENGTH) - ); + setOpenMode(IndexWriterConfig.OpenMode.CREATE)); final MyDoc[] mydocs = new MyDoc[ndocs]; int v1EmptyPercent = 50; Index: solr/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java =================================================================== --- solr/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java (revision 1059811) +++ solr/src/test/org/apache/solr/spelling/IndexBasedSpellCheckerTest.java (working copy) @@ -284,9 +284,7 @@ Directory dir = newFSDirectory(altIndexDir); IndexWriter iw = new IndexWriter( dir, - new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)). - setMaxFieldLength(IndexWriterConfig.UNLIMITED_FIELD_LENGTH) - ); + new IndexWriterConfig(TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); for (int i = 0; i < ALT_DOCS.length; i++) { Document doc = new Document(); doc.add(new Field("title", ALT_DOCS[i], Field.Store.YES, Field.Index.ANALYZED)); Index: solr/src/java/org/apache/solr/update/SolrIndexWriter.java =================================================================== --- solr/src/java/org/apache/solr/update/SolrIndexWriter.java (revision 1059811) +++ solr/src/java/org/apache/solr/update/SolrIndexWriter.java (working copy) @@ -86,6 +86,10 @@ setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND). setIndexDeletionPolicy(delPolicy) ); + + if (config.maxFieldLength != -1) + setMaxFieldLength(config.maxFieldLength); + log.debug("Opened Writer " + name); this.name = name; Index: solr/src/java/org/apache/solr/update/SolrIndexConfig.java =================================================================== --- solr/src/java/org/apache/solr/update/SolrIndexConfig.java (revision 1059811) +++ solr/src/java/org/apache/solr/update/SolrIndexConfig.java (working copy) @@ -153,9 +153,6 @@ if (termIndexInterval != -1) iwc.setTermIndexInterval(termIndexInterval); - if (maxFieldLength != -1) - iwc.setMaxFieldLength(maxFieldLength); - if (writeLockTimeout != -1) iwc.setWriteLockTimeout(writeLockTimeout); Index: solr/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java =================================================================== --- solr/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java (revision 1059811) +++ solr/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java (working copy) @@ -92,7 +92,6 @@ new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, fieldType.getAnalyzer()). setMaxBufferedDocs(150). setMergePolicy(mp). - setMaxFieldLength(IndexWriterConfig.UNLIMITED_FIELD_LENGTH). setOpenMode(IndexWriterConfig.OpenMode.CREATE) ); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1059811) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -947,7 +947,8 @@ public void testHighFreqTerm() throws IOException { MockDirectoryWrapper dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setMaxFieldLength(100000000).setRAMBufferSizeMB(0.01)); + TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setRAMBufferSizeMB(0.01)); + writer.setMaxFieldLength(100000000); // Massive doc that has 128 K a's StringBuilder b = new StringBuilder(1024*1024); for(int i=0;i<4096;i++) { @@ -1395,7 +1396,8 @@ Directory dir = newDirectory(); IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig( - TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT)).setMaxFieldLength(100000)); + TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); + writer.setMaxFieldLength(100000); Document doc = new Document(); StringBuilder b = new StringBuilder(); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (revision 1059811) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (working copy) @@ -65,7 +65,6 @@ assertEquals(WhitespaceAnalyzer.class, conf.getAnalyzer().getClass()); assertNull(conf.getIndexCommit()); assertEquals(KeepOnlyLastCommitDeletionPolicy.class, conf.getIndexDeletionPolicy().getClass()); - assertEquals(IndexWriterConfig.UNLIMITED_FIELD_LENGTH, conf.getMaxFieldLength()); assertEquals(ConcurrentMergeScheduler.class, conf.getMergeScheduler().getClass()); assertEquals(OpenMode.CREATE_OR_APPEND, conf.getOpenMode()); assertTrue(Similarity.getDefault() == conf.getSimilarity()); @@ -87,7 +86,6 @@ getters.add("getAnalyzer"); getters.add("getIndexCommit"); getters.add("getIndexDeletionPolicy"); - getters.add("getMaxFieldLength"); getters.add("getMergeScheduler"); getters.add("getOpenMode"); getters.add("getSimilarity"); @@ -129,7 +127,6 @@ // Tests that the values of the constants does not change assertEquals(1000, IndexWriterConfig.WRITE_LOCK_TIMEOUT); assertEquals(128, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL); - assertEquals(Integer.MAX_VALUE, IndexWriterConfig.UNLIMITED_FIELD_LENGTH); assertEquals(-1, IndexWriterConfig.DISABLE_AUTO_FLUSH); assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS); assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS); @@ -269,9 +266,6 @@ writer.setMaxBufferedDocs(10); assertEquals(10, writer.getConfig().getMaxBufferedDocs()); - - writer.setMaxFieldLength(10); - assertEquals(10, writer.getConfig().getMaxFieldLength()); writer.setMergeScheduler(new SerialMergeScheduler()); assertEquals(SerialMergeScheduler.class, writer.getConfig().getMergeScheduler().getClass()); Index: lucene/src/java/org/apache/lucene/index/DocumentsWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (revision 1059811) +++ lucene/src/java/org/apache/lucene/index/DocumentsWriter.java (working copy) @@ -124,7 +124,7 @@ private boolean aborting; // True if an abort is pending PrintStream infoStream; - int maxFieldLength = IndexWriterConfig.UNLIMITED_FIELD_LENGTH; + int maxFieldLength = IndexWriter.DEFAULT_MAX_FIELD_LENGTH; Similarity similarity; // max # simultaneous threads; if there are more than Index: lucene/src/java/org/apache/lucene/index/IndexWriter.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriter.java (revision 1059811) +++ lucene/src/java/org/apache/lucene/index/IndexWriter.java (working copy) @@ -33,6 +33,8 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.LimitTokenCountAnalyzer; +import org.apache.lucene.analysis.LimitTokenCountFilter; import org.apache.lucene.document.Document; import org.apache.lucene.index.IndexWriterConfig.OpenMode; import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor; @@ -232,7 +234,7 @@ * @deprecated see {@link IndexWriterConfig} */ @Deprecated - public final static int DEFAULT_MAX_FIELD_LENGTH = 10000; + public final static int DEFAULT_MAX_FIELD_LENGTH = MaxFieldLength.UNLIMITED.getLimit(); /** * Default value is 128. Change using {@link #setTermIndexInterval(int)}. @@ -877,8 +879,8 @@ public IndexWriter(Directory d, Analyzer a, boolean create, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { this(d, new IndexWriterConfig(Version.LUCENE_31, a).setOpenMode( - create ? OpenMode.CREATE : OpenMode.APPEND).setMaxFieldLength( - mfl.getLimit())); + create ? OpenMode.CREATE : OpenMode.APPEND)); + setMaxFieldLength(mfl.getLimit()); } /** @@ -903,8 +905,8 @@ @Deprecated public IndexWriter(Directory d, Analyzer a, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { - this(d, new IndexWriterConfig(Version.LUCENE_31, a) - .setMaxFieldLength(mfl.getLimit())); + this(d, new IndexWriterConfig(Version.LUCENE_31, a)); + setMaxFieldLength(mfl.getLimit()); } /** @@ -929,8 +931,8 @@ @Deprecated public IndexWriter(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { - this(d, new IndexWriterConfig(Version.LUCENE_31, a).setMaxFieldLength( - mfl.getLimit()).setIndexDeletionPolicy(deletionPolicy)); + this(d, new IndexWriterConfig(Version.LUCENE_31, a).setIndexDeletionPolicy(deletionPolicy)); + setMaxFieldLength(mfl.getLimit()); } /** @@ -962,8 +964,8 @@ public IndexWriter(Directory d, Analyzer a, boolean create, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl) throws CorruptIndexException, LockObtainFailedException, IOException { this(d, new IndexWriterConfig(Version.LUCENE_31, a).setOpenMode( - create ? OpenMode.CREATE : OpenMode.APPEND).setMaxFieldLength( - mfl.getLimit()).setIndexDeletionPolicy(deletionPolicy)); + create ? OpenMode.CREATE : OpenMode.APPEND).setIndexDeletionPolicy(deletionPolicy)); + setMaxFieldLength(mfl.getLimit()); } /** @@ -1003,8 +1005,8 @@ public IndexWriter(Directory d, Analyzer a, IndexDeletionPolicy deletionPolicy, MaxFieldLength mfl, IndexCommit commit) throws CorruptIndexException, LockObtainFailedException, IOException { this(d, new IndexWriterConfig(Version.LUCENE_31, a) - .setOpenMode(OpenMode.APPEND).setMaxFieldLength(mfl.getLimit()) - .setIndexDeletionPolicy(deletionPolicy).setIndexCommit(commit)); + .setOpenMode(OpenMode.APPEND).setIndexDeletionPolicy(deletionPolicy).setIndexCommit(commit)); + setMaxFieldLength(mfl.getLimit()); } /** @@ -1041,7 +1043,6 @@ directory = d; analyzer = conf.getAnalyzer(); infoStream = defaultInfoStream; - maxFieldLength = conf.getMaxFieldLength(); termIndexInterval = conf.getTermIndexInterval(); writeLockTimeout = conf.getWriteLockTimeout(); similarity = conf.getSimilarity(); @@ -1338,18 +1339,25 @@ /** * The maximum number of terms that will be indexed for a single field in a - * document. This limits the amount of memory required for indexing, so that + * document. This limits the amount of memory required for indexing, so that * collections with very large files will not crash the indexing process by - * running out of memory. This setting refers to the number of running terms, - * not to the number of different terms.

- * Note: this silently truncates large documents, excluding from the - * index all terms that occur further in the document. If you know your source - * documents are large, be sure to set this value high enough to accomodate - * the expected size. If you set it to Integer.MAX_VALUE, then the only limit - * is your memory, but you should anticipate an OutOfMemoryError.

- * By default, no more than {@link #DEFAULT_MAX_FIELD_LENGTH} terms - * will be indexed for a field. - * @deprecated use {@link IndexWriterConfig#setMaxFieldLength(int)} instead + * running out of memory. This setting refers to the number of running terms, + * not to the number of different terms. + *

+ * Note: this silently truncates large documents, excluding + * from the index all terms that occur further in the document. If you know + * your source documents are large, be sure to set this value high enough to + * accomodate the expected size. If you set it to Integer.MAX_VALUE, then the + * only limit is your memory, but you should anticipate an OutOfMemoryError. + *

+ * By default, no more than {@link #DEFAULT_MAX_FIELD_LENGTH} terms will be + * indexed for a field. + * + * @deprecated use {@link LimitTokenCountAnalyzer} instead. Note that the + * behvaior slightly changed - the analyzer limits the number of + * tokens per token stream created, while this setting limits the + * total number of tokens to index. This only matters if you index + * many multi-valued fields though. */ @Deprecated public void setMaxFieldLength(int maxFieldLength) { @@ -1358,16 +1366,13 @@ docWriter.setMaxFieldLength(maxFieldLength); if (infoStream != null) message("setMaxFieldLength " + maxFieldLength); - // Required so config.getMaxFieldLength returns the right value. But this - // will go away together with the method in 4.0. - config.setMaxFieldLength(maxFieldLength); } /** * Returns the maximum number of terms that will be * indexed for a single field in a document. * @see #setMaxFieldLength - * @deprecated use {@link IndexWriterConfig#getMaxFieldLength()} instead + * @deprecated use {@link LimitTokenCountAnalyzer} to limit number of tokens. */ @Deprecated public int getMaxFieldLength() { @@ -1956,8 +1961,9 @@ * By default, no more than 10,000 terms will be indexed for a field. * * @see MaxFieldLength + * @deprecated remove in 4.0 */ - private int maxFieldLength; + private int maxFieldLength = DEFAULT_MAX_FIELD_LENGTH; /** * Adds a document to this index. If the document contains more than @@ -4281,7 +4287,7 @@ * {@link #DEFAULT_MAX_FIELD_LENGTH} * */ public static final MaxFieldLength LIMITED - = new MaxFieldLength("LIMITED", DEFAULT_MAX_FIELD_LENGTH); + = new MaxFieldLength("LIMITED", 10000); } /** If {@link #getReader} has been called (ie, this writer Index: lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java =================================================================== --- lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java (revision 1059811) +++ lucene/src/java/org/apache/lucene/index/IndexWriterConfig.java (working copy) @@ -40,8 +40,6 @@ */ public final class IndexWriterConfig implements Cloneable { - public static final int UNLIMITED_FIELD_LENGTH = Integer.MAX_VALUE; - /** * Specifies the open mode for {@link IndexWriter}: *