Index: src/test/org/apache/lucene/index/TestSegmentTermDocs.java =================================================================== --- src/test/org/apache/lucene/index/TestSegmentTermDocs.java (revision 597187) +++ src/test/org/apache/lucene/index/TestSegmentTermDocs.java (working copy) @@ -18,6 +18,7 @@ */ import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.LuceneProperties; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.store.MockRAMDirectory; import org.apache.lucene.store.Directory; @@ -53,8 +54,9 @@ public void testTermDocs(int indexDivisor) throws IOException { //After adding the document, we should be able to read it back in - SegmentReader reader = SegmentReader.get(info); - reader.setTermInfosIndexDivisor(indexDivisor); + LuceneProperties props = new LuceneProperties(); + SegmentIndexProperties.setTermIndexDivisor(props, indexDivisor); + SegmentReader reader = SegmentReader.get(info, props); assertTrue(reader != null); SegmentTermDocs segTermDocs = new SegmentTermDocs(reader); assertTrue(segTermDocs != null); @@ -76,8 +78,9 @@ public void testBadSeek(int indexDivisor) throws IOException { { //After adding the document, we should be able to read it back in - SegmentReader reader = SegmentReader.get(info); - reader.setTermInfosIndexDivisor(indexDivisor); + LuceneProperties props = new LuceneProperties(); + SegmentIndexProperties.setTermIndexDivisor(props, indexDivisor); + SegmentReader reader = SegmentReader.get(info, props); assertTrue(reader != null); SegmentTermDocs segTermDocs = new SegmentTermDocs(reader); assertTrue(segTermDocs != null); @@ -87,8 +90,9 @@ } { //After adding the document, we should be able to read it back in - SegmentReader reader = SegmentReader.get(info); - reader.setTermInfosIndexDivisor(indexDivisor); + LuceneProperties props = new LuceneProperties(); + SegmentIndexProperties.setTermIndexDivisor(props, indexDivisor); + SegmentReader reader = SegmentReader.get(info, props); assertTrue(reader != null); SegmentTermDocs segTermDocs = new SegmentTermDocs(reader); assertTrue(segTermDocs != null); @@ -122,9 +126,10 @@ writer.optimize(); writer.close(); - IndexReader reader = IndexReader.open(dir); - reader.setTermInfosIndexDivisor(indexDivisor); - assertEquals(indexDivisor, reader.getTermInfosIndexDivisor()); + LuceneProperties props = new LuceneProperties(); + SegmentIndexProperties.setTermIndexDivisor(props, indexDivisor); + IndexReader reader = IndexReader.open(dir, props); + assertEquals(indexDivisor, SegmentIndexProperties.getTermIndexDivisor(props, 0)); TermDocs tdocs = reader.termDocs(); @@ -239,21 +244,6 @@ testSkipTo(2); } - public void testIndexDivisorAfterLoad() throws IOException { - dir = new MockRAMDirectory(); - testDoc = new Document(); - DocHelper.setupDoc(testDoc); - SegmentInfo si = DocHelper.writeDoc(dir, testDoc); - SegmentReader reader = SegmentReader.get(si); - assertEquals(1, reader.docFreq(new Term("keyField", "Keyword"))); - try { - reader.setTermInfosIndexDivisor(2); - fail("did not hit IllegalStateException exception"); - } catch (IllegalStateException ise) { - // expected - } - } - private void addDoc(IndexWriter writer, String value) throws IOException { Document doc = new Document(); Index: src/test/org/apache/lucene/index/TestSegmentReader.java =================================================================== --- src/test/org/apache/lucene/index/TestSegmentReader.java (revision 597187) +++ src/test/org/apache/lucene/index/TestSegmentReader.java (working copy) @@ -23,6 +23,7 @@ import java.util.List; import org.apache.lucene.util.LuceneTestCase; +import org.apache.lucene.util.LuceneProperties; import org.apache.lucene.document.Document; import org.apache.lucene.document.Fieldable; @@ -212,9 +213,11 @@ testDoc = new Document(); DocHelper.setupDoc(testDoc); SegmentInfo si = DocHelper.writeDoc(dir, testDoc); + + LuceneProperties props = new LuceneProperties(); + SegmentIndexProperties.setTermIndexDivisor(props, 3); - reader = SegmentReader.get(si); - reader.setTermInfosIndexDivisor(3); + reader = SegmentReader.get(si, props); testDocument(); testDelete(); testGetFieldNameVariations(); Index: src/java/org/apache/lucene/index/DirectoryIndexReader.java =================================================================== --- src/java/org/apache/lucene/index/DirectoryIndexReader.java (revision 597187) +++ src/java/org/apache/lucene/index/DirectoryIndexReader.java (working copy) @@ -22,6 +22,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.util.LuceneProperties; /** * IndexReader implementation that has access to a Directory. @@ -58,7 +59,7 @@ init(directory, segmentInfos, closeDirectory); } - static DirectoryIndexReader open(final Directory directory, final boolean closeDirectory, final IndexDeletionPolicy deletionPolicy) throws CorruptIndexException, IOException { + static DirectoryIndexReader open(final Directory directory, final boolean closeDirectory, final IndexDeletionPolicy deletionPolicy, final LuceneProperties props) throws CorruptIndexException, IOException { return (DirectoryIndexReader) new SegmentInfos.FindSegmentsFile(directory) { @@ -70,9 +71,9 @@ DirectoryIndexReader reader; if (infos.size() == 1) { // index is optimized - reader = SegmentReader.get(infos, infos.info(0), closeDirectory); + reader = SegmentReader.get(infos, infos.info(0), closeDirectory, props); } else { - reader = new MultiSegmentReader(directory, infos, closeDirectory); + reader = new MultiSegmentReader(directory, infos, closeDirectory, props); } reader.setDeletionPolicy(deletionPolicy); return reader; Index: src/java/org/apache/lucene/index/SegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/SegmentReader.java (revision 597187) +++ src/java/org/apache/lucene/index/SegmentReader.java (working copy) @@ -35,6 +35,7 @@ import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.BitVector; +import org.apache.lucene.util.LuceneProperties; import java.io.IOException; import java.util.*; @@ -197,15 +198,20 @@ * @throws IOException if there is a low-level IO error */ public static SegmentReader get(SegmentInfo si) throws CorruptIndexException, IOException { - return get(si.dir, si, null, false, false, BufferedIndexInput.BUFFER_SIZE, true); + return get(si.dir, si, null, false, false, BufferedIndexInput.BUFFER_SIZE, true, null); } + // nocommit javadocs + public static SegmentReader get(SegmentInfo si, LuceneProperties props) throws CorruptIndexException, IOException { + return get(si.dir, si, null, false, false, BufferedIndexInput.BUFFER_SIZE, true, props); + } + /** * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ static SegmentReader get(SegmentInfo si, boolean doOpenStores) throws CorruptIndexException, IOException { - return get(si.dir, si, null, false, false, BufferedIndexInput.BUFFER_SIZE, doOpenStores); + return get(si.dir, si, null, false, false, BufferedIndexInput.BUFFER_SIZE, doOpenStores, null); } /** @@ -213,7 +219,7 @@ * @throws IOException if there is a low-level IO error */ public static SegmentReader get(SegmentInfo si, int readBufferSize) throws CorruptIndexException, IOException { - return get(si.dir, si, null, false, false, readBufferSize, true); + return get(si.dir, si, null, false, false, readBufferSize, true, null); } /** @@ -221,7 +227,7 @@ * @throws IOException if there is a low-level IO error */ static SegmentReader get(SegmentInfo si, int readBufferSize, boolean doOpenStores) throws CorruptIndexException, IOException { - return get(si.dir, si, null, false, false, readBufferSize, doOpenStores); + return get(si.dir, si, null, false, false, readBufferSize, doOpenStores, null); } /** @@ -230,19 +236,28 @@ */ public static SegmentReader get(SegmentInfos sis, SegmentInfo si, boolean closeDir) throws CorruptIndexException, IOException { - return get(si.dir, si, sis, closeDir, true, BufferedIndexInput.BUFFER_SIZE, true); + return get(si.dir, si, sis, closeDir, true, BufferedIndexInput.BUFFER_SIZE, true, null); } /** * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ + public static SegmentReader get(SegmentInfos sis, SegmentInfo si, + boolean closeDir, LuceneProperties props) throws CorruptIndexException, IOException { + return get(si.dir, si, sis, closeDir, true, BufferedIndexInput.BUFFER_SIZE, true, props); + } + + /** + * @throws CorruptIndexException if the index is corrupt + * @throws IOException if there is a low-level IO error + */ public static SegmentReader get(Directory dir, SegmentInfo si, SegmentInfos sis, boolean closeDir, boolean ownDir, int readBufferSize) throws CorruptIndexException, IOException { - return get(dir, si, sis, closeDir, ownDir, readBufferSize, true); + return get(dir, si, sis, closeDir, ownDir, readBufferSize, true, null); } /** @@ -253,7 +268,8 @@ SegmentInfos sis, boolean closeDir, boolean ownDir, int readBufferSize, - boolean doOpenStores) + boolean doOpenStores, + LuceneProperties props) throws CorruptIndexException, IOException { SegmentReader instance; try { @@ -262,11 +278,11 @@ throw new RuntimeException("cannot load SegmentReader class: " + e, e); } instance.init(dir, sis, closeDir); - instance.initialize(si, readBufferSize, doOpenStores); + instance.initialize(si, readBufferSize, doOpenStores, props); return instance; } - private void initialize(SegmentInfo si, int readBufferSize, boolean doOpenStores) throws CorruptIndexException, IOException { + private void initialize(SegmentInfo si, int readBufferSize, boolean doOpenStores, LuceneProperties props) throws CorruptIndexException, IOException { segment = si.name; this.si = si; this.readBufferSize = readBufferSize; @@ -318,7 +334,7 @@ } } - tis = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize); + tis = new TermInfosReader(cfsDir, segment, fieldInfos, readBufferSize, props); loadDeletedDocs(); @@ -700,14 +716,6 @@ return si.docCount; } - public void setTermInfosIndexDivisor(int indexDivisor) throws IllegalStateException { - tis.setIndexDivisor(indexDivisor); - } - - public int getTermInfosIndexDivisor() { - return tis.getIndexDivisor(); - } - /** * @see IndexReader#getFieldNames(IndexReader.FieldOption fldOption) */ Index: src/java/org/apache/lucene/index/TermInfosReader.java =================================================================== --- src/java/org/apache/lucene/index/TermInfosReader.java (revision 597187) +++ src/java/org/apache/lucene/index/TermInfosReader.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.BufferedIndexInput; +import org.apache.lucene.util.LuceneProperties; /** This stores a monotonically increasing set of pairs in a * Directory. Pairs are accessed either by Term or by ordinal position the @@ -40,17 +41,24 @@ private long[] indexPointers; private SegmentTermEnum indexEnum; + private static int DEFAULT_INDEX_DIVISOR = 1; - private int indexDivisor = 1; + private int indexDivisor; private int totalIndexInterval; + private LuceneProperties props; TermInfosReader(Directory dir, String seg, FieldInfos fis) throws CorruptIndexException, IOException { - this(dir, seg, fis, BufferedIndexInput.BUFFER_SIZE); + this(dir, seg, fis, BufferedIndexInput.BUFFER_SIZE, null); } TermInfosReader(Directory dir, String seg, FieldInfos fis, int readBufferSize) throws CorruptIndexException, IOException { + this(dir, seg, fis, readBufferSize, null); + } + + TermInfosReader(Directory dir, String seg, FieldInfos fis, int readBufferSize, LuceneProperties props) + throws CorruptIndexException, IOException { boolean success = false; try { @@ -66,6 +74,8 @@ indexEnum = new SegmentTermEnum(directory.openInput(segment + ".tii", readBufferSize), fieldInfos, true); + this.props = props; + success = true; } finally { // With lock-less commits, it's entirely possible (and @@ -87,43 +97,6 @@ return origEnum.maxSkipLevels; } - /** - *

Sets the indexDivisor, which subsamples the number - * of indexed terms loaded into memory. This has a - * similar effect as {@link - * IndexWriter#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1.

- * - * NOTE: you must call this before the term - * index is loaded. If the index is already loaded, - * an IllegalStateException is thrown. - * - + @throws IllegalStateException if the term index has - * already been loaded into memory. - */ - public void setIndexDivisor(int indexDivisor) throws IllegalStateException { - if (indexDivisor < 1) - throw new IllegalArgumentException("indexDivisor must be > 0: got " + indexDivisor); - - if (indexTerms != null) - throw new IllegalStateException("index terms are already loaded"); - - this.indexDivisor = indexDivisor; - totalIndexInterval = origEnum.indexInterval * indexDivisor; - } - - /** Returns the indexDivisor. - * @see #setIndexDivisor - */ - public int getIndexDivisor() { - return indexDivisor; - } - final void close() throws IOException { if (origEnum != null) origEnum.close(); @@ -149,6 +122,15 @@ private synchronized void ensureIndexIsRead() throws IOException { if (indexTerms != null) // index already read return; // do nothing + + if (props == null) + indexDivisor = DEFAULT_INDEX_DIVISOR; + else + indexDivisor = SegmentIndexProperties.getTermIndexDivisor(props, DEFAULT_INDEX_DIVISOR); + + this.indexDivisor = indexDivisor; + totalIndexInterval = origEnum.indexInterval * indexDivisor; + try { int indexSize = 1+((int)indexEnum.size-1)/indexDivisor; // otherwise read index Index: src/java/org/apache/lucene/index/IndexReader.java =================================================================== --- src/java/org/apache/lucene/index/IndexReader.java (revision 597187) +++ src/java/org/apache/lucene/index/IndexReader.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.search.Similarity; import org.apache.lucene.store.*; +import org.apache.lucene.util.LuceneProperties; import java.io.File; import java.io.FileOutputStream; @@ -188,6 +189,10 @@ return open(directory, false, null); } + public static IndexReader open(final Directory directory, LuceneProperties props) throws CorruptIndexException, IOException { + return open(directory, false, null, props); + } + /** Expert: returns an IndexReader reading the index in the given * Directory, with a custom {@link IndexDeletionPolicy}. * @param directory the index directory @@ -202,9 +207,13 @@ } private static IndexReader open(final Directory directory, final boolean closeDirectory, final IndexDeletionPolicy deletionPolicy) throws CorruptIndexException, IOException { - return DirectoryIndexReader.open(directory, closeDirectory, deletionPolicy); + return DirectoryIndexReader.open(directory, closeDirectory, deletionPolicy, null); } + private static IndexReader open(final Directory directory, final boolean closeDirectory, final IndexDeletionPolicy deletionPolicy, final LuceneProperties props) throws CorruptIndexException, IOException { + return DirectoryIndexReader.open(directory, closeDirectory, deletionPolicy, props); + } + /** * Refreshes an IndexReader if the index has changed since this instance * was (re)opened. @@ -354,35 +363,6 @@ throw new UnsupportedOperationException("This reader does not support this method."); } - /**

For IndexReader implementations that use - * TermInfosReader to read terms, this sets the - * indexDivisor to subsample the number of indexed terms - * loaded into memory. This has the same effect as {@link - * IndexWriter#setTermIndexInterval} except that setting - * must be done at indexing time while this setting can be - * set per reader. When set to N, then one in every - * N*termIndexInterval terms in the index is loaded into - * memory. By setting this to a value > 1 you can reduce - * memory usage, at the expense of higher latency when - * loading a TermInfo. The default value is 1.

- * - * NOTE: you must call this before the term - * index is loaded. If the index is already loaded, - * an IllegalStateException is thrown. - * @throws IllegalStateException if the term index has already been loaded into memory - */ - public void setTermInfosIndexDivisor(int indexDivisor) throws IllegalStateException { - throw new UnsupportedOperationException("This reader does not support this method."); - } - - /**

For IndexReader implementations that use - * TermInfosReader to read terms, this returns the - * current indexDivisor. - * @see #setTermInfosIndexDivisor */ - public int getTermInfosIndexDivisor() { - throw new UnsupportedOperationException("This reader does not support this method."); - } - /** * Check whether this IndexReader is still using the * current (i.e., most recently committed) version of the Index: src/java/org/apache/lucene/index/MultiSegmentReader.java =================================================================== --- src/java/org/apache/lucene/index/MultiSegmentReader.java (revision 597187) +++ src/java/org/apache/lucene/index/MultiSegmentReader.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.LuceneProperties; import java.io.IOException; import java.util.Collection; @@ -43,6 +44,10 @@ /** Construct reading the named set of readers. */ MultiSegmentReader(Directory directory, SegmentInfos sis, boolean closeDirectory) throws IOException { + this(directory, sis, closeDirectory, null); + } + + MultiSegmentReader(Directory directory, SegmentInfos sis, boolean closeDirectory, LuceneProperties props) throws IOException { super(directory, sis, closeDirectory); // To reduce the chance of hitting FileNotFound // (and having to retry), we open segments in @@ -52,7 +57,7 @@ SegmentReader[] readers = new SegmentReader[sis.size()]; for (int i = sis.size()-1; i >= 0; i--) { try { - readers[i] = SegmentReader.get(sis.info(i)); + readers[i] = SegmentReader.get(sis.info(i), props); } catch (IOException e) { // Close all readers we had opened: for(i++;i 0) - return subReaders[0].getTermInfosIndexDivisor(); - else - throw new IllegalStateException("no readers"); - } - static class MultiTermEnum extends TermEnum { private SegmentMergeQueue queue;