Index: src/test/org/apache/lucene/index/DocHelper.java
===================================================================
--- src/test/org/apache/lucene/index/DocHelper.java	(revision 596211)
+++ src/test/org/apache/lucene/index/DocHelper.java	(working copy)
@@ -231,8 +231,23 @@
    */ 
   public static SegmentInfo writeDoc(Directory dir, Analyzer analyzer, Similarity similarity, Document doc) throws IOException
   {
+    return writeDoc(dir, analyzer, similarity, doc, IndexWriter.DEFAULT_TERM_INDEX_INTERVAL);
+  }
+
+  /**
+   * Writes the document to the directory segment using the analyzer and the similarity score
+   * @param dir
+   * @param analyzer
+   * @param similarity
+   * @param doc
+   * @param termIndexInterval
+   * @throws IOException
+   */ 
+  public static SegmentInfo writeDoc(Directory dir, Analyzer analyzer, Similarity similarity, Document doc, int termIndexInterval) throws IOException
+  {
     IndexWriter writer = new IndexWriter(dir, analyzer);
     writer.setSimilarity(similarity);
+    writer.setTermIndexInterval(termIndexInterval);
     //writer.setUseCompoundFile(false);
     writer.addDocument(doc);
     writer.flush();
@@ -240,7 +255,7 @@
     writer.close();
     return info;
   }
-
+  
   public static int numFields(Document doc) {
     return doc.getFields().size();
   }
Index: src/test/org/apache/lucene/index/TestSegmentTermDocs.java
===================================================================
--- src/test/org/apache/lucene/index/TestSegmentTermDocs.java	(revision 596211)
+++ src/test/org/apache/lucene/index/TestSegmentTermDocs.java	(working copy)
@@ -32,6 +32,10 @@
   private Document testDoc = new Document();
   private Directory dir = new RAMDirectory();
   private SegmentInfo info;
+  
+  private int termInfosIndexDivisor = 1;
+  private TermInfosConfigurer termInfosConfigurer = null;
+  private int expectedTermInfosIndexDivisor = 1;
 
   public TestSegmentTermDocs(String s) {
     super(s);
@@ -48,14 +52,11 @@
   }
   
   public void testTermDocs() throws IOException {
-    testTermDocs(1);
-  }
-
-  public void testTermDocs(int indexDivisor) throws IOException {
     //After adding the document, we should be able to read it back in
     SegmentReader reader = SegmentReader.get(info);
-    reader.setTermInfosIndexDivisor(indexDivisor);
     assertTrue(reader != null);
+    reader.setTermInfosIndexDivisor(termInfosIndexDivisor);
+    reader.setTermInfosConfigurer(termInfosConfigurer);
     SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
     assertTrue(segTermDocs != null);
     segTermDocs.seek(new Term(DocHelper.TEXT_FIELD_2_KEY, "field"));
@@ -70,15 +71,12 @@
   }  
   
   public void testBadSeek() throws IOException {
-    testBadSeek(1);
-  }
-
-  public void testBadSeek(int indexDivisor) throws IOException {
     {
       //After adding the document, we should be able to read it back in
       SegmentReader reader = SegmentReader.get(info);
-      reader.setTermInfosIndexDivisor(indexDivisor);
       assertTrue(reader != null);
+      reader.setTermInfosIndexDivisor(termInfosIndexDivisor);
+      reader.setTermInfosConfigurer(termInfosConfigurer);
       SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
       assertTrue(segTermDocs != null);
       segTermDocs.seek(new Term("textField2", "bad"));
@@ -88,8 +86,9 @@
     {
       //After adding the document, we should be able to read it back in
       SegmentReader reader = SegmentReader.get(info);
-      reader.setTermInfosIndexDivisor(indexDivisor);
       assertTrue(reader != null);
+      reader.setTermInfosIndexDivisor(termInfosIndexDivisor);
+      reader.setTermInfosConfigurer(termInfosConfigurer);
       SegmentTermDocs segTermDocs = new SegmentTermDocs(reader);
       assertTrue(segTermDocs != null);
       segTermDocs.seek(new Term("junk", "bad"));
@@ -99,10 +98,6 @@
   }
   
   public void testSkipTo() throws IOException {
-    testSkipTo(1);
-  }
-
-  public void testSkipTo(int indexDivisor) throws IOException {
     Directory dir = new RAMDirectory();
     IndexWriter writer = new IndexWriter(dir, new WhitespaceAnalyzer(), true);
     
@@ -123,8 +118,11 @@
     writer.close();
     
     IndexReader reader = IndexReader.open(dir);
-    reader.setTermInfosIndexDivisor(indexDivisor);
-    assertEquals(indexDivisor, reader.getTermInfosIndexDivisor());
+    reader.setTermInfosIndexDivisor(termInfosIndexDivisor);
+    assertEquals(termInfosIndexDivisor, reader. getTermInfosIndexDivisor());
+    reader.setTermInfosConfigurer(termInfosConfigurer);
+    assertEquals(termInfosConfigurer, reader. getTermInfosConfigurer());
+    assertEquals(termInfosIndexDivisor, expectedTermInfosIndexDivisor);
 
     TermDocs tdocs = reader.termDocs();
     
@@ -229,16 +227,56 @@
     dir.close();
   }
   
+  public void testTermInfosConfigurer() throws IOException {
+    dir = new MockRAMDirectory();
+    testDoc = new Document();
+    DocHelper.setupDoc(testDoc);
+    DocHelper.writeDoc(dir, new WhitespaceAnalyzer(), Similarity.getDefault(), testDoc, 3);
+    
+    final boolean[] goodTest = {false};
+    termInfosConfigurer = new TermInfosConfigurer(){
+      public int getMaxTermsCached(String segmentName, int segmentNumDocs, long segmentNumTerms) {
+        if (segmentNumTerms>=6) {
+          goodTest[0] = true;
+          expectedTermInfosIndexDivisor = 2;
+          return (int) (segmentNumTerms/3-1);
+        } else {
+          expectedTermInfosIndexDivisor = 1;
+          return (int) segmentNumTerms;
+        }
+      }
+    };
+    
+    try {
+      testTermDocs();
+      testBadSeek();
+      testSkipTo();
+      assertTrue(goodTest[0]);
+    } finally {
+      termInfosConfigurer = null;
+      expectedTermInfosIndexDivisor = 1;
+    }
+  }
+  
   public void testIndexDivisor() throws IOException {
     dir = new MockRAMDirectory();
     testDoc = new Document();
     DocHelper.setupDoc(testDoc);
-    DocHelper.writeDoc(dir, testDoc);
-    testTermDocs(2);
-    testBadSeek(2);
-    testSkipTo(2);
+    DocHelper.writeDoc(dir, new WhitespaceAnalyzer(), Similarity.getDefault(), testDoc, 3);
+    
+    termInfosIndexDivisor = 2;
+    expectedTermInfosIndexDivisor = 2;
+   
+    try {
+      testTermDocs();
+      testBadSeek();
+      testSkipTo();
+    } finally {
+     termInfosIndexDivisor = 1;
+     expectedTermInfosIndexDivisor = 1;
+    }
   }
-  
+   
   public void testIndexDivisorAfterLoad() throws IOException {
     dir = new MockRAMDirectory();
     testDoc = new Document();
Index: src/test/org/apache/lucene/index/TestSegmentReader.java
===================================================================
--- src/test/org/apache/lucene/index/TestSegmentReader.java	(revision 596211)
+++ src/test/org/apache/lucene/index/TestSegmentReader.java	(working copy)
@@ -207,11 +207,35 @@
     assertTrue("We do not have 4 term freq vectors, we have: " + results.length, results.length == 4);      
   }    
   
+  public void textTermInfosConfigurer() throws IOException {
+    dir = new RAMDirectory();
+    testDoc = new Document();
+    DocHelper.setupDoc(testDoc);
+    SegmentInfo si = DocHelper.writeDoc(dir, new WhitespaceAnalyzer(), Similarity.getDefault(), testDoc, 2);
+    
+    reader = SegmentReader.get(si);
+    reader.setTermInfosConfigurer(new TermInfosConfigurer(){
+      public int getMaxTermsCached(String segmentName, int segmentNumDocs, long segmentNumTerms) {
+        assertTrue("Not enough terms in segment to test TermInfosConfigurer:  " + segmentNumTerms, segmentNumTerms>=8);
+        return (int) (segmentNumTerms/4-1);
+      }
+    });
+    assertEquals(3, reader.getTermInfosIndexDivisor());
+
+    testDocument();
+    testDelete();
+    testGetFieldNameVariations();
+    testNorms();
+    testTerms();
+    testTermVectors();
+    
+  }
+  
   public void testIndexDivisor() throws IOException {
     dir = new MockRAMDirectory();
     testDoc = new Document();
     DocHelper.setupDoc(testDoc);
-    SegmentInfo si = DocHelper.writeDoc(dir, testDoc);
+    SegmentInfo si = DocHelper.writeDoc(dir, new WhitespaceAnalyzer(), Similarity.getDefault(), testDoc, 5);
     
     reader = SegmentReader.get(si);
     reader.setTermInfosIndexDivisor(3);
Index: src/java/org/apache/lucene/index/ParallelReader.java
===================================================================
--- src/java/org/apache/lucene/index/ParallelReader.java	(revision 596211)
+++ src/java/org/apache/lucene/index/ParallelReader.java	(working copy)
@@ -451,6 +451,30 @@
     return fieldSet;
   }
 
+  public void setTermInfosConfigurer(TermInfosConfigurer configurer) throws IllegalStateException {
+    for (int i = 0; i < readers.size(); i++)
+      ((IndexReader)readers.get(i)).setTermInfosConfigurer(configurer);
+  }
+ 
+  public TermInfosConfigurer getTermInfosConfigurer() throws IllegalStateException {
+    if (readers.size() > 0)
+      return ((IndexReader)readers.get(0)).getTermInfosConfigurer();
+    else
+      throw new IllegalStateException("no readers");
+  }
+
+  public void setTermInfosIndexDivisor(int indexDivisor) throws IllegalStateException {
+    for (int i = 0; i < readers.size(); i++)
+      ((IndexReader)readers.get(i)).setTermInfosIndexDivisor(indexDivisor);
+  }
+ 
+  public int getTermInfosIndexDivisor() throws IllegalStateException {
+    if (readers.size() > 0)
+      return ((IndexReader)readers.get(0)).getTermInfosIndexDivisor();
+    else
+      throw new IllegalStateException("no readers");
+  }
+
   private class ParallelTermEnum extends TermEnum {
     private String field;
     private Iterator fieldIterator;
Index: src/java/org/apache/lucene/index/SegmentReader.java
===================================================================
--- src/java/org/apache/lucene/index/SegmentReader.java	(revision 596211)
+++ src/java/org/apache/lucene/index/SegmentReader.java	(working copy)
@@ -51,6 +51,8 @@
   private FieldsReader fieldsReader;
 
   TermInfosReader tis;
+  private TermInfosConfigurer termInfosConfigurer = null;
+  
   TermVectorsReader termVectorsReaderOrig = null;
   ThreadLocal termVectorsLocal = new ThreadLocal();
 
@@ -350,6 +352,16 @@
     }
   }
   
+  private void configure() {
+    if (termInfosConfigurer!=null) {
+      int maxCacheTerms = termInfosConfigurer.getMaxTermsCached(si.name, si.docCount, tis.size());
+      long numIndexTerms = tis.getNumIndexTerms();
+      if (maxCacheTerms<numIndexTerms) {
+        tis.setIndexDivisor((int) (1+(numIndexTerms-1)/maxCacheTerms));
+      }
+    }
+  }
+  
   private void loadDeletedDocs() throws IOException {
     // NOTE: the bitvector is stored using the regular directory, not cfs
     if (hasDeletions(si)) {
@@ -415,6 +427,7 @@
   
       clone.fieldInfos = fieldInfos;
       clone.tis = tis;
+      clone.termInfosConfigurer = termInfosConfigurer;
       clone.freqStream = freqStream;
       clone.proxStream = proxStream;
       clone.termVectorsReaderOrig = termVectorsReaderOrig;
@@ -700,6 +713,18 @@
     return si.docCount;
   }
 
+  public void setTermInfosConfigurer(TermInfosConfigurer configurer) {
+    if (tis.isIndexRead())
+      throw new IllegalStateException("index terms are already loaded");
+
+    this.termInfosConfigurer = configurer;
+    configure();
+  }
+
+  public TermInfosConfigurer getTermInfosConfigurer() {
+    return termInfosConfigurer;
+  }
+  
   public void setTermInfosIndexDivisor(int indexDivisor) throws IllegalStateException {
     tis.setIndexDivisor(indexDivisor);
   }
Index: src/java/org/apache/lucene/index/TermInfosConfigurer.java
===================================================================
--- src/java/org/apache/lucene/index/TermInfosConfigurer.java	(revision 0)
+++ src/java/org/apache/lucene/index/TermInfosConfigurer.java	(revision 0)
@@ -0,0 +1,25 @@
+/*
+ * TermInfosConfigurer.java
+ *
+ * Created on November 18, 2007, 7:56 AM
+ */
+
+package org.apache.lucene.index;
+
+/**
+ * Support for dynamic configuration of index term caching
+ *
+ * @author chuck
+ */
+public interface TermInfosConfigurer {
+    
+  /** Determine the maximum number of terms that can be cached in memory for segment.
+   *  The actual number of terms cached will be <= this bound.
+   * @param segmentName the name of this segment (base file name of segment files)
+   * @param segmentNumDocs number of documents in a segment
+   * @param segmentNumTerms number of terms in a segment index
+   * @return the maximum number of terms allowed to be cached in memory for this segment
+   */
+  public int getMaxTermsCached(String segmentName, int segmentNumDocs, long segmentNumTerms);
+    
+}
Index: src/java/org/apache/lucene/index/TermInfosReader.java
===================================================================
--- src/java/org/apache/lucene/index/TermInfosReader.java	(revision 596211)
+++ src/java/org/apache/lucene/index/TermInfosReader.java	(working copy)
@@ -124,6 +124,15 @@
     return indexDivisor;
   }
   
+  /** Return true iff the index has already been read */
+  public boolean isIndexRead() {
+    return indexTerms!=null;
+  }
+
+  long getNumIndexTerms() {
+    return indexEnum.size;
+  }
+  
   final void close() throws IOException {
     if (origEnum != null)
       origEnum.close();
Index: src/java/org/apache/lucene/index/IndexReader.java
===================================================================
--- src/java/org/apache/lucene/index/IndexReader.java	(revision 596211)
+++ src/java/org/apache/lucene/index/IndexReader.java	(working copy)
@@ -354,6 +354,36 @@
     throw new UnsupportedOperationException("This reader does not support this method.");
   }
 
+  /** <p>For IndexReader implementations that use
+   *  TermInfosReader to read terms, this sets the
+   *  termInfosConfigurer, which can be used to dynamically
+   *  vary configuration parameters, e.g. termInfosIndexDivisor,
+   *  as a function of the segment.</p>
+   *
+   * @see TermInfosConfigurer
+   * @see #setTermInfosIndexDivisor
+   *
+   * <b>NOTE:</b> you must call this before the term
+   * index is loaded.  If the index is already loaded,
+   * an IllegalStateException is thrown.
+   *
+   * @throws IllegalStateException if the term index has
+   * already been loaded into memory.
+   */
+  public void setTermInfosConfigurer(TermInfosConfigurer configurer) throws IllegalStateException {
+    throw new UnsupportedOperationException("This reader does not support this method.");
+  }
+
+  /** <p>For IndexReader implementations that use
+   *  TermInfosReader to read terms, this returns the
+   *  termInfosConfigurer.
+   *
+   * @see #setTermInfosConfigurer
+   */
+  public TermInfosConfigurer getTermInfosConfigurer() {
+    throw new UnsupportedOperationException("This reader does not support this method.");
+  }
+  
   /**<p>For IndexReader implementations that use
    * TermInfosReader to read terms, this sets the
    * indexDivisor to subsample the number of indexed terms
Index: src/java/org/apache/lucene/index/MultiSegmentReader.java
===================================================================
--- src/java/org/apache/lucene/index/MultiSegmentReader.java	(revision 596211)
+++ src/java/org/apache/lucene/index/MultiSegmentReader.java	(working copy)
@@ -432,6 +432,18 @@
     return subReaders;
   }
 
+  public void setTermInfosConfigurer(TermInfosConfigurer configurer) throws IllegalStateException {
+    for (int i = 0; i < subReaders.length; i++)
+      subReaders[i].setTermInfosConfigurer(configurer);
+  }
+ 
+  public TermInfosConfigurer getTermInfosConfigurer() throws IllegalStateException {
+    if (subReaders.length > 0)
+      return subReaders[0].getTermInfosConfigurer();
+    else
+      throw new IllegalStateException("no readers");
+  }
+
   public void setTermInfosIndexDivisor(int indexDivisor) throws IllegalStateException {
     for (int i = 0; i < subReaders.length; i++)
       subReaders[i].setTermInfosIndexDivisor(indexDivisor);