Index: src/test/org/apache/lucene/index/TestDocumentWriter.java
===================================================================
--- src/test/org/apache/lucene/index/TestDocumentWriter.java	(revision 414705)
+++ src/test/org/apache/lucene/index/TestDocumentWriter.java	(working copy)
@@ -16,11 +16,15 @@
  * limitations under the License.
  */
 
+import java.util.LinkedList;
+import java.util.List;
 import junit.framework.TestCase;
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.WhitespaceAnalyzer;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
+import org.apache.lucene.analysis.TokenSelector;
 import org.apache.lucene.document.*;
 import org.apache.lucene.search.Similarity;
 import org.apache.lucene.store.RAMDirectory;
@@ -54,6 +58,16 @@
     Analyzer analyzer = new WhitespaceAnalyzer();
     Similarity similarity = Similarity.getDefault();
     DocumentWriter writer = new DocumentWriter(dir, analyzer, similarity, 50);
+    writer.setTermVectorTokenSelector(new TokenSelector(){
+      public boolean accept(String field, Token t) {
+        return Character.isLowerCase(t.termText().charAt(0));
+      }
+    });
+    writer.setPositionsTokenSelector(new TokenSelector(){
+      public boolean accept(String field, Token t) {
+        return Character.isLowerCase(t.termText().charAt(0));
+      }
+    });
     String segName = "test";
     writer.addDocument(segName, testDoc);
     //After adding the document, we should be able to read it back in
@@ -84,6 +98,31 @@
     fields = doc.getFields(DocHelper.TEXT_FIELD_3_KEY);
     assertTrue(fields != null && fields.length == 1);
     assertTrue(fields[0].stringValue().equals(DocHelper.FIELD_3_TEXT));
+    
+    fields = doc.getFields(DocHelper.TEXT_FIELD_UTF2_KEY);
+    assertTrue(fields != null && fields.length == 1);
+    assertTrue(fields[0].stringValue().equals(DocHelper.FIELD_UTF2_TEXT));
+    assertTrue(fields[0].isTermVectorStored());
+    TermFreqVector tv = reader.getTermFreqVector(0, DocHelper.TEXT_FIELD_UTF2_KEY);
+    assertTrue(tv != null);
+    String[] words = DocHelper.FIELD_UTF2_TEXT.split("\\s+");
+    String[] tvwords = tv.getTerms();
+    List uniques = new LinkedList();
+    int omitted = 0;
+    for (int i=0; i<words.length; i++)
+      if (!uniques.contains(words[i])) {
+        uniques.add(words[i]);
+        if (!Character.isLowerCase(words[i].charAt(0)))
+          omitted++;
+      }
+    assertTrue(omitted!=0);
+    assertTrue(omitted!=uniques.size());
+    assertEquals(uniques.size()-omitted, tvwords.length);
+    for (int i=0; i<uniques.size(); i++) {
+      for (int j=0; j<tvwords.length; j++)
+        if (uniques.get(i).equals(tvwords[j]))
+          assertTrue(Character.isLowerCase(((String)uniques.get(i)).charAt(0)));
+    }      
 
     // test that the norm file is not present if omitNorms is true
     for (int i = 0; i < reader.fieldInfos.size(); i++) {
Index: src/java/org/apache/lucene/analysis/TokenSelector.java
===================================================================
--- src/java/org/apache/lucene/analysis/TokenSelector.java	(revision 0)
+++ src/java/org/apache/lucene/analysis/TokenSelector.java	(revision 0)
@@ -0,0 +1,24 @@
+/*
+ * TokenSelector.java
+ *
+ * Created on June 13, 2006, 12:18 PM
+ *
+ */
+
+package org.apache.lucene.analysis;
+
+/**
+ * An interface for selecting a subset of a token stream
+ *
+ * @author Chuck Wiliams
+ */
+public interface TokenSelector {
+    
+  /** Determine if a token should be selected
+   * @param fieldName field in which token was found
+   * @param token a token
+   * @return true iff token should be selected
+   */
+  public boolean accept(String fieldName, Token token);
+    
+}
Index: src/java/org/apache/lucene/analysis/PerFieldTokenSelectorWrapper.java
===================================================================
--- src/java/org/apache/lucene/analysis/PerFieldTokenSelectorWrapper.java	(revision 0)
+++ src/java/org/apache/lucene/analysis/PerFieldTokenSelectorWrapper.java	(revision 0)
@@ -0,0 +1,44 @@
+/*
+ * PerFieldTokenSelectorWrapper.java
+ *
+ * Created on June 13, 2006, 4:09 PM
+ *
+ */
+
+package org.apache.lucene.analysis;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Expert: TokenSelector that implements a mapping from field names to TokenSelectors
+ *
+ * @author Chuck Williams
+ */
+public class PerFieldTokenSelectorWrapper implements TokenSelector {
+  
+  private Map selectors = new HashMap();
+  private TokenSelector defaultSelector;
+  
+  /** Expert: create a PerFieldTokenSelector with given default selector (null means select all) */
+  public PerFieldTokenSelectorWrapper(TokenSelector defaultSelector) {
+    this.defaultSelector = defaultSelector;
+  }
+  
+  /** Add a token selector for the named field */
+  public void addSelector(String fieldName, TokenSelector selector) {
+    selectors.put(fieldName, selector);
+  }
+  
+  /** Determine if token is accepted by fieldName */
+  public boolean accept(String fieldName, Token token) {
+    TokenSelector selector = (TokenSelector) selectors.get(fieldName);
+    if (selector!=null)
+        return selector.accept(fieldName, token);
+    else if (defaultSelector!=null)
+        return defaultSelector.accept(fieldName, token);
+    else
+        return true;
+  }
+    
+}
\ No newline at end of file
Index: src/java/org/apache/lucene/index/IndexWriter.java
===================================================================
--- src/java/org/apache/lucene/index/IndexWriter.java	(revision 414705)
+++ src/java/org/apache/lucene/index/IndexWriter.java	(working copy)
@@ -17,6 +17,7 @@
  */
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenSelector;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.search.Similarity;
 import org.apache.lucene.store.Directory;
@@ -100,8 +101,10 @@
    */
   public final static int DEFAULT_TERM_INDEX_INTERVAL = 128;
   
-  private Directory directory;  // where this index resides
-  private Analyzer analyzer;    // how to analyze text
+  private Directory directory;                      // where this index resides
+  private Analyzer analyzer;                        // how to analyze text
+  private TokenSelector termVectorTokenSelector;    // subset of token stream stored in term vectors
+  private TokenSelector positionsTokenSelector;     // subset of token stream for which positions are stored
 
   private Similarity similarity = Similarity.getDefault(); // how to normalize
 
@@ -153,6 +156,38 @@
     return this.similarity;
   }
 
+  /** Expert:  Set the TokenSelector used to determine subset of tokens stored in term vectors.
+   * @param selector the term vector TokenSelector
+   */
+  public void setTermVectorTokenSelector(TokenSelector selector) {
+    this.termVectorTokenSelector = selector;
+  }
+  
+  /** Expert: Set the TokenSelector used to determine subset of tokens stored in term vectors.
+   * @return the TokenSelector used to determine term vector tokens
+   */
+  public TokenSelector getTermVectorTokenSelector() {
+    return termVectorTokenSelector;
+  }
+
+  /** Expert:  Set the TokenSelector used to determine subset of tokens for which positions are stored.
+   *           (At least one position is always stored for each term in each doc to ensure the term stays in
+   *            the index so long as any docs reference it)
+   * @param selector the positions TokenSelector
+   */
+  public void setPositionsTokenSelector(TokenSelector selector) {
+    this.positionsTokenSelector = selector;
+  }
+  
+  /** Expert: Set the TokenSelector used to determine subset of tokens for which freq and positions are stored..
+   *          (At least one position is always stored for each term in each doc to ensure the term stays in
+   *           the index so long as any docs reference it)
+   * @return the positions TokenSelector
+   */
+  public TokenSelector getPositionsTokenSelector() {
+    return positionsTokenSelector;
+  }
+
   /** Expert: Set the interval between indexed terms.  Large values cause less
    * memory to be used by IndexReader, but slow random-access to terms.  Small
    * values cause more memory to be used by an IndexReader, and speed
@@ -471,6 +506,8 @@
   public void addDocument(Document doc, Analyzer analyzer) throws IOException {
     DocumentWriter dw =
       new DocumentWriter(ramDirectory, analyzer, this);
+    dw.setTermVectorTokenSelector(termVectorTokenSelector);
+    dw.setPositionsTokenSelector(positionsTokenSelector);
     dw.setInfoStream(infoStream);
     String segmentName = newSegmentName();
     dw.addDocument(segmentName, doc);
Index: src/java/org/apache/lucene/index/DocumentWriter.java
===================================================================
--- src/java/org/apache/lucene/index/DocumentWriter.java	(revision 414705)
+++ src/java/org/apache/lucene/index/DocumentWriter.java	(working copy)
@@ -17,6 +17,7 @@
  */
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenSelector;
 import org.apache.lucene.analysis.Token;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.document.Document;
@@ -35,6 +36,8 @@
 
 final class DocumentWriter {
   private Analyzer analyzer;
+  private TokenSelector termVectorTokenSelector;
+  private TokenSelector positionsTokenSelector;
   private Directory directory;
   private Similarity similarity;
   private FieldInfos fieldInfos;
@@ -142,9 +145,9 @@
         if (!field.isTokenized()) {		  // un-tokenized field
           String stringValue = field.stringValue();
           if(field.isStoreOffsetWithTermVector())
-            addPosition(fieldName, stringValue, position++, new TermVectorOffsetInfo(offset, offset + stringValue.length()));
+            addPosition(fieldName, stringValue, position++, new TermVectorOffsetInfo(offset, offset + stringValue.length()), false, false);
           else
-            addPosition(fieldName, stringValue, position++, null);
+            addPosition(fieldName, stringValue, position++, null, false, false);
           offset += stringValue.length();
           length++;
         } else 
@@ -165,10 +168,16 @@
             for (Token t = stream.next(); t != null; t = stream.next()) {
               position += (t.getPositionIncrement() - 1);
               
-              if(field.isStoreOffsetWithTermVector())
-                addPosition(fieldName, t.termText(), position++, new TermVectorOffsetInfo(offset + t.startOffset(), offset + t.endOffset()));
-              else
-                addPosition(fieldName, t.termText(), position++, null);
+              boolean omittv = false, omitpos = false;
+              if (termVectorTokenSelector!=null && !termVectorTokenSelector.accept(field.name(), t))
+                  omittv  = true;
+              if (positionsTokenSelector !=null && !positionsTokenSelector. accept(field.name(), t))
+                  omitpos = true;
+              
+              addPosition(fieldName, t.termText(), position++,
+                          field.isStoreOffsetWithTermVector() && !omittv ? new TermVectorOffsetInfo(offset + t.startOffset(), offset + t.endOffset())
+                                                                         : null,
+                          omittv, omitpos);
               
               lastToken = t;
               if (++length > maxFieldLength) {
@@ -196,20 +205,24 @@
 
   private final Term termBuffer = new Term("", ""); // avoid consing
 
-  private final void addPosition(String field, String text, int position, TermVectorOffsetInfo offset) {
+  private final void addPosition(String field, String text, int position, TermVectorOffsetInfo offset,
+                                 boolean omitFromTermVector, boolean omitPosition) {
     termBuffer.set(field, text);
     //System.out.println("Offset: " + offset);
     Posting ti = (Posting) postingTable.get(termBuffer);
     if (ti != null) {				  // word seen before
       int freq = ti.freq;
-      if (ti.positions.length == freq) {	  // positions array is full
-        int[] newPositions = new int[freq * 2];	  // double size
-        int[] positions = ti.positions;
-        for (int i = 0; i < freq; i++)		  // copy old positions to new
-          newPositions[i] = positions[i];
-        ti.positions = newPositions;
+      
+      if (!omitPosition) {
+        if (ti.positions.length == freq) {        // positions array is full
+          int[] newPositions = new int[freq * 2]; // double size
+          int[] positions = ti.positions;
+          for (int i = 0; i < freq; i++)          // copy old positions to new
+            newPositions[i] = positions[i];
+          ti.positions = newPositions;
+        }
+        ti.positions[freq] = position;            // add new position
       }
-      ti.positions[freq] = position;		  // add new position
 
       if (offset != null) {
         if (ti.offsets.length == freq){
@@ -223,10 +236,12 @@
         }
         ti.offsets[freq] = offset;
       }
-      ti.freq = freq + 1;			  // update frequency
-    } else {					  // word not seen before
+      
+      if (!omitPosition)
+        ti.freq = freq + 1;                       // update frequency
+    } else {                                      // word not seen before
       Term term = new Term(field, text, false);
-      postingTable.put(term, new Posting(term, position, offset));
+      postingTable.put(term, new Posting(term, position, offset, omitFromTermVector));
     }
   }
 
@@ -351,7 +366,7 @@
             termVectorWriter.closeField();
           }
         }
-        if (termVectorWriter != null && termVectorWriter.isFieldOpen()) {
+        if (termVectorWriter != null && termVectorWriter.isFieldOpen() && !posting.omitFromTermVector) {
             termVectorWriter.addTerm(posting.term.text(), postingFreq, posting.positions, posting.offsets);
         }
       }
@@ -390,6 +405,16 @@
     this.infoStream = infoStream;
   }
 
+  /** If non-null, this will be used to select which tokens are stored in term vectors */
+  void setTermVectorTokenSelector(TokenSelector selector) {
+    this.termVectorTokenSelector = selector;
+  }
+
+  /** If non-null, this will be used to select which tokens have positions stored in the index. */
+  void setPositionsTokenSelector(TokenSelector selector) {
+    this.positionsTokenSelector = selector;
+  }
+
 }
 
 final class Posting {				  // info about a Term in a doc
@@ -397,17 +422,17 @@
   int freq;					  // its frequency in doc
   int[] positions;				  // positions it occurs at
   TermVectorOffsetInfo [] offsets;
+  boolean omitFromTermVector;                     // if true, omit from term vector
 
-  Posting(Term t, int position, TermVectorOffsetInfo offset) {
+  Posting(Term t, int position, TermVectorOffsetInfo offset, boolean omitFromTermVector) {
     term = t;
     freq = 1;
     positions = new int[1];
     positions[0] = position;
-    if(offset != null){
-    offsets = new TermVectorOffsetInfo[1];
-    offsets[0] = offset;
+    if(offset != null) {
+      offsets = new TermVectorOffsetInfo[1];
+      offsets[0] = offset;
     }
-    else
-      offsets = null;
+    this.omitFromTermVector = omitFromTermVector;
   }
 }