Index: src/java/org/apache/lucene/document/Fieldable.java
===================================================================
--- src/java/org/apache/lucene/document/Fieldable.java	(revision 785064)
+++ src/java/org/apache/lucene/document/Fieldable.java	(working copy)
@@ -89,9 +89,8 @@
    * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
   public byte[] binaryValue();
   
-  /** The value of the field as a TokenStream, or null.  If null, the Reader value,
-   * String value, or binary value is used. Exactly one of stringValue(), 
-   * readerValue(), binaryValue(), and tokenStreamValue() must be set. */
+  /** The TokesStream for this field to be used when indexing, or null.  If null, the Reader value,
+   * or String value is used to produce the indexed tokens. */
   public TokenStream tokenStreamValue();
 
   /** True iff the value of the field is to be stored in the index for return
Index: src/java/org/apache/lucene/document/AbstractField.java
===================================================================
--- src/java/org/apache/lucene/document/AbstractField.java	(revision 785064)
+++ src/java/org/apache/lucene/document/AbstractField.java	(working copy)
@@ -16,7 +16,8 @@
  */
 
 import org.apache.lucene.search.PhraseQuery; // for javadocs
-import org.apache.lucene.search.spans.SpanQuery; // for javadocs
+import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.analysis.TokenStream; // for javadocs
 
 
 /**
@@ -38,9 +39,11 @@
   protected boolean lazy = false;
   protected boolean omitTermFreqAndPositions = false;
   protected float boost = 1.0f;
-  // the one and only data object for all different kind of field values
+  // the data object for all different kind of field values
   protected Object fieldsData = null;
-  //length/offset for all primitive types
+  // pre-analyzed tokenStream for indexed fields
+  protected TokenStream tokenStream;
+  // length/offset for all primitive types
   protected int binaryLength;
   protected int binaryOffset;
 
Index: src/java/org/apache/lucene/document/Field.java
===================================================================
--- src/java/org/apache/lucene/document/Field.java	(revision 785064)
+++ src/java/org/apache/lucene/document/Field.java	(working copy)
@@ -160,18 +160,18 @@
   
   
   /** The value of the field as a String, or null.  If null, the Reader value,
-   * binary value, or TokenStream value is used.  Exactly one of stringValue(), 
-   * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
+   * binary value is used.  Exactly one of stringValue(),
+   * readerValue(), and getBinaryValue() must be set. */
   public String stringValue()   { return fieldsData instanceof String ? (String)fieldsData : null; }
   
-  /** The value of the field as a Reader, or null.  If null, the String value,
-   * binary value, or TokenStream value is used.  Exactly one of stringValue(), 
-   * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
+  /** The value of the field as a String, or null.  If null, the Reader value,
+   * binary value is used.  Exactly one of stringValue(),
+   * readerValue(), and getBinaryValue() must be set. */
   public Reader readerValue()   { return fieldsData instanceof Reader ? (Reader)fieldsData : null; }
   
   /** The value of the field in Binary, or null.  If null, the Reader value,
-   * String value, or TokenStream value is used. Exactly one of stringValue(), 
-   * readerValue(), getBinaryValue(), and tokenStreamValue() must be set.
+   * or String value is used. Exactly one of stringValue(),
+   * readerValue(), and getBinaryValue() must be set.
    * @deprecated This method must allocate a new byte[] if
    * the {@link AbstractField#getBinaryOffset()} is non-zero
    * or {@link AbstractField#getBinaryLength()} is not the
@@ -191,10 +191,9 @@
     return ret;    
   }
   
-  /** The value of the field as a TokesStream, or null.  If null, the Reader value,
-   * String value, or binary value is used. Exactly one of stringValue(), 
-   * readerValue(), getBinaryValue(), and tokenStreamValue() must be set. */
-  public TokenStream tokenStreamValue()   { return fieldsData instanceof TokenStream ? (TokenStream)fieldsData : null; }
+  /** The TokesStream for this field to be used when indexing, or null.  If null, the Reader value,
+   * or String value is used to produce the indexed tokens. */
+  public TokenStream tokenStreamValue()   { return tokenStream; }
   
 
   /** <p>Expert: change the value of this field.  This can
@@ -258,9 +257,15 @@
     if (isStored) {
       throw new IllegalArgumentException("cannot set a TokenStream value on a stored field");
     }
-    fieldsData = value;
+    fieldsData = null;
+    tokenStream = value;
   }
 
+  /** Expert: sets the token stream to be used for indexing.  May be combined with other types of stored fields. */
+  public void setTokenStream(TokenStream tokenStream) {
+    this.tokenStream = tokenStream;
+  }
+
   /**
    * Create a field by specifying its name, value and how it will
    * be saved in the index. Term vectors will not be stored in the index.
@@ -459,8 +464,9 @@
       throw new NullPointerException("tokenStream cannot be null");
     
     this.name = name.intern();        // field names are interned
-    this.fieldsData = tokenStream;
-    
+    this.fieldsData = null;
+    this.tokenStream = tokenStream;
+
     this.isStored = false;
     this.isCompressed = false;