Index: src/java/org/apache/lucene/analysis/AttributedToken.java
===================================================================
--- src/java/org/apache/lucene/analysis/AttributedToken.java	(revision 0)
+++ src/java/org/apache/lucene/analysis/AttributedToken.java	(revision 0)
@@ -0,0 +1,275 @@
+package org.apache.lucene.analysis;
+
+import java.io.Serializable;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.lucene.analysis.tokenattributes.Attribute;
+import org.apache.lucene.util.ArrayUtil;
+
+public class AttributedToken implements Serializable, Cloneable {
+  private static int MIN_BUFFER_SIZE = 10;
+  
+  private transient Map<Class<? extends Attribute>, Attribute> attributes;
+  
+  public void addAttribute(Attribute att) {
+    if (this.attributes == null) {
+      this.attributes = new HashMap<Class<? extends Attribute>, Attribute>();
+    }
+
+    this.attributes.put(att.getClass(), att);
+  }
+  
+  public boolean hasAttribute(Class<? extends Attribute> attClass) {
+    return this.attributes.containsKey(attClass);
+  }
+  
+  public Attribute getAttribute(Class<? extends Attribute> attClass) {
+    Attribute att = this.attributes.get(attClass);
+    if (att == null) {
+      throw new IllegalArgumentException("This token does not have the attribute '" + attClass + "'.");
+    }
+    
+    return att;
+  }
+  
+  public void copyFrom(AttributedToken prototype) {
+    prototype.initTermBuffer();
+    setTermBuffer(prototype.termBuffer, 0, prototype.termLength);
+    Iterator<Entry<Class<? extends Attribute>, Attribute>> it = prototype.attributes.entrySet().iterator();
+    while (it.hasNext()) {
+      Entry<Class<? extends Attribute>, Attribute> entry = it.next();
+      getAttribute(entry.getKey()).copyFrom(entry.getValue());
+    }
+    
+  }
+  
+  public Object clone() {
+    try {
+      AttributedToken t = (AttributedToken)super.clone();
+      // Do a deep clone
+      if (termBuffer != null) {
+        t.termBuffer = (char[]) termBuffer.clone();
+      }
+      if (attributes != null) {
+        t.attributes = new HashMap<Class<? extends Attribute>, Attribute>();
+        Iterator<Entry<Class<? extends Attribute>, Attribute>> it = attributes.entrySet().iterator();
+        while (it.hasNext()) {
+          Entry<Class<? extends Attribute>, Attribute> entry = it.next();
+          Attribute clone = (Attribute) entry.getValue().clone();
+          t.attributes.put(entry.getKey(), clone);
+        }
+      }
+      
+      return t;
+    } catch (CloneNotSupportedException e) {
+      throw new RuntimeException(e);  // shouldn't happen
+    }
+  }
+  
+  /** Resets the term text, payload, flags, and positionIncrement to default.
+   * Other fields such as startOffset, endOffset and the token type are
+   * not reset since they are normally overwritten by the tokenizer. */
+  public void clear() {
+    // Leave termBuffer to allow re-use
+    termLength = 0;
+  }
+  
+  /**
+   * Characters for the term text.
+   * @deprecated This will be made private. Instead, use:
+   * {@link termBuffer()}, 
+   * {@link #setTermBuffer(char[], int, int)},
+   * {@link #setTermBuffer(String)}, or
+   * {@link #setTermBuffer(String, int, int)}
+   */
+  char[] termBuffer;
+
+  /**
+   * Length of term text in the buffer.
+   * @deprecated This will be made private. Instead, use:
+   * {@link termLength()}, or @{link setTermLength(int)}.
+   */
+  int termLength;
+
+  
+  /** Constructs a Token will null text. */
+  public AttributedToken() {
+  }
+
+  /**
+   *  Constructs a Token with the given term buffer (offset
+   *  & length), start and end
+   *  offsets
+   * @param startTermBuffer
+   * @param termBufferOffset
+   * @param termBufferLength
+   * @param start
+   * @param end
+   */
+  public AttributedToken(char[] startTermBuffer, int termBufferOffset, int termBufferLength) {
+    setTermBuffer(startTermBuffer, termBufferOffset, termBufferLength);
+  }
+
+  /** Copies the contents of buffer, starting at offset for
+   *  length characters, into the termBuffer array.
+   *  @param buffer the buffer to copy
+   *  @param offset the index in the buffer of the first character to copy
+   *  @param length the number of characters to copy
+   */
+  public void setTermBuffer(char[] buffer, int offset, int length) {
+    char[] newCharBuffer = growTermBuffer(length);
+    if (newCharBuffer != null) {
+      termBuffer = newCharBuffer;
+    }
+    System.arraycopy(buffer, offset, termBuffer, 0, length);
+    termLength = length;
+  }
+  
+  /** Copies the contents of buffer into the termBuffer array.
+   *  @param buffer the buffer to copy
+   */
+  public void setTermBuffer(String buffer) {
+    int length = buffer.length();
+    char[] newCharBuffer = growTermBuffer(length);
+    if (newCharBuffer != null) {
+      termBuffer = newCharBuffer;
+    }
+    buffer.getChars(0, length, termBuffer, 0);
+    termLength = length;
+  }
+
+  /** Copies the contents of buffer, starting at offset and continuing
+   *  for length characters, into the termBuffer array.
+   *  @param buffer the buffer to copy
+   *  @param offset the index in the buffer of the first character to copy
+   *  @param length the number of characters to copy
+   */
+  public void setTermBuffer(String buffer, int offset, int length) {
+    assert offset <= buffer.length();
+    assert offset + length <= buffer.length();
+    char[] newCharBuffer = growTermBuffer(length);
+    if (newCharBuffer != null) {
+      termBuffer = newCharBuffer;
+    }
+    buffer.getChars(offset, offset + length, termBuffer, 0);
+    termLength = length;
+  }
+
+  /** Returns the internal termBuffer character array which
+   *  you can then directly alter.  If the array is too
+   *  small for your token, use {@link
+   *  #resizeTermBuffer(int)} to increase it.  After
+   *  altering the buffer be sure to call {@link
+   *  #setTermLength} to record the number of valid
+   *  characters that were placed into the termBuffer. */
+  public final char[] termBuffer() {
+    initTermBuffer();
+    return termBuffer;
+  }
+
+  /** Grows the termBuffer to at least size newSize, preserving the
+   *  existing content. Note: If the next operation is to change
+   *  the contents of the term buffer use
+   *  {@link #setTermBuffer(char[], int, int)},
+   *  {@link #setTermBuffer(String)}, or
+   *  {@link #setTermBuffer(String, int, int)}
+   *  to optimally combine the resize with the setting of the termBuffer.
+   *  @param newSize minimum size of the new termBuffer
+   *  @return newly created termBuffer with length >= newSize
+   */
+  public char[] resizeTermBuffer(int newSize) {
+    char[] newCharBuffer = growTermBuffer(newSize);
+    if (termBuffer == null) {
+      // If there were termText, then preserve it.
+      // note that if termBuffer is null then newCharBuffer cannot be null
+      assert newCharBuffer != null;
+      termBuffer = newCharBuffer;
+    } else if (newCharBuffer != null) {
+      // Note: if newCharBuffer != null then termBuffer needs to grow.
+      // If there were a termBuffer, then preserve it
+      System.arraycopy(termBuffer, 0, newCharBuffer, 0, termBuffer.length);
+      termBuffer = newCharBuffer;      
+    }
+    return termBuffer;
+  }
+
+
+  /** Allocates a buffer char[] of at least newSize
+   *  @param newSize minimum size of the buffer
+   *  @return newly created buffer with length >= newSize or null if the current termBuffer is big enough
+   */
+  char[] growTermBuffer(int newSize) {
+    if (termBuffer != null) {
+      if (termBuffer.length >= newSize)
+        // Already big enough
+        return null;
+      else
+        // Not big enough; create a new array with slight
+        // over allocation:
+        return new char[ArrayUtil.getNextSize(newSize)];
+    } else {
+
+      // determine the best size
+      // The buffer is always at least MIN_BUFFER_SIZE
+      if (newSize < MIN_BUFFER_SIZE) {
+        newSize = MIN_BUFFER_SIZE;
+      }
+
+      return new char[newSize];
+    }
+  }
+  
+  // TODO: once we remove the deprecated termText() method in Token
+  // and switch entirely to char[] termBuffer we don't need
+  // to use this method anymore
+  void initTermBuffer() {
+    if (termBuffer == null) {
+        termBuffer = new char[MIN_BUFFER_SIZE];
+        termLength = 0;
+    } 
+  }
+
+  /** Return number of valid characters (length of the term)
+   *  in the termBuffer array. */
+  public final int termLength() {
+    initTermBuffer();
+    return termLength;
+  }
+
+  /** Set number of valid characters (length of the term) in
+   *  the termBuffer array. Use this to truncate the termBuffer
+   *  or to synchronize with external manipulation of the termBuffer.
+   *  Note: to grow the size of the array,
+   *  use {@link #resizeTermBuffer(int)} first.
+   *  @param length the truncated length
+   */
+  public final void setTermLength(int length) {
+    initTermBuffer();
+    if (length > termBuffer.length)
+      throw new IllegalArgumentException("length " + length + " exceeds the size of the termBuffer (" + termBuffer.length + ")");
+    termLength = length;
+  }
+
+  public String toString() {
+    StringBuffer sb = new StringBuffer();
+    sb.append('(');
+    initTermBuffer();
+    if (termBuffer == null)
+      sb.append("null");
+    else
+      sb.append(termBuffer, 0, termLength);
+    
+    Iterator<Attribute> it = attributes.values().iterator();
+    while (it.hasNext()) {
+      sb.append(',');
+      sb.append(it.next().toString());
+    }
+    
+    sb.append(')');
+    return sb.toString();
+  }
+
+}
Index: src/java/org/apache/lucene/analysis/CharTokenizer.java
===================================================================
--- src/java/org/apache/lucene/analysis/CharTokenizer.java	(revision 704292)
+++ src/java/org/apache/lucene/analysis/CharTokenizer.java	(working copy)
@@ -20,11 +20,16 @@
 import java.io.IOException;
 import java.io.Reader;
 
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+
 /** An abstract base class for simple, character-oriented tokenizers.*/
 public abstract class CharTokenizer extends Tokenizer {
   public CharTokenizer(Reader input) {
     super(input);
   }
+  
+  private OffsetAttribute offsetAttribute;
+  private AttributedToken reusableToken;
 
   private int offset = 0, bufferIndex = 0, dataLen = 0;
   private static final int MAX_WORD_LEN = 255;
@@ -44,9 +49,18 @@
     return c;
   }
 
-  public final Token next(final Token reusableToken) throws IOException {
+  public AttributedToken prototypeToken() throws IOException {
+    reusableToken = new AttributedToken();
+    offsetAttribute = new OffsetAttribute();
+    reusableToken.addAttribute(offsetAttribute);
+    return reusableToken;
+  }
+  
+  public boolean nextToken() throws IOException {
     assert reusableToken != null;
     reusableToken.clear();
+    offsetAttribute.clear();
+    
     int length = 0;
     int start = bufferIndex;
     char[] buffer = reusableToken.termBuffer();
@@ -59,7 +73,7 @@
           if (length > 0)
             break;
           else
-            return null;
+            return false;
         }
         bufferIndex = 0;
       }
@@ -83,15 +97,61 @@
     }
 
     reusableToken.setTermLength(length);
+    offsetAttribute.setStartOffset(start);
+    offsetAttribute.setEndOffset(start+length);
+    return true;
+  }
+  
+  public final Token next(final Token reusableToken) throws IOException {
+    assert reusableToken != null;
+    reusableToken.clear();
+    int length = 0;
+    int start = bufferIndex;
+    char[] buffer = reusableToken.termBuffer();
+    while (true) {
+
+      if (bufferIndex >= dataLen) {
+        offset += dataLen;
+        dataLen = input.read(ioBuffer);
+        if (dataLen == -1) {
+          if (length > 0)
+            break;
+          else
+            return null;
+        }
+        bufferIndex = 0;
+      }
+
+      final char c = ioBuffer[bufferIndex++];
+
+      if (isTokenChar(c)) {               // if it's a token char
+
+        if (length == 0)                 // start of token
+          start = offset + bufferIndex - 1;
+        else if (length == buffer.length)
+          buffer = reusableToken.resizeTermBuffer(1+length);
+
+        buffer[length++] = normalize(c); // buffer it, normalized
+
+        if (length == MAX_WORD_LEN)      // buffer overflow!
+          break;
+
+      } else if (length > 0)             // at non-Letter w/ chars
+        break;                           // return 'em
+    }
+
+    reusableToken.setTermLength(length);
     reusableToken.setStartOffset(start);
     reusableToken.setEndOffset(start+length);
     return reusableToken;
   }
 
+
   public void reset(Reader input) throws IOException {
     super.reset(input);
     bufferIndex = 0;
     offset = 0;
     dataLen = 0;
   }
+  
 }
Index: src/java/org/apache/lucene/analysis/NewCachingTokenFilter.java
===================================================================
--- src/java/org/apache/lucene/analysis/NewCachingTokenFilter.java	(revision 0)
+++ src/java/org/apache/lucene/analysis/NewCachingTokenFilter.java	(revision 0)
@@ -0,0 +1,80 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * This class can be used if the Tokens of a TokenStream
+ * are intended to be consumed more than once. It caches
+ * all Tokens locally in a List.
+ * 
+ * CachingTokenFilter implements the optional method
+ * {@link TokenStream#reset()}, which repositions the
+ * stream to the first Token. 
+ *
+ */
+public class NewCachingTokenFilter extends TokenFilter {
+  private List<AttributedToken> cache;
+  private Iterator<AttributedToken> iterator;
+  private AttributedToken reusableToken;
+  
+  public NewCachingTokenFilter(TokenStream input) {
+    super(input);
+  }
+  
+  public AttributedToken prototypeToken() throws IOException {
+    reusableToken = input.prototypeToken();
+    return reusableToken;
+  }
+  
+  public boolean nextToken() throws IOException {
+    assert reusableToken != null;
+    if (cache == null) {
+      // fill cache lazily
+      cache = new LinkedList<AttributedToken>();
+      fillCache();
+      iterator = cache.iterator();
+    }
+    
+    if (!iterator.hasNext()) {
+      // the cache is exhausted, return null
+      return false;
+    }
+    // Since the TokenFilter can be reset, the tokens need to be preserved as immutable.
+    AttributedToken t = iterator.next();
+    reusableToken.copyFrom(t);
+    return true;
+  }
+  
+  public void reset() throws IOException {
+    if(cache != null) {
+      iterator = cache.iterator();
+    }
+  }
+  
+  private void fillCache() throws IOException {
+    while(input.nextToken()) {
+      cache.add((AttributedToken)reusableToken.clone());
+    }
+  }
+
+}
Index: src/java/org/apache/lucene/analysis/tokenattributes/Attribute.java
===================================================================
--- src/java/org/apache/lucene/analysis/tokenattributes/Attribute.java	(revision 0)
+++ src/java/org/apache/lucene/analysis/tokenattributes/Attribute.java	(revision 0)
@@ -0,0 +1,13 @@
+package org.apache.lucene.analysis.tokenattributes;
+
+import java.io.Serializable;
+
+public abstract class Attribute implements Cloneable, Serializable {
+  public abstract void clear();
+  public abstract String toString();
+  public abstract void copyFrom(Attribute prototype);
+  
+  public Object clone() throws CloneNotSupportedException {
+    return super.clone();
+  } 
+}
Index: src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java
===================================================================
--- src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java	(revision 0)
+++ src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java	(revision 0)
@@ -0,0 +1,45 @@
+package org.apache.lucene.analysis.tokenattributes;
+
+import java.io.Serializable;
+
+public class FlagsAttribute extends Attribute implements Cloneable, Serializable {
+  private int flags = 0;
+  
+  /**
+   * EXPERIMENTAL:  While we think this is here to stay, we may want to change it to be a long.
+   * <p/>
+   *
+   * Get the bitset for any bits that have been set.  This is completely distinct from {@link #type()}, although they do share similar purposes.
+   * The flags can be used to encode information about the token for use by other {@link org.apache.lucene.analysis.TokenFilter}s.
+   *
+   *
+   * @return The bits
+   */
+  public int getFlags() {
+    return flags;
+  }
+
+  /**
+   * @see #getFlags()
+   */
+  public void setFlags(int flags) {
+    this.flags = flags;
+  }
+  
+  @Override
+  public void clear() {
+    flags = 0;
+  }
+
+  @Override
+  public void copyFrom(Attribute prototype) {
+    FlagsAttribute flagsAtt = (FlagsAttribute) prototype;
+    flags = flagsAtt.flags;
+  }
+
+  @Override
+  public String toString() {
+    return "flags=" + flags;
+  }
+
+}
Index: src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java
===================================================================
--- src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java	(revision 0)
+++ src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java	(revision 0)
@@ -0,0 +1,57 @@
+package org.apache.lucene.analysis.tokenattributes;
+
+import java.io.Serializable;
+
+public class OffsetAttribute extends Attribute implements Cloneable, Serializable {
+  private int startOffset;
+  private int endOffset;
+
+  /** Returns this Token's starting offset, the position of the first character
+  corresponding to this token in the source text.
+
+  Note that the difference between endOffset() and startOffset() may not be
+  equal to termText.length(), as the term text may have been altered by a
+  stemmer or some other filter. */
+  public final int startOffset() {
+    return startOffset;
+  }
+
+  /** Set the starting offset.
+    @see #startOffset() */
+  public void setStartOffset(int offset) {
+    this.startOffset = offset;
+  }
+
+  /** Returns this Token's ending offset, one greater than the position of the
+  last character corresponding to this token in the source text. The length
+  of the token in the source text is (endOffset - startOffset). */
+  public final int endOffset() {
+    return endOffset;
+  }
+
+  /** Set the ending offset.
+    @see #endOffset() */
+  public void setEndOffset(int offset) {
+    this.endOffset = offset;
+  }
+
+  public void clear() {
+    startOffset = 0;
+    endOffset = 0;
+  }
+  
+  public String toString() {
+    return "start=" + startOffset + ",end=" + endOffset;
+  }
+  
+  public void copyFrom(Attribute prototype) {
+    OffsetAttribute p = (OffsetAttribute) prototype;
+    startOffset = p.startOffset;
+    endOffset = p.endOffset;
+  }
+  
+  public Object clone() throws CloneNotSupportedException {
+    return super.clone();
+  } 
+
+}
Index: src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java
===================================================================
--- src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java	(revision 0)
+++ src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java	(revision 0)
@@ -0,0 +1,56 @@
+package org.apache.lucene.analysis.tokenattributes;
+
+import java.io.Serializable;
+
+import org.apache.lucene.index.Payload;
+
+public class PayloadAttribute extends Attribute implements Cloneable, Serializable {
+  private Payload payload;  
+  
+  public PayloadAttribute() {}
+  
+  public PayloadAttribute(Payload payload) {
+    this.payload = payload;
+  }
+  
+  /**
+   * Returns this Token's payload.
+   */ 
+  public Payload getPayload() {
+    return this.payload;
+  }
+
+  /** 
+   * Sets this Token's payload.
+   */
+  public void setPayload(Payload payload) {
+    this.payload = payload;
+  }
+  
+  @Override
+  public void clear() {
+    payload = null;
+  }
+
+  @Override
+  public void copyFrom(Attribute prototype) {
+    PayloadAttribute payloadAtt = (PayloadAttribute) prototype;
+    if (payloadAtt.payload != null) {
+      this.payload = (Payload) payloadAtt.payload.clone();
+    }
+  }
+
+  @Override
+  public String toString() {
+    return payload.toString();
+  }
+  
+  public Object clone() throws CloneNotSupportedException {
+    PayloadAttribute clone = (PayloadAttribute) super.clone();
+    if (payload != null) {
+      clone.payload = (Payload) payload.clone();
+    }
+    return clone;
+  }
+
+}
Index: src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java
===================================================================
--- src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java	(revision 0)
+++ src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java	(revision 0)
@@ -0,0 +1,62 @@
+package org.apache.lucene.analysis.tokenattributes;
+
+import java.io.Serializable;
+
+import org.apache.lucene.analysis.TokenStream;
+
+public class PositionIncrementAttribute extends Attribute implements Cloneable, Serializable {
+  private int positionIncrement = 1;
+  
+  /** Set the position increment.  This determines the position of this token
+   * relative to the previous Token in a {@link TokenStream}, used in phrase
+   * searching.
+   *
+   * <p>The default value is one.
+   *
+   * <p>Some common uses for this are:<ul>
+   *
+   * <li>Set it to zero to put multiple terms in the same position.  This is
+   * useful if, e.g., a word has multiple stems.  Searches for phrases
+   * including either stem will match.  In this case, all but the first stem's
+   * increment should be set to zero: the increment of the first instance
+   * should be one.  Repeating a token with an increment of zero can also be
+   * used to boost the scores of matches on that token.
+   *
+   * <li>Set it to values greater than one to inhibit exact phrase matches.
+   * If, for example, one does not want phrases to match across removed stop
+   * words, then one could build a stop word filter that removes stop words and
+   * also sets the increment to the number of stop words removed before each
+   * non-stop word.  Then exact phrase queries will only match when the terms
+   * occur with no intervening stop words.
+   *
+   * </ul>
+   * @param positionIncrement the distance from the prior term
+   * @see org.apache.lucene.index.TermPositions
+   */
+  public void setPositionIncrement(int positionIncrement) {
+    if (positionIncrement < 0)
+      throw new IllegalArgumentException
+        ("Increment must be zero or greater: " + positionIncrement);
+    this.positionIncrement = positionIncrement;
+  }
+
+  /** Returns the position increment of this Token.
+   * @see #setPositionIncrement
+   */
+  public int getPositionIncrement() {
+    return positionIncrement;
+  }
+
+  public void clear() {
+    this.positionIncrement = 1;
+  }
+  
+  public String toString() {
+    return "positionIncrement=" + positionIncrement;
+  }
+
+  public void copyFrom(Attribute prototype) {
+    PositionIncrementAttribute p = (PositionIncrementAttribute) prototype;
+    positionIncrement = p.positionIncrement;
+  }
+}
Index: src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java
===================================================================
--- src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java	(revision 0)
+++ src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java	(revision 0)
@@ -0,0 +1,44 @@
+package org.apache.lucene.analysis.tokenattributes;
+
+import java.io.Serializable;
+
+public class TypeAttribute extends Attribute implements Cloneable, Serializable {
+  private String type;
+  public static final String DEFAULT_TYPE = "word";
+  
+  public TypeAttribute() {
+    this(DEFAULT_TYPE); 
+  }
+  
+  public TypeAttribute(String type) {
+    this.type = type;
+  }
+  
+  /** Returns this Token's lexical type.  Defaults to "word". */
+  public final String type() {
+    return type;
+  }
+
+  /** Set the lexical type.
+      @see #type() */
+  public final void setType(String type) {
+    this.type = type;
+  }
+
+  @Override
+  public void clear() {
+    type = DEFAULT_TYPE;    
+  }
+
+  @Override
+  public void copyFrom(Attribute prototype) {
+    // String is immutable, no clone necessary
+    TypeAttribute typeAtt = (TypeAttribute) prototype;
+    type = typeAtt.type;
+  }
+
+  @Override
+  public String toString() {
+    return "type=" + type;
+  }
+}
Index: src/java/org/apache/lucene/analysis/TokenFilter.java
===================================================================
--- src/java/org/apache/lucene/analysis/TokenFilter.java	(revision 704292)
+++ src/java/org/apache/lucene/analysis/TokenFilter.java	(working copy)
@@ -45,4 +45,12 @@
     super.reset();
     input.reset();
   }
+  
+  public AttributedToken prototypeToken() throws IOException {
+    return input.prototypeToken();
+  }
+  
+  public boolean nextToken() throws IOException {
+    return input.nextToken();
+  }
 }
Index: src/java/org/apache/lucene/analysis/Tokenizer.java
===================================================================
--- src/java/org/apache/lucene/analysis/Tokenizer.java	(revision 704292)
+++ src/java/org/apache/lucene/analysis/Tokenizer.java	(working copy)
@@ -43,7 +43,7 @@
   protected Tokenizer(Reader input) {
     this.input = input;
   }
-
+  
   /** By default, closes the input Reader. */
   public void close() throws IOException {
     input.close();
Index: src/java/org/apache/lucene/analysis/TokenStream.java
===================================================================
--- src/java/org/apache/lucene/analysis/TokenStream.java	(revision 704292)
+++ src/java/org/apache/lucene/analysis/TokenStream.java	(working copy)
@@ -37,6 +37,17 @@
   */
 
 public abstract class TokenStream {
+  public TokenStream() {
+    super();
+  }
+  
+  public AttributedToken prototypeToken() throws IOException {
+      return null;
+  }
+  
+  public boolean nextToken() throws IOException {
+    return false;
+  }
 
   /** Returns the next token in the stream, or null at EOS.
    *  @deprecated The returned Token is a "full private copy" (not
Index: src/test/org/apache/lucene/analysis/TestNewTokenStreamAPI.java
===================================================================
--- src/test/org/apache/lucene/analysis/TestNewTokenStreamAPI.java	(revision 0)
+++ src/test/org/apache/lucene/analysis/TestNewTokenStreamAPI.java	(revision 0)
@@ -0,0 +1,169 @@
+package org.apache.lucene.analysis;
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+
+public class TestNewTokenStreamAPI {
+  static StringReader getStringReader() {
+    String s = "Hello, this is the new Token API!!";
+    return new StringReader(s);
+  }
+  
+  static void testPerformanceNew(int numTokens, int numReads) throws Exception {
+    TokenStream ts = new NewCachingTokenFilter(new RepeatingTokenFilter(new WhitespaceTokenizer(getStringReader()), numTokens));
+    AttributedToken reusableToken = ts.prototypeToken();
+    reusableToken.addAttribute(new PayloadAttribute(null));
+    reusableToken.addAttribute(new PositionIncrementAttribute());
+    reusableToken.addAttribute(new FlagsAttribute());
+    reusableToken.addAttribute(new TypeAttribute());
+    
+    for (int i = 0; i < numReads; i++) {
+      while (ts.nextToken()) {
+      }
+      ts.reset();
+    }
+    
+    
+  }
+
+  static void testPerformanceOld(int numTokens, int numReads) throws Exception {
+    TokenStream ts = new CachingTokenFilter(new RepeatingTokenFilter(new WhitespaceTokenizer(getStringReader()), numTokens));
+
+    Token reusableToken = new Token();
+    
+    for (int i = 0; i < numReads; i++) {
+      while (reusableToken != null) {
+        reusableToken = ts.next(reusableToken);
+      }
+      ts.reset();
+      reusableToken = new Token();
+    }
+    
+    
+  }
+
+  
+  static class RepeatingTokenFilter extends TokenFilter {
+    protected RepeatingTokenFilter(TokenStream input, int n) {
+      super(input);
+      this.n = n;
+    }
+    
+    int n;
+    public boolean nextToken() throws IOException {
+      boolean next = input.nextToken();
+      if (!next && n > 0) {
+        n--;
+        ((WhitespaceTokenizer)input).reset(getStringReader());
+        return input.nextToken();
+      }
+      
+      return next;
+    }
+    
+    public Token next(Token reusableToken) throws IOException {
+      Token token = input.next(reusableToken);
+      if (token == null && n > 0) {
+        n--;
+        ((WhitespaceTokenizer)input).reset(getStringReader());
+        token = input.next(reusableToken);
+      }
+      
+      return token;
+    }
+  }
+
+  static void testPerformance() throws Exception {
+    final int numReads = 200;
+    final int numTokens = 10000;
+    
+    long start = System.currentTimeMillis();
+    testPerformanceOld(numTokens, numReads);
+    long end = System.currentTimeMillis();
+    System.out.println("Time old: " + (end - start) + "ms.");
+    
+    start = System.currentTimeMillis();
+    testPerformanceNew(numTokens, numReads);
+    end = System.currentTimeMillis();
+    System.out.println("Time new: " + (end - start) + "ms.");
+  }
+  
+  public static void main(String[] args) throws Exception {
+    //testPerformance();
+    demo();
+  }
+  
+  static void demo() throws Exception {
+    MyTokenFilter tf = new MyTokenFilter(new NewCachingTokenFilter(new WhitespaceTokenizer(getStringReader())));
+    
+    AttributedToken token = tf.prototypeToken();
+    PositionIncrementAttribute posAtt = (PositionIncrementAttribute) token.getAttribute(PositionIncrementAttribute.class);
+    OffsetAttribute offAtt = (OffsetAttribute) token.getAttribute(OffsetAttribute.class);
+    
+    tf.setIncrement(5);
+    
+    int totalLength = 0;
+    int position = 0;
+    
+    while (tf.nextToken()) {
+      System.out.println(token);
+      totalLength += (offAtt.endOffset() - offAtt.startOffset());
+      position += posAtt.getPositionIncrement();
+    }
+    
+    System.out.println("Position: " + position);
+    System.out.println("Total legnth: " + totalLength);
+    System.out.println("\n");
+    
+    tf.reset();
+    tf.setIncrement(10);
+    
+    totalLength = 0;
+    position = 0;
+    
+    while (tf.nextToken()) {
+      System.out.println(token);
+      totalLength += (offAtt.endOffset() - offAtt.startOffset());
+      position += posAtt.getPositionIncrement();
+    }
+    
+    System.out.println("Position: " + position);
+    System.out.println("Total legnth: " + totalLength);
+
+  }
+  
+  
+  private static class MyTokenFilter extends TokenFilter {
+    private PositionIncrementAttribute positionIncrement;
+    private int increment;
+    
+    protected MyTokenFilter(TokenStream input) {
+      super(input);
+      this.increment = 1;
+    }
+    
+    void setIncrement(int increment) {
+      this.increment = increment;
+    }
+
+    public AttributedToken prototypeToken() throws IOException {
+      AttributedToken t = input.prototypeToken();
+      positionIncrement = new PositionIncrementAttribute();
+      t.addAttribute(positionIncrement);
+      return t;
+    }
+    
+    public boolean nextToken() throws IOException {
+      boolean next = input.nextToken();
+      positionIncrement.setPositionIncrement(increment);
+      return next;
+    }
+  }
+  
+}