Index: src/java/org/apache/lucene/analysis/Tokenizer.java
===================================================================
--- src/java/org/apache/lucene/analysis/Tokenizer.java	(revision 813488)
+++ src/java/org/apache/lucene/analysis/Tokenizer.java	(working copy)
@@ -34,7 +34,7 @@
 
 public abstract class Tokenizer extends TokenStream {
   /** The text source for this Tokenizer. */
-  protected CharStream input;
+  protected Reader input;
 
   /** Construct a tokenizer with null input. */
   protected Tokenizer() {}
@@ -53,7 +53,7 @@
    *  analyzer (in its reusableTokenStream method) will use
    *  this to re-use a previously created tokenizer. */
   public void reset(Reader input) throws IOException {
-    this.input = CharReader.get(input);
+    this.input = input;
   }
 }
 
Index: src/test/org/apache/lucene/analysis/TestTokenizer.java
===================================================================
--- src/test/org/apache/lucene/analysis/TestTokenizer.java	(revision 0)
+++ src/test/org/apache/lucene/analysis/TestTokenizer.java	(revision 0)
@@ -0,0 +1,70 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+import java.io.StringReader;
+import java.io.IOException;
+import org.apache.lucene.util.LuceneTestCase;
+
+public class TestTokenizer extends LuceneTestCase {
+
+  public TestTokenizer(String name) {
+    super(name);
+  }
+  
+  public void testChangeToCharStream29() throws IOException {
+    Reader reader=new StringReader("test");
+    final Tokenizer testTokenizer=new Tokenizer(reader) {
+      boolean exhausted = false;
+      
+      public Token next(Token reusableToken) throws IOException {
+        assert reusableToken != null;
+        if (exhausted) return null;
+        exhausted = true;
+        reusableToken.clear();
+        StringBuffer sb = new StringBuffer();
+        int ch;
+        while ((ch = input.read()) != -1) {
+          sb.append((char) ch);
+        }
+        reusableToken.setTermBuffer(sb.toString());
+        return reusableToken;
+      }
+      
+      public void reset(Reader in) throws IOException {
+        super.reset(in);
+        exhausted = false;
+      }
+    };
+    
+    Token t = new Token();
+    assertNotNull(t = testTokenizer.next(t));
+    assertEquals("test", t.term());
+    assertNull(t = testTokenizer.next(t));
+    testTokenizer.close();
+    
+    t = new Token();
+    testTokenizer.reset(new StringReader("other"));
+    assertNotNull(t = testTokenizer.next(t));
+    assertEquals("other", t.term());
+    assertNull(t = testTokenizer.next(t));
+    testTokenizer.close();
+  }
+
+}

Property changes on: src\test\org\apache\lucene\analysis\TestTokenizer.java
___________________________________________________________________
Added: svn:keywords
   + Date Author Id Revision HeadURL
Added: svn:eol-style
   + native