Index: test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java
===================================================================
--- test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java	(revision 1530946)
+++ test-framework/src/java/org/apache/lucene/analysis/MockTokenizer.java	(working copy)
@@ -80,6 +80,8 @@
   private State streamState = State.CLOSE;
   private int lastOffset = 0; // only for asserting
   private boolean enableChecks = true;
+  private Integer lastCodePoint = null;
+  private int lastOff;
   
   // evil: but we don't change the behavior with this random, we only switch up how we read
   private final Random random = new Random(RandomizedContext.current().getRandom().nextLong());
@@ -121,8 +123,16 @@
                             : "incrementToken() called while in wrong state: " + streamState;
     clearAttributes();
     for (;;) {
-      int startOffset = off;
-      int cp = readCodePoint();
+      int startOffset;
+      int cp;
+      if ( lastCodePoint != null ) {
+        cp = lastCodePoint;
+        lastCodePoint = null;
+        startOffset = lastOff;
+      } else {
+        startOffset = off;
+        cp = readCodePoint();
+      }
       if (cp < 0) {
         break;
       } else if (isTokenChar(cp)) {
@@ -135,8 +145,10 @@
           if (termAtt.length() >= maxTokenLength) {
             break;
           }
+          lastOff = off;
           cp = readCodePoint();
         } while (cp >= 0 && isTokenChar(cp));
+        lastCodePoint = cp;
         
         int correctedStartOffset = correctOffset(startOffset);
         int correctedEndOffset = correctOffset(endOffset);
Index: core/src/test/org/apache/lucene/analysis/TestMockTokenizer.java
===================================================================
--- core/src/test/org/apache/lucene/analysis/TestMockTokenizer.java	(revision 0)
+++ core/src/test/org/apache/lucene/analysis/TestMockTokenizer.java	(working copy)
@@ -0,0 +1,65 @@
+package org.apache.lucene.analysis;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.automaton.CharacterRunAutomaton;
+import org.apache.lucene.util.automaton.RegExp;
+
+
+/**
+ * Test for MockTokenizer.
+ */
+public class TestMockTokenizer extends LuceneTestCase {
+  public void testSingleCharacterTokenizer() throws IOException {
+    TokenStream tokenizer = new MockTokenizer( new StringReader( "abcd" ),
+      new CharacterRunAutomaton( new RegExp(".").toAutomaton() ), true );
+    tokenizer.reset();
+    CharTermAttribute termAtt = tokenizer.addAttribute( CharTermAttribute.class );
+    assertTrue( tokenizer.incrementToken() );
+    assertEquals( "a", termAtt.toString() );
+    assertTrue( tokenizer.incrementToken() );
+    assertEquals( "b", termAtt.toString() );
+    assertTrue( tokenizer.incrementToken() );
+    assertEquals( "c", termAtt.toString() );
+    assertTrue( tokenizer.incrementToken() );
+    assertEquals( "d", termAtt.toString() );
+    assertFalse( tokenizer.incrementToken() );
+    tokenizer.end();
+    tokenizer.close();
+  }
+
+  public void testMultiCharacterTokenizer() throws IOException {
+    TokenStream tokenizer = new MockTokenizer( new StringReader( "abcd" ),
+      new CharacterRunAutomaton( new RegExp("...").toAutomaton() ), true );
+    tokenizer.reset();
+    CharTermAttribute termAtt = tokenizer.addAttribute( CharTermAttribute.class );
+    assertTrue( tokenizer.incrementToken() );
+    assertEquals( "abc", termAtt.toString() );
+    assertTrue( tokenizer.incrementToken() );
+    assertEquals( "d", termAtt.toString() );
+    assertFalse( tokenizer.incrementToken() );
+    tokenizer.end();
+    tokenizer.close();
+  }
+}