Index: contrib/analyzers/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java
===================================================================
--- contrib/analyzers/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java	(revision 563468)
+++ contrib/analyzers/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java	(working copy)
@@ -175,8 +175,9 @@
                             length = 0;
                             preIsTokened = false;
 
-                            break;
+                            continue;
                         } else {
+                            tokenType = "double";
                             break;
                         }
                     }
@@ -236,8 +237,13 @@
             }
         }
 
-        return new Token(new String(buffer, 0, length), start, start + length,
+        String tokenString = new String(buffer, 0, length) ;
+        if( dataLen == -1 && "".equals(tokenString)) {
+          return null ;
+        } else {
+          return new Token(tokenString, start, start + length,
                          tokenType
                         );
+        }
     }
 }
Index: contrib/analyzers/src/test/org/apache/lucene/analysis/cjk/TestCJKTokenizer.java
===================================================================
--- contrib/analyzers/src/test/org/apache/lucene/analysis/cjk/TestCJKTokenizer.java	(revision 0)
+++ contrib/analyzers/src/test/org/apache/lucene/analysis/cjk/TestCJKTokenizer.java	(revision 0)
@@ -0,0 +1,141 @@
+package org.apache.lucene.analysis.cjk;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.io.StringReader;
+
+import junit.framework.TestCase;
+
+import org.apache.lucene.analysis.Token;
+
+public class TestCJKTokenizer extends TestCase{
+
+  public Token newToken(String termText, int start, int end, String typ) {
+    Token token = new Token(termText, start, end, typ);
+    return token;
+  }
+
+  public void checkCJKToken( final String str, final Token[] out_tokens ) throws IOException {
+    CJKTokenizer tokenizer = new CJKTokenizer(new StringReader(str));
+    Token token;
+    int i = 0 ;
+    System.out.println("string[" + str + "]") ;
+    System.out.print("token [") ;
+    while ((token = tokenizer.next()) != null)
+    {
+      if( token.termText().equals(out_tokens[i].termText()) &&
+          token.startOffset() == out_tokens[i].startOffset() &&
+          token.endOffset() == out_tokens[i].endOffset() &&
+          token.type().equals(out_tokens[i].type()) )
+      {
+        System.out.print( token.termText() + " ") ;
+      }
+      else {
+        fail( token.termText() + "!=" + out_tokens[i].termText() ) ;
+        break ;
+      }
+      i ++ ;
+    }
+    System.out.println("]") ;
+  }
+  public void testJa1() throws IOException
+  {
+    String str = "一二三四五六七八九十";
+       
+    Token[] out_tokens = { 
+        newToken("一二", 0, 2, "double"), 
+        newToken("二三", 1, 3, "double"),
+        newToken("三四", 2, 4, "double"),
+        newToken("四五", 3, 5, "double"), 
+        newToken("五六", 4, 6, "double"), 
+        newToken("六七", 5, 7, "double"),
+        newToken("七八", 6, 8, "double"),
+        newToken("八九", 7, 9, "double"),
+        newToken("九十", 8,10, "double")
+    };
+    checkCJKToken( str, out_tokens ) ;
+  }
+  public void testJa2() throws IOException
+  {
+    String str = "一 二三四 五六七八九 十";
+       
+    Token[] out_tokens = { 
+        newToken("一", 0, 1, "double"), 
+        newToken("二三", 2, 4, "double"),
+        newToken("三四", 3, 5, "double"),
+        newToken("五六", 6, 8, "double"), 
+        newToken("六七", 7, 9, "double"),
+        newToken("七八", 8, 10, "double"),
+        newToken("八九", 9, 11, "double"),
+        newToken("十", 12,13, "double")
+    };
+    checkCJKToken( str, out_tokens ) ;
+  }
+  public void testC() throws IOException
+  {
+    String str = "abc defgh ijklmn opqrstu vwxy z";
+       
+    Token[] out_tokens = { 
+        newToken("abc", 0, 3, "single"), 
+        newToken("defgh", 4, 9, "single"),
+        newToken("ijklmn", 10, 16, "single"),
+        newToken("opqrstu", 17, 24, "single"), 
+        newToken("vwxy", 25, 29, "single"), 
+        newToken("z", 30, 31, "single"),
+    };
+    checkCJKToken( str, out_tokens ) ;
+  }
+  public void testMix() throws IOException
+  {
+    String str = "あいうえおabcかきくけこ";
+       
+    Token[] out_tokens = { 
+        newToken("あい", 0, 2, "double"), 
+        newToken("いう", 1, 3, "double"),
+        newToken("うえ", 2, 4, "double"),
+        newToken("えお", 3, 5, "double"), 
+        newToken("abc", 5, 8, "single"), 
+        newToken("かき", 8, 10, "double"),
+        newToken("きく", 9, 11, "double"),
+        newToken("くけ", 10,12, "double"),
+        newToken("けこ", 11,13, "double")
+    };
+    checkCJKToken( str, out_tokens ) ;
+  }  
+  public void testMix2() throws IOException
+  {
+    String str = "あいうえおabんcかきくけ こ";
+       
+    Token[] out_tokens = { 
+        newToken("あい", 0, 2, "double"), 
+        newToken("いう", 1, 3, "double"),
+        newToken("うえ", 2, 4, "double"),
+        newToken("えお", 3, 5, "double"), 
+        newToken("ab", 5, 7, "single"), 
+        newToken("ん", 7, 8, "double"), 
+        newToken("c", 8, 9, "single"), 
+        newToken("かき", 9, 11, "double"),
+        newToken("きく", 10, 12, "double"),
+        newToken("くけ", 11,13, "double"),
+        newToken("こ", 14,15, "double")
+    };
+    checkCJKToken( str, out_tokens ) ;
+  }
+}
+