Index: modules/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestRBBITokenizer.java
===================================================================
--- modules/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestRBBITokenizer.java	(revision 0)
+++ modules/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestRBBITokenizer.java	(revision 0)
@@ -0,0 +1,215 @@
+package org.apache.lucene.analysis.icu.segmentation;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.ReusableAnalyzerBase;
+import org.apache.lucene.analysis.Tokenizer;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.Arrays;
+
+public class TestRBBITokenizer extends BaseTokenStreamTestCase {
+  
+  public void testHugeDoc() throws IOException {
+    StringBuilder sb = new StringBuilder();
+    char whitespace[] = new char[4094];
+    Arrays.fill(whitespace, ' ');
+    sb.append(whitespace);
+    sb.append("testing 1234");
+    String input = sb.toString();
+    RBBITokenizer tokenizer = new RBBITokenizer(new StringReader(input));
+    assertTokenStreamContents(tokenizer, new String[] { "testing", "1234" });
+  }
+  
+  public void testHugeTerm2() throws IOException {
+    StringBuilder sb = new StringBuilder();
+    for (int i = 0; i < 40960; i++) {
+      sb.append('a');
+    }
+    String input = sb.toString();
+    RBBITokenizer tokenizer = new RBBITokenizer(new StringReader(input));
+    char token[] = new char[4096];
+    Arrays.fill(token, 'a');
+    String expectedToken = new String(token);
+    String expected[] = { 
+        expectedToken, expectedToken, expectedToken, 
+        expectedToken, expectedToken, expectedToken,
+        expectedToken, expectedToken, expectedToken,
+        expectedToken
+    };
+    assertTokenStreamContents(tokenizer, expected);
+  }
+  
+  private Analyzer a = new ReusableAnalyzerBase() {
+    @Override
+    protected TokenStreamComponents createComponents(String fieldName,
+        Reader reader) {
+      Tokenizer tokenizer = new RBBITokenizer(reader);
+      return new TokenStreamComponents(tokenizer);
+    }
+  };
+  public void testArmenian() throws Exception {
+    assertAnalyzesTo(a, "Վիքիպեդիայի 13 միլիոն հոդվածները (4,600` հայերեն վիքիպեդիայում) գրվել են կամավորների կողմից ու համարյա բոլոր հոդվածները կարող է խմբագրել ցանկաց մարդ ով կարող է բացել Վիքիպեդիայի կայքը։",
+        new String[] { "Վիքիպեդիայի", "13", "միլիոն", "հոդվածները", "4,600", "հայերեն", "վիքիպեդիայում", "գրվել", "են", "կամավորների", "կողմից", 
+        "ու", "համարյա", "բոլոր", "հոդվածները", "կարող", "է", "խմբագրել", "ցանկաց", "մարդ", "ով", "կարող", "է", "բացել", "Վիքիպեդիայի", "կայքը" } );
+  }
+  
+  public void testAmharic() throws Exception {
+    assertAnalyzesTo(a, "ዊኪፔድያ የባለ ብዙ ቋንቋ የተሟላ ትክክለኛና ነጻ መዝገበ ዕውቀት (ኢንሳይክሎፒዲያ) ነው። ማንኛውም",
+        new String[] { "ዊኪፔድያ", "የባለ", "ብዙ", "ቋንቋ", "የተሟላ", "ትክክለኛና", "ነጻ", "መዝገበ", "ዕውቀት", "ኢንሳይክሎፒዲያ", "ነው", "ማንኛውም" } );
+  }
+  
+  public void testArabic() throws Exception {
+    assertAnalyzesTo(a, "الفيلم الوثائقي الأول عن ويكيبيديا يسمى \"الحقيقة بالأرقام: قصة ويكيبيديا\" (بالإنجليزية: Truth in Numbers: The Wikipedia Story)، سيتم إطلاقه في 2008.",
+        new String[] { "الفيلم", "الوثائقي", "الأول", "عن", "ويكيبيديا", "يسمى", "الحقيقة", "بالأرقام", "قصة", "ويكيبيديا",
+        "بالإنجليزية", "Truth", "in", "Numbers", "The", "Wikipedia", "Story", "سيتم", "إطلاقه", "في", "2008" } ); 
+  }
+  
+  public void testAramaic() throws Exception {
+    assertAnalyzesTo(a, "ܘܝܩܝܦܕܝܐ (ܐܢܓܠܝܐ: Wikipedia) ܗܘ ܐܝܢܣܩܠܘܦܕܝܐ ܚܐܪܬܐ ܕܐܢܛܪܢܛ ܒܠܫܢ̈ܐ ܣܓܝܐ̈ܐ܂ ܫܡܗ ܐܬܐ ܡܢ ܡ̈ܠܬܐ ܕ\"ܘܝܩܝ\" ܘ\"ܐܝܢܣܩܠܘܦܕܝܐ\"܀",
+        new String[] { "ܘܝܩܝܦܕܝܐ", "ܐܢܓܠܝܐ", "Wikipedia", "ܗܘ", "ܐܝܢܣܩܠܘܦܕܝܐ", "ܚܐܪܬܐ", "ܕܐܢܛܪܢܛ", "ܒܠܫܢ̈ܐ", "ܣܓܝܐ̈ܐ", "ܫܡܗ",
+        "ܐܬܐ", "ܡܢ", "ܡ̈ܠܬܐ", "ܕ", "ܘܝܩܝ", "ܘ", "ܐܝܢܣܩܠܘܦܕܝܐ"});
+  }
+  
+  public void testBengali() throws Exception {
+    assertAnalyzesTo(a, "এই বিশ্বকোষ পরিচালনা করে উইকিমিডিয়া ফাউন্ডেশন (একটি অলাভজনক সংস্থা)। উইকিপিডিয়ার শুরু ১৫ জানুয়ারি, ২০০১ সালে। এখন পর্যন্ত ২০০টিরও বেশী ভাষায় উইকিপিডিয়া রয়েছে।",
+        new String[] { "এই", "বিশ্বকোষ", "পরিচালনা", "করে", "উইকিমিডিয়া", "ফাউন্ডেশন", "একটি", "অলাভজনক", "সংস্থা", "উইকিপিডিয়ার",
+        "শুরু", "১৫", "জানুয়ারি", "২০০১", "সালে", "এখন", "পর্যন্ত", "২০০টিরও", "বেশী", "ভাষায়", "উইকিপিডিয়া", "রয়েছে" });
+  }
+  
+  public void testFarsi() throws Exception {
+    assertAnalyzesTo(a, "ویکی پدیای انگلیسی در تاریخ ۲۵ دی ۱۳۷۹ به صورت مکملی برای دانشنامهٔ تخصصی نوپدیا نوشته شد.",
+        new String[] { "ویکی", "پدیای", "انگلیسی", "در", "تاریخ", "۲۵", "دی", "۱۳۷۹", "به", "صورت", "مکملی",
+        "برای", "دانشنامهٔ", "تخصصی", "نوپدیا", "نوشته", "شد" });
+  }
+  
+  public void testGreek() throws Exception {
+    assertAnalyzesTo(a, "Γράφεται σε συνεργασία από εθελοντές με το λογισμικό wiki, κάτι που σημαίνει ότι άρθρα μπορεί να προστεθούν ή να αλλάξουν από τον καθένα.",
+        new String[] { "Γράφεται", "σε", "συνεργασία", "από", "εθελοντές", "με", "το", "λογισμικό", "wiki", "κάτι", "που",
+        "σημαίνει", "ότι", "άρθρα", "μπορεί", "να", "προστεθούν", "ή", "να", "αλλάξουν", "από", "τον", "καθένα" });
+  }
+
+  public void testThai() throws Exception {
+    assertAnalyzesTo(a, "การที่ได้ต้องแสดงว่างานดี. แล้วเธอจะไปไหน? ๑๒๓๔",
+        new String[] { "การที่ได้ต้องแสดงว่างานดี", "แล้วเธอจะไปไหน", "๑๒๓๔" });
+  }
+  
+  public void testLao() throws Exception {
+    assertAnalyzesTo(a, "ສາທາລະນະລັດ ປະຊາທິປະໄຕ ປະຊາຊົນລາວ", 
+        new String[] { "ສາທາລະນະລັດ", "ປະຊາທິປະໄຕ", "ປະຊາຊົນລາວ" });
+  }
+  
+  public void testTibetan() throws Exception {
+    assertAnalyzesTo(a, "སྣོན་མཛོད་དང་ལས་འདིས་བོད་ཡིག་མི་ཉམས་གོང་འཕེལ་དུ་གཏོང་བར་ཧ་ཅང་དགེ་མཚན་མཆིས་སོ། །",
+                     new String[] { "སྣོན", "མཛོད", "དང", "ལས", "འདིས", "བོད", "ཡིག", 
+                                    "མི", "ཉམས", "གོང", "འཕེལ", "དུ", "གཏོང", "བར", 
+                                    "ཧ", "ཅང", "དགེ", "མཚན", "མཆིས", "སོ" });
+  }
+  
+  /*
+   * For chinese, tokenize as char (these can later form bigrams or whatever)
+   */
+  public void testChinese() throws Exception {
+    assertAnalyzesTo(a, "我是中国人。 １２３４ Ｔｅｓｔｓ ",
+        new String[] { "我", "是", "中", "国", "人", "Ｔｅｓｔｓ"});
+  }
+  
+  public void testEmpty() throws Exception {
+    assertAnalyzesTo(a, "", new String[] {});
+    assertAnalyzesTo(a, ".", new String[] {});
+    assertAnalyzesTo(a, " ", new String[] {});
+  }
+  
+  /* test various jira issues this analyzer is related to */
+  
+  public void testLUCENE1545() throws Exception {
+    /*
+     * Standard analyzer does not correctly tokenize combining character U+0364 COMBINING LATIN SMALL LETTRE E.
+     * The word "moͤchte" is incorrectly tokenized into "mo" "chte", the combining character is lost.
+     * Expected result is only on token "moͤchte".
+     */
+    assertAnalyzesTo(a, "moͤchte", new String[] { "moͤchte" }); 
+  }
+  
+  /* Tests from StandardAnalyzer, just to show behavior is similar */
+  public void testAlphanumericSA() throws Exception {
+    // alphanumeric tokens
+    assertAnalyzesTo(a, "B2B", new String[]{"B2B"});
+    assertAnalyzesTo(a, "2B", new String[]{"2B"});
+  }
+
+  public void testDelimitersSA() throws Exception {
+    // other delimiters: "-", "/", ","
+    assertAnalyzesTo(a, "some-dashed-phrase", new String[]{"some", "dashed", "phrase"});
+    assertAnalyzesTo(a, "dogs,chase,cats", new String[]{"dogs", "chase", "cats"});
+    assertAnalyzesTo(a, "ac/dc", new String[]{"ac", "dc"});
+  }
+
+  public void testApostrophesSA() throws Exception {
+    // internal apostrophes: O'Reilly, you're, O'Reilly's
+    assertAnalyzesTo(a, "O'Reilly", new String[]{"O'Reilly"});
+    assertAnalyzesTo(a, "you're", new String[]{"you're"});
+    assertAnalyzesTo(a, "she's", new String[]{"she's"});
+    assertAnalyzesTo(a, "Jim's", new String[]{"Jim's"});
+    assertAnalyzesTo(a, "don't", new String[]{"don't"});
+    assertAnalyzesTo(a, "O'Reilly's", new String[]{"O'Reilly's"});
+  }
+
+  public void testNumericSA() throws Exception {
+    // floating point, serial, model numbers, ip addresses, etc.
+    assertAnalyzesTo(a, "21.35", new String[]{"21.35"});
+    assertAnalyzesTo(a, "R2D2 C3PO", new String[]{"R2D2", "C3PO"});
+    assertAnalyzesTo(a, "216.239.63.104", new String[]{"216.239.63.104"});
+    assertAnalyzesTo(a, "216.239.63.104", new String[]{"216.239.63.104"});
+  }
+
+  public void testTextWithNumbersSA() throws Exception {
+    // numbers
+    assertAnalyzesTo(a, "David has 5000 bones", new String[]{"David", "has", "5000", "bones"});
+  }
+
+  public void testVariousTextSA() throws Exception {
+    // various
+    assertAnalyzesTo(a, "C embedded developers wanted", new String[]{"C", "embedded", "developers", "wanted"});
+    assertAnalyzesTo(a, "foo bar FOO BAR", new String[]{"foo", "bar", "FOO", "BAR"});
+    assertAnalyzesTo(a, "foo      bar .  FOO <> BAR", new String[]{"foo", "bar", "FOO", "BAR"});
+    assertAnalyzesTo(a, "\"QUOTED\" word", new String[]{"QUOTED", "word"});
+  }
+
+  public void testKoreanSA() throws Exception {
+    // Korean words
+    assertAnalyzesTo(a, "안녕하세요 한글입니다", new String[]{"안녕하세요", "한글입니다"});
+  }
+  
+  public void testOffsets() throws Exception {
+    assertAnalyzesTo(a, "David has 5000 bones", 
+        new String[] {"David", "has", "5000", "bones"},
+        new int[] {0, 6, 10, 15},
+        new int[] {5, 9, 14, 20});
+  }
+  
+  public void testTypes() throws Exception {
+    assertAnalyzesTo(a, "David has 5000 bones", 
+        new String[] {"David", "has", "5000", "bones"},
+        new String[] { "<WORD>", "<WORD>", "<NUM>", "<WORD>" });
+  }
+}
Index: modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/RBBITokenizerOnlyAnalyzer.java
===================================================================
--- modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/RBBITokenizerOnlyAnalyzer.java	(revision 0)
+++ modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/RBBITokenizerOnlyAnalyzer.java	(revision 0)
@@ -0,0 +1,36 @@
+package org.apache.lucene.analysis.icu.segmentation;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+
+import java.io.IOException;
+import java.io.Reader;
+
+public final class RBBITokenizerOnlyAnalyzer extends ReusableAnalyzerBase {
+  @Override
+  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
+    final RBBITokenizer tokenizer = new RBBITokenizer(reader);
+    return new TokenStreamComponents(tokenizer) {
+      @Override
+      protected boolean reset(final Reader reader) throws IOException {
+        return super.reset(reader);
+      }
+    };
+  }
+}
Index: modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerOnlyAnalyzer.java
===================================================================
--- modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerOnlyAnalyzer.java	(revision 0)
+++ modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerOnlyAnalyzer.java	(revision 0)
@@ -0,0 +1,36 @@
+package org.apache.lucene.analysis.icu.segmentation;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+
+import java.io.IOException;
+import java.io.Reader;
+
+public final class ICUTokenizerOnlyAnalyzer extends ReusableAnalyzerBase {
+  @Override
+  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
+    final ICUTokenizer tokenizer = new ICUTokenizer(reader);
+    return new TokenStreamComponents(tokenizer) {
+      @Override
+      protected boolean reset(final Reader reader) throws IOException {
+        return super.reset(reader);
+      }
+    };
+  }
+}
Index: modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/RBBITokenizer.java
===================================================================
--- modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/RBBITokenizer.java	(revision 0)
+++ modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/RBBITokenizer.java	(revision 0)
@@ -0,0 +1,180 @@
+package org.apache.lucene.analysis.icu.segmentation;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import com.ibm.icu.lang.UCharacter;
+import com.ibm.icu.text.BreakIterator;
+import com.ibm.icu.text.RuleBasedBreakIterator;
+import com.ibm.icu.util.ULocale;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+
+import java.io.IOException;
+import java.io.Reader;
+
+/**
+ * Breaks text into words according to UAX #29: Unicode Text Segmentation
+ * (http://www.unicode.org/reports/tr29/)
+ */
+public final class RBBITokenizer extends Tokenizer {
+  private static final int IOBUFFER = 4096;
+  private CharArrayIterator charArrayIterator = new CharArrayIterator();
+  private RuleBasedBreakIterator rbbi 
+    = (RuleBasedBreakIterator)BreakIterator.getWordInstance(ULocale.ROOT);
+  private final char buffer[] = new char[IOBUFFER];
+  /** true length of text in the buffer */
+  private int length = 0; 
+  /** length in buffer that can be evaluated safely, up to a safe end point */
+  private int usableLength = 0; 
+  /** accumulated offset of previous buffers for this reader, for offsetAtt */
+  private int offset = 0; 
+
+  private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+  private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+  private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+
+  /** Token type for words containing ideographic characters */
+  public static final String WORD_IDEO = "<IDEO>";
+  /** Token type for words containing Japanese kana */
+  public static final String WORD_KANA = "<KANA>";
+  /** Token type for words that contain letters */
+  public static final String WORD_LETTER = "<WORD>";
+  /** Token type for words that appear to be numbers */
+  public static final String WORD_NUMBER = "<NUM>";
+
+  public RBBITokenizer(Reader input) {
+    super(input);
+    rbbi.setText(charArrayIterator);
+  }
+
+  @Override
+  public boolean incrementToken() throws IOException {
+    clearAttributes();
+    if (length == 0)
+      refill();
+    while (!incrementTokenBuffer()) {
+      refill();
+      if (length <= 0) // no more bytes to read;
+        return false;
+    }
+    return true;
+  }
+  
+  @Override
+  public void reset() throws IOException {
+    super.reset();
+    length = usableLength = offset = 0;
+  }
+
+  @Override
+  public void reset(Reader input) throws IOException {
+    super.reset(input);
+    reset();
+  }
+  
+  @Override
+  public void end() throws IOException {
+    final int finalOffset = (length < 0) ? offset : offset + length;
+    offsetAtt.setOffset(finalOffset, finalOffset);
+  }  
+
+  /*
+   * This tokenizes text based upon the longest matching rule, and because of 
+   * this, isn't friendly to a Reader.
+   * 
+   * Text is read from the input stream in 4kB chunks. Within a 4kB chunk of
+   * text, the last unambiguous break point is found (in this implementation:
+   * white space character) Any remaining characters represent possible partial
+   * words, so are appended to the front of the next chunk.
+   * 
+   * There is the possibility that there are no unambiguous break points within
+   * an entire 4kB chunk of text (binary data). So there is a maximum word limit
+   * of 4kB since it will not try to grow the buffer in this case.
+   */
+
+  /**
+   * Returns the last unambiguous break position in the text.
+   * 
+   * @return position of character, or -1 if one does not exist
+   */
+  private int findSafeEnd() {
+    for (int i = length - 1; i >= 0; i--)
+      if (UCharacter.isWhitespace(buffer[i]))
+        return i + 1;
+    return -1;
+  }
+
+  // Refill the buffer, accumulating the offset and setting usableLength to the
+  // last unambiguous break position
+  private void refill() throws IOException {
+    offset += usableLength;
+    int leftover = length - usableLength;
+    System.arraycopy(buffer, usableLength, buffer, 0, leftover);
+    int requested = buffer.length - leftover;
+    int returned = input.read(buffer, leftover, requested);
+    length = returned < 0 ? leftover : returned + leftover;
+    if (returned < requested) /* reader has been emptied, process the rest */
+      usableLength = length;
+    else { /* still more data to be read, find a safe-stopping place */
+      usableLength = findSafeEnd();
+      if (usableLength < 0)
+        usableLength = length; /*
+                                * more than IOBUFFER of text without space,
+                                * gonna possibly truncate tokens
+                                */
+    }
+    charArrayIterator.setText(buffer, 0, Math.max(0, usableLength));
+  }
+
+  /*
+   * return true if there is a token from the buffer, or null if it is
+   * exhausted.
+   */
+  private boolean incrementTokenBuffer() {
+    int start = rbbi.current();
+    if (start == BreakIterator.DONE)
+      return false; // BreakIterator exhausted
+
+    // find the next set of boundaries, skipping over non-tokens (rule status 0)
+    int end = rbbi.next();
+    while (start != BreakIterator.DONE && rbbi.getRuleStatus() == 0) {
+      start = end;
+      end = rbbi.next();
+    }
+
+    if (start == BreakIterator.DONE)
+      return false; // BreakIterator exhausted
+
+    termAtt.copyBuffer(buffer, start, end - start);
+    offsetAtt.setOffset(correctOffset(offset + start), correctOffset(offset + end));
+    typeAtt.setType(ruleStatusToTokenType(rbbi.getRuleStatus()));
+    return true;
+  }
+  
+  private String ruleStatusToTokenType(int ruleStatus) {
+    switch (ruleStatus) {
+        case RuleBasedBreakIterator.WORD_IDEO: return WORD_IDEO;
+        case RuleBasedBreakIterator.WORD_KANA: return WORD_KANA;
+      case RuleBasedBreakIterator.WORD_LETTER: return WORD_LETTER;
+      case RuleBasedBreakIterator.WORD_NUMBER: return WORD_NUMBER;
+                                      default: return "<OTHER>";
+    }
+  }
+}
Index: modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerOnlyAnalyzer.java
===================================================================
--- modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerOnlyAnalyzer.java	(revision 0)
+++ modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerOnlyAnalyzer.java	(revision 0)
@@ -0,0 +1,41 @@
+package org.apache.lucene.analysis.standard;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+import org.apache.lucene.util.Version;
+
+import java.io.IOException;
+import java.io.Reader;
+
+public final class StandardTokenizerOnlyAnalyzer extends ReusableAnalyzerBase {
+  public static final int MAX_TOKEN_LENGTH = 255;
+
+  @Override
+  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
+    final StandardTokenizer src = new StandardTokenizer(Version.LUCENE_40, reader);
+    src.setMaxTokenLength(MAX_TOKEN_LENGTH);
+    return new TokenStreamComponents(src) {
+      @Override
+      protected boolean reset(final Reader reader) throws IOException {
+        src.setMaxTokenLength(MAX_TOKEN_LENGTH);
+        return super.reset(reader);
+      }
+    };
+  }
+}
Index: modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29TokenizerOnlyAnalyzer.java
===================================================================
--- modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29TokenizerOnlyAnalyzer.java	(revision 0)
+++ modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29TokenizerOnlyAnalyzer.java	(revision 0)
@@ -0,0 +1,39 @@
+package org.apache.lucene.analysis.standard;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+
+import java.io.IOException;
+import java.io.Reader;
+
+public final class UAX29TokenizerOnlyAnalyzer extends ReusableAnalyzerBase {
+  public static final int MAX_TOKEN_LENGTH = 255;
+
+  @Override
+  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
+    final UAX29Tokenizer tokenizer = new UAX29Tokenizer(reader);
+    tokenizer.setMaxTokenLength(MAX_TOKEN_LENGTH);
+    return new TokenStreamComponents(tokenizer) {
+      @Override
+      protected boolean reset(final Reader reader) throws IOException {
+        return super.reset(reader);
+      }
+    };
+  }
+}
Index: modules/analysis/common/src/java/org/apache/lucene/analysis/standard/NewStandardTokenizerOnlyAnalyzer.java
===================================================================
--- modules/analysis/common/src/java/org/apache/lucene/analysis/standard/NewStandardTokenizerOnlyAnalyzer.java	(revision 0)
+++ modules/analysis/common/src/java/org/apache/lucene/analysis/standard/NewStandardTokenizerOnlyAnalyzer.java	(revision 0)
@@ -0,0 +1,39 @@
+package org.apache.lucene.analysis.standard;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.util.ReusableAnalyzerBase;
+
+import java.io.IOException;
+import java.io.Reader;
+
+public final class NewStandardTokenizerOnlyAnalyzer extends ReusableAnalyzerBase {
+  public static final int MAX_TOKEN_LENGTH = 255;
+
+  @Override
+  protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
+    final NewStandardTokenizer tokenizer = new NewStandardTokenizer(reader);
+    tokenizer.setMaxTokenLength(MAX_TOKEN_LENGTH);
+    return new TokenStreamComponents(tokenizer) {
+      @Override
+      protected boolean reset(final Reader reader) throws IOException {
+        return super.reset(reader);
+      }
+    };
+  }
+}
Index: lucene/contrib/benchmark/conf/uax29-tokenizers.alg
===================================================================
--- lucene/contrib/benchmark/conf/uax29-tokenizers.alg	(revision 0)
+++ lucene/contrib/benchmark/conf/uax29-tokenizers.alg	(revision 0)
@@ -0,0 +1,52 @@
+#/**
+# * Licensed to the Apache Software Foundation (ASF) under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The ASF licenses this file to You under the Apache License, Version 2.0
+# * (the "License"); you may not use this file except in compliance with
+# * the License.  You may obtain a copy of the License at
+# *
+# *     http://www.apache.org/licenses/LICENSE-2.0
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# */
+# -------------------------------------------------------------------------------------
+# multi val params are iterated by NewRound's, added to reports, start with column name.
+
+content.source=org.apache.lucene.benchmark.byTask.feeds.ReutersContentSource
+doc.tokenized=false
+doc.body.tokenized=true
+docs.dir=reuters-out
+log.step=1000
+
+{ "Rounds"
+
+    -NewAnalyzer(standard.StandardTokenizerOnlyAnalyzer)
+    -ResetInputs
+    { "StandardTokenizer" { ReadTokens > : 10000 }
+
+    -NewAnalyzer(standard.UAX29TokenizerOnlyAnalyzer)
+    -ResetInputs
+    { "UAX29Tokenizer" { ReadTokens > : 10000 }
+
+    -NewAnalyzer(standard.NewStandardTokenizerOnlyAnalyzer)
+    -ResetInputs
+    { "NewStandardTokenizer" { ReadTokens > : 10000 }
+
+    -NewAnalyzer(org.apache.lucene.analysis.icu.segmentation.ICUTokenizerOnlyAnalyzer)
+    -ResetInputs
+    { "ICUTokenizer" { ReadTokens > : 10000 }
+    NewRound
+
+    -NewAnalyzer(org.apache.lucene.analysis.icu.segmentation.RBBITokenizerOnlyAnalyzer)
+    -ResetInputs
+    { "RBBITokenizer" { ReadTokens > : 10000 }
+    NewRound
+
+} : 5
+
+RepSumByNameRound
Index: lucene/contrib/benchmark/scripts/compare.uax29.analyzers.sh
===================================================================
--- lucene/contrib/benchmark/scripts/compare.uax29.analyzers.sh	(revision 0)
+++ lucene/contrib/benchmark/scripts/compare.uax29.analyzers.sh	(revision 0)
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+LUCENE_WORKSPACE=C:/Users/us/IdeaProjects/lucene/dev/trunk
+ICU4J_JAR=$LUCENE_WORKSPACE/modules/analysis/icu/lib/icu4j-4_4.jar
+COMMON_ANALYSIS_JAR=$LUCENE_WORKSPACE/modules/analysis/build/common/lucene-analyzers-common-4.0-dev.jar
+ICU_ANALYSIS_JAR=$LUCENE_WORKSPACE/modules/analysis/build/icu/lucene-analyzers-icu-4.0-dev.jar
+ant run-task -Dtask.alg=conf/uax29-tokenizers.alg -Dtask.mem=512M \
+    -Dbenchmark.ext.classpath="$ICU4J_JAR;$COMMON_ANALYSIS_JAR;$ICU_ANALYSIS_JAR" | perl -ne \
+'if (/^\s+\[java\]\s+([^-\s]+Tokenizer)[- ]+\d+[- ]+\d+[- ]+(\d+)[- ]+([,.\d]+)[- ]+([.,\d]+)/) {
+  $name=$1;$recs=$2;$rate=$3;$time=$4; 
+  $c{$name} = { rate => [ $rate, @{$c{$name}{rate}} ],
+                recs => $recs, 
+                time => [ $time, @{$c{$name}{time}} ] }; 
+} END { 
+  for $name (sort keys %c) { 
+    print "|$name|$c{$name}{recs}|",
+          (sort { $b <=> $a } @{$c{$name}{rate}})[0], "|",
+          (sort { $a <=> $b } @{$c{$name}{time}})[0], "|\n"; 
+  }
+}'