Index: lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseNumberFilter.java
===================================================================
--- lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseNumberFilter.java	(revision 0)
+++ lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseNumberFilter.java	(working copy)
@@ -0,0 +1,516 @@
+package org.apache.lucene.analysis.ja;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.math.BigDecimal;
+
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+
+/**
+ * Normalizes Japanese numbers
+ */
+public class JapaneseNumberFilter extends TokenFilter {
+
+  private final CharTermAttribute termAttr = addAttribute(CharTermAttribute.class);
+  private final OffsetAttribute offsetAttr = addAttribute(OffsetAttribute.class);
+  private final KeywordAttribute keywordAttr = addAttribute(KeywordAttribute.class);
+
+  private static char NO_NUMERAL = Character.MAX_VALUE;
+
+  private static char[] numerals;
+
+  private static char[] exponents;
+
+  private StringBuilder numeral;
+
+  private State state;
+
+  private int startOffset;
+
+  private int endOffset;
+
+  static {
+    numerals = new char[0x10000];
+    for (int i = 0; i < numerals.length; i++) {
+      numerals[i] = NO_NUMERAL;
+    }
+    numerals['〇'] = 0; // 〇 U+3007 0
+    numerals['一'] = 1; // 一 U+4E00 1
+    numerals['二'] = 2; // 二 U+4E8C 2
+    numerals['三'] = 3; // 三 U+4E09 3
+    numerals['四'] = 4; // 四 U+56DB 4
+    numerals['五'] = 5; // 五 U+4E94 5
+    numerals['六'] = 6; // 六 U+516D 6
+    numerals['七'] = 7; // 七 U+4E03 7
+    numerals['八'] = 8; // 八 U+516B 8
+    numerals['九'] = 9; // 九 U+4E5D 9
+
+    exponents = new char[0x10000];
+    for (int i = 0; i < exponents.length; i++) {
+      exponents[i] = 0;
+    }
+    exponents['十'] = 1;  // 十 U+5341 10
+    exponents['百'] = 2;  // 百 U+767E 100
+    exponents['千'] = 3;  // 千 U+5343 1,000
+    exponents['万'] = 4;  // 万 U+4E07 10,000
+    exponents['億'] = 8;  // 億 U+5104 100,000,000
+    exponents['兆'] = 12; // 兆 U+5146 1,000,000,000,000
+    exponents['京'] = 16; // 京 U+4EAC 10,000,000,000,000,000
+    exponents['垓'] = 20; // 垓 U+5793 100,000,000,000,000,000,000
+  }
+
+  public JapaneseNumberFilter(TokenStream input) {
+    super(input);
+  }
+
+  @Override
+  public final boolean incrementToken() throws IOException {
+
+    // Emit previously captured token we read past earlier
+    if (state != null) {
+      restoreState(state);
+      state = null;
+      return true;
+    }
+
+    if (!input.incrementToken()) {
+      return false;
+    }
+
+    if (keywordAttr.isKeyword()) {
+      return true;
+    }
+
+    boolean moreTokens = true;
+    boolean composedToken = false;
+    String term = termAttr.toString();
+
+    while (moreTokens && isNumeral(term)) {
+      if (!composedToken) {
+        startOffset = offsetAttr.startOffset();
+      }
+
+      composedToken = true;
+      endOffset = offsetAttr.endOffset();
+      numeral.append(term);
+
+      moreTokens = input.incrementToken();
+      
+      if (moreTokens) {
+        term = termAttr.toString();
+      }
+    }
+
+    if (composedToken) {
+
+      if (moreTokens) {
+        // We have read past all numerals and there are still tokens left, so
+        // capture the state of this token and emit it on our next incrementToken()
+        state = captureState();
+      }
+
+      String normalizedNumber = normalizeNumber(numeral.toString());
+
+      termAttr.setEmpty();
+      termAttr.append(normalizedNumber);
+      offsetAttr.setOffset(startOffset, endOffset);
+
+      numeral = new StringBuilder();
+      return true;
+    }
+
+    return moreTokens;
+  }
+
+  @Override
+  public void reset() throws IOException {
+    super.reset();
+    startOffset = 0;
+    endOffset = 0;
+    state = null;
+    numeral = new StringBuilder();
+  }
+
+  /**
+   * Normalizes a Japanese number
+   *
+   * @param number number or normalize
+   * @return normalized number, or number to normalize on error (no op)
+   */
+  public String normalizeNumber(String number) {
+    try {
+      BigDecimal normalizedNumber = parseNumber(new NumberBuffer(number));
+      if (normalizedNumber == null) {
+        return number;
+      }
+      return normalizedNumber.toBigIntegerExact().toString();
+
+    } catch (NumberFormatException | ArithmeticException e) {
+      // Return the source number in case of error, i.e. malformed input
+      return number;
+    }
+  }
+
+  /**
+   * Parses a Japanese number
+   *
+   * @param buffer buffer to parse
+   * @return parsed number, or null on error or end of input
+   */
+  private BigDecimal parseNumber(NumberBuffer buffer) {
+    BigDecimal sum = BigDecimal.ZERO;
+    BigDecimal result = parseLargePair(buffer);
+
+    if (result == null) {
+      return null;
+    }
+
+    while (result != null) {
+      sum = sum.add(result);
+      result = parseLargePair(buffer);
+    }
+
+    return sum;
+  }
+
+  /**
+   * Parses a pair of large numbers, i.e. large kanji factor is 10,000（万）or larger
+   *
+   * @param buffer buffer to parse
+   * @return parsed pair, or null on error or end of input
+   */
+  private BigDecimal parseLargePair(NumberBuffer buffer) {
+    BigDecimal first = parseMediumNumber(buffer);
+    BigDecimal second = parseLargeKanjiNumeral(buffer);
+
+    if (first == null && second == null) {
+      return null;
+    }
+
+    if (second == null) {
+      // If there's no second factor, we return the first one
+      // This can happen if we our number is smaller than 10,000 (万)
+      return first;
+    }
+
+    if (first == null) {
+      // If there's no first factor, just return the second one,
+      // which is the same as multiplying by 1, i.e. with 万
+      return second;
+    }
+
+    return first.multiply(second);
+  }
+
+  /**
+   * Parses a "medium sized" number, typically less than 10,000（万）, but might be larger
+   * due to a larger factor from {link parseBasicNumber}.
+   *
+   * @param buffer buffer to parse
+   * @return parsed number, or null on error or end of input
+   */
+  private BigDecimal parseMediumNumber(NumberBuffer buffer) {
+    BigDecimal sum = BigDecimal.ZERO;
+    BigDecimal result = parseMediumPair(buffer);
+
+    if (result == null) {
+      return null;
+    }
+
+    while (result != null) {
+      sum = sum.add(result);
+      result = parseMediumPair(buffer);
+    }
+
+    return sum;
+  }
+
+  /**
+   * Parses a pair of "medium sized" numbers, i.e. large kanji factor is at most 1,000（千）
+   *
+   * @param buffer buffer to parse
+   * @return parsed pair, or null on error or end of input
+   */
+  private BigDecimal parseMediumPair(NumberBuffer buffer) {
+
+    BigDecimal first = parseBasicNumber(buffer);
+    BigDecimal second = parseMediumKanjiNumeral(buffer);
+
+    if (first == null && second == null) {
+      return null;
+    }
+
+    if (second == null) {
+      // If there's no second factor, we return the first one
+      // This can happen if we just have a plain number such as 五
+      return first;
+    }
+
+    if (first == null) {
+      // If there's no first factor, just return the second one,
+      // which is the same as multiplying by 1, i.e. with 千
+      return second;
+    }
+
+    // Return factors multiplied
+    return first.multiply(second);
+  }
+
+  /**
+   * Parse a basic number, which is a sequence of Arabic numbers or a sequence or 0-9 kanji numerals (〇 to 九).
+   *
+   * @param buffer buffer to parse
+   * @return parsed number, or null on error or end of input
+   */
+  private BigDecimal parseBasicNumber(NumberBuffer buffer) {
+    StringBuilder builder = new StringBuilder();
+    int i = buffer.position();
+
+    while (i < buffer.length()) {
+      char c = buffer.charAt(i);
+
+      if (isArabicNumeral(c)) {
+        // Arabic numerals; 0 to 9 or ０ to ９ (full-width)
+        builder.append(arabicNumeralValue(c));
+      } else if (isKanjiNumeral(c)) {
+        // Kanji numerals; 〇, 一, 二, 三, 四, 五, 六, 七, 八, or 九
+        builder.append(kanjiNumeralValue(c));
+      } else if (isDecimalPoint(c)) {
+        builder.append(".");
+      } else if (isThousandSeparator(c)) {
+        // Just skip and move to the next character
+      } else {
+        // We don't have an Arabic nor kanji numeral, nor separation or punctuation, so we'll stop.
+        break;
+      }
+
+      i++;
+      buffer.advance();
+    }
+
+    if (builder.length() == 0) {
+      // We didn't build anything, so we don't have a number
+      return null;
+    }
+
+    return new BigDecimal(builder.toString());
+  }
+
+  /**
+   * Parse large kanji numerals (ten thousands or larger)
+   *
+   * @param buffer buffer to parse
+   * @return parsed number, or null on error or end of input
+   */
+  public BigDecimal parseLargeKanjiNumeral(NumberBuffer buffer) {
+    int i = buffer.position();
+
+    if (i >= buffer.length()) {
+      return null;
+    }
+
+    char c = buffer.charAt(i);
+    int power = exponents[c];
+
+    if (power > 3) {
+      buffer.advance();
+      return BigDecimal.TEN.pow(power);
+    }
+
+    return null;
+  }
+
+  /**
+   * Parse medium kanji numerals (tens, hundreds or thousands)
+   *
+   * @param buffer buffer to parse
+   * @return parsed number or null on error
+   */
+  public BigDecimal parseMediumKanjiNumeral(NumberBuffer buffer) {
+    int i = buffer.position();
+
+    if (i >= buffer.length()) {
+      return null;
+    }
+
+    char c = buffer.charAt(i);
+    int power = exponents[c];
+
+    if (1 <= power && power <= 3) {
+      buffer.advance();
+      return BigDecimal.TEN.pow(power);
+    }
+
+    return null;
+  }
+
+  /**
+   * Numeral predicate
+   *
+   * @param input string to test
+   * @return true if and only if input is a numeral
+   */
+  public boolean isNumeral(String input) {
+    for (int i = 0; i < input.length(); i++) {
+      if (!isNumeral(input.charAt(i))) {
+        return false;
+      }
+    }
+    return true;
+  }
+
+  /**
+   * Numeral predicate
+   *
+   * @param c character to test
+   * @return true if and only if c is a numeral
+   */
+  public boolean isNumeral(char c) {
+    return isArabicNumeral(c) || isKanjiNumeral(c) || exponents[c] > 0 || isDecimalPoint(c) || isThousandSeparator(c);
+  }
+
+  /**
+   * Arabic numeral predicate. Both half-width and full-width characters are supported
+   *
+   * @param c character to test
+   * @return true if and only if c is an Arabic numeral
+   */
+  public boolean isArabicNumeral(char c) {
+    return isHalfWidthArabicNumeral(c) || isFullWidthArabicNumeral(c);
+  }
+
+  /**
+   * Arabic half-width numeral predicate
+   *
+   * @param c character to test
+   * @return true if and only if c is a half-width Arabic numeral
+   */
+  private boolean isHalfWidthArabicNumeral(char c) {
+    // 0 U+0030 - 9 U+0039
+    return '0' <= c && c <= '9';
+  }
+
+  /**
+   * Arabic full-width numeral predicate
+   *
+   * @param c character to test
+   * @return true if and only if c is a full-width Arabic numeral
+   */
+  private boolean isFullWidthArabicNumeral(char c) {
+    // ０ U+FF10 - ９ U+FF19
+    return '０' <= c && c <= '９';
+  }
+
+  /**
+   * Returns the numeric value for the specified character Arabic numeral.
+   * Behavior is undefined if a non-Arabic numeral is provided
+   *
+   * @param c arabic numeral character
+   * @return numeral value
+   */
+  private int arabicNumeralValue(char c) {
+    int offset;
+    if (isHalfWidthArabicNumeral(c)) {
+      offset = '0';
+    } else {
+      offset = '０';
+    }
+    return c - offset;
+  }
+
+  /**
+   * Kanji numeral predicate that tests if the provided character is one of 〇, 一, 二, 三, 四, 五, 六, 七, 八, or 九.
+   * Larger number kanji gives a false value.
+   *
+   * @param c character to test
+   * @return true if and only is character is one of 〇, 一, 二, 三, 四, 五, 六, 七, 八, or 九 (0 to 9)
+   */
+  private boolean isKanjiNumeral(char c) {
+    return numerals[c] != NO_NUMERAL;
+  }
+
+  /**
+   * Returns the value for the provided kanji numeral. Only numeric values for the characters where
+   * {link isKanjiNumeral} return true are supported - behavior is undefined for other characters.
+   *
+   * @param c kanji numeral character
+   * @return numeral value
+   * @see #isKanjiNumeral(char)
+   */
+  private int kanjiNumeralValue(char c) {
+    return numerals[c];
+  }
+
+  /**
+   * Decimal point predicate
+   *
+   * @param c character to test
+   * @return true if and only if c is a decimal point
+   */
+  private boolean isDecimalPoint(char c) {
+    return c == '.'   // U+002E FULL STOP 
+        || c == '．'; // U+FF0E FULLWIDTH FULL STOP
+  }
+
+  /**
+   * Thousand separator predicate
+   *
+   * @param c character to test
+   * @return true if and only if c is a thousand separator predicate
+   */
+  private boolean isThousandSeparator(char c) {
+    return c == ','   // U+002C COMMA
+        || c == '，'; // U+FF0C FULLWIDTH COMMA
+  }
+
+  /**
+   * Buffer that holds a Japanese number string and a position index used as a parsed-to marker
+   */
+  public static class NumberBuffer {
+
+    private int position;
+
+    private String string;
+
+    public NumberBuffer(String string) {
+      this.string = string;
+      this.position = 0;
+    }
+
+    public char charAt(int index) {
+      return string.charAt(index);
+    }
+
+    public int length() {
+      return string.length();
+    }
+
+    public void advance() {
+      position++;
+    }
+
+    public int position() {
+      return position;
+    }
+  }
+}
\ No newline at end of file
Index: lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseNumberFilter.java
===================================================================
--- lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseNumberFilter.java	(revision 0)
+++ lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/TestJapaneseNumberFilter.java	(working copy)
@@ -0,0 +1,187 @@
+package org.apache.lucene.analysis.ja;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestJapaneseNumberFilter extends BaseTokenStreamTestCase {
+
+  private Analyzer analyzer = new Analyzer() {
+    @Override
+    protected TokenStreamComponents createComponents(String fieldName) {
+      Tokenizer tokenizer = new JapaneseTokenizer(newAttributeFactory(), null, false, JapaneseTokenizer.DEFAULT_MODE);
+      return new TokenStreamComponents(tokenizer, new JapaneseNumberFilter(tokenizer));
+    }
+  };
+
+  @Test
+  public void testBasics() throws IOException {
+
+    assertAnalyzesTo(analyzer, "本日十万二千五百円のワインを買った",
+        new String[]{"本日", "102500", "円", "の", "ワイン", "を", "買っ", "た"},
+        new int[]{0, 2, 8, 9, 10, 13, 14, 16},
+        new int[]{2, 8, 9, 10, 13, 14, 16, 17}
+    );
+
+    assertAnalyzesTo(analyzer, "昨日のお寿司は１０万円でした。",
+        new String[]{"昨日", "の", "お", "寿司", "は", "100000", "円", "でし", "た", "。"},
+        new int[]{0, 2, 3, 4, 6, 7, 10, 11, 13, 14},
+        new int[]{2, 3, 4, 6, 7, 10, 11, 13, 14, 15}
+    );
+
+    assertAnalyzesTo(analyzer, "アティリカの資本金は６００万円です",
+        new String[]{"アティリカ", "の", "資本", "金", "は", "6000000", "円", "です"},
+        new int[]{0, 5, 6, 8, 9, 10, 14, 15},
+        new int[]{5, 6, 8, 9, 10, 14, 15, 17}
+    );
+  }
+
+  @Test
+  public void testVariants() throws IOException {
+    // Test variants of three
+    assertAnalyzesTo(analyzer, "3", new String[]{"3"});
+    assertAnalyzesTo(analyzer, "３", new String[]{"3"});
+    assertAnalyzesTo(analyzer, "三", new String[]{"3"});
+
+    // Test three variations with trailing zero
+    assertAnalyzesTo(analyzer, "03", new String[]{"3"});
+    assertAnalyzesTo(analyzer, "０３", new String[]{"3"});
+    assertAnalyzesTo(analyzer, "〇三", new String[]{"3"});
+    assertAnalyzesTo(analyzer, "003", new String[]{"3"});
+    assertAnalyzesTo(analyzer, "００３", new String[]{"3"});
+    assertAnalyzesTo(analyzer, "〇〇三", new String[]{"3"});
+
+    // Test thousand variants
+    assertAnalyzesTo(analyzer, "千", new String[]{"1000"});
+    assertAnalyzesTo(analyzer, "1千", new String[]{"1000"});
+    assertAnalyzesTo(analyzer, "１千", new String[]{"1000"});
+    assertAnalyzesTo(analyzer, "一千", new String[]{"1000"});
+    assertAnalyzesTo(analyzer, "一〇〇〇", new String[]{"1000"});
+    assertAnalyzesTo(analyzer, "１０百", new String[]{"1000"}); // Strange, but supported
+  }
+
+  @Test
+  public void testLargeVariants() throws IOException {
+    // Test large numbers
+    assertAnalyzesTo(analyzer, "三五七八九", new String[]{"35789"});
+    assertAnalyzesTo(analyzer, "六百二万五千一", new String[]{"6025001"});
+    assertAnalyzesTo(analyzer, "兆六百万五千一", new String[]{"1000006005001"});
+    assertAnalyzesTo(analyzer, "十兆六百万五千一", new String[]{"10000006005001"});
+    assertAnalyzesTo(analyzer, "一京一", new String[]{"10000000000000001"});
+    assertAnalyzesTo(analyzer, "十京十", new String[]{"100000000000000010"});
+    assertAnalyzesTo(analyzer, "垓京兆億万千百十一", new String[]{"100010001000100011111"});
+  }
+
+  @Test
+  public void testMixed() throws IOException {
+    // Test mixed numbers
+    assertAnalyzesTo(analyzer, "三千2百２十三", new String[]{"3223"});
+    assertAnalyzesTo(analyzer, "３２二三", new String[]{"3223"});
+  }
+
+  @Test
+  public void testFunny() throws IOException {
+    // Test some oddities for inconsistent input
+    assertAnalyzesTo(analyzer, "十十", new String[]{"20"}); // 100?
+    assertAnalyzesTo(analyzer, "百百百", new String[]{"300"}); // 10,000?
+    assertAnalyzesTo(analyzer, "千千千千", new String[]{"4000"}); // 1,000,000,000,000?
+  }
+
+  @Test
+  public void testKanjiArabic() throws IOException {
+    // Test kanji numerals used as Arabic numbers (with head zero)
+    assertAnalyzesTo(analyzer, "〇一二三四五六七八九九八七六五四三二一〇",
+        new String[]{"1234567899876543210"}
+    );
+
+    // I'm Bond, James "normalized" Bond...
+    assertAnalyzesTo(analyzer, "〇〇七", new String[]{"7"});
+  }
+
+  @Test
+  public void testDecimalPunctuation() throws IOException {
+    // Test Arabic numbers with punctuation, i.e. 3.2 thousands
+    assertAnalyzesTo(analyzer, "３．２千円",
+        new String[]{"3200", "円"}
+    );
+  }
+
+  @Test
+  public void testThousandSeparator() throws IOException {
+    assertAnalyzesTo(analyzer, "4,647",
+        new String[]{"4647"}
+    );
+  }
+
+  @Test
+  public void testEmpty() throws IOException {
+    assertAnalyzesTo(analyzer, "", new String[]{});
+  }
+
+  @Test
+  public void testRandomStrings() throws Exception {
+    checkRandomData(random(), analyzer, 1000 * RANDOM_MULTIPLIER);
+  }
+
+  // nocommit
+  @Ignore("Fails using: ant test  -Dtestcase=TestJapaneseNumberFilter -Dtests.method=testRandomHugeStrings -Dtests.seed=C8544B8473D5EE13")
+  @Test
+  public void testRandomHugeStrings() throws Exception {
+    checkRandomData(random(), analyzer, 50 * RANDOM_MULTIPLIER, 8192);
+  }
+
+  // nocommit
+  @Ignore("Temporarily ignore - attempt to reproduce the above error")
+  @Test
+  public void testLargeRandomIssue() throws Exception {
+    String input = "\u3002\u3023\u3000\u302e\u3006\u3005 \u300b\u3019\u3007\u302e\u3029\u3039\u3023\u3033\u302b\u300c \u04b5 hkrld tzqfwfy clkaa wyvzrc mBTbdQEoE \u20ed\u20d2\u20f8\u20f7\u20f5\u20d8\u20ee w\u947b^\uc98b\uda5f\udc60*t\uda0e\udc0dK\u0000\u0e5a \u0738\u0721\u0716\u071a\u071f\u0738\u074d\u072c \ua920\ua907\ua91d\ua924\ua90c \ud834\udcab\ud834\udcd9 \u886a\u06c5 \u4dd6\u4dde\u4dd8\u4dd1 \u2d50\u2d77\u2d6e\u2d75\u2d54\u2d5e\u2d39\u2d3d bbytzpc lra bdjwzqju \ud9f2\udd86\uf622\ued2aA\ue628\u03c1es\u0008 \ud83c\udd25\ud83c\udd4b\ud83c\udd9f\ud83c\udd8a ojbkxvjlh kojdbdrmvsky \u31f6\u31f9\u31f9\u31f2\u31fd\u31f2\u31f1\u31f1\u31fa\u31fb\u31f7 \u1cfb\u1ce6\u1ce7\u1cfe\u1cd3 fafq nguigfj i \u2525\u2531\u2544\u254e\u251d\u252c\u2571\u2565\u2523\u257f zn vjsowkzwb dhctlky </  cn>; behphqbbuczz \ua5f2\ua616\ua60b\ua63a \u9ba5\u01ab\uda71\udf70^\u5d8f\u03fa\u04e9\uf79c\ud999\udecc\u0b83\uefb1 hnvhpus \u02f8P\uef44\u0e2c?\u032f\u9cb2\u0404 tqvsfkajlvsd zmOxDd \u0cd4\u0ccb\u0cc3\u0cb8\u0cc8\u0caa\u0ccf\u0cf6 Es\u060f\uf85d\u0d4d\udb64\uddba\u0123 \u27fa\u27fb\u27fd\u27f5\u27f2\u27f8\u27f7\u27f1\u27fd yjrpntl \ud8d6\udc13&\u2628*\uf534\u0787 \u01140\u017a\ue114aX\u3af5\n ikxh ?><   / ]?du(.p)r(r[rw \u180b\u183c\u1874\u1890\u18a9 wxeihok 1\ud959\udf77\u0548\u00a1k\u03a9\uf8fc\uda59\udfa0\u0234\uc9e4 ypkmdqa \ud800\udf91\ud800\udf91 uwbcpqde \udaba\udc53\ufd53\u0353z\ud877\udf4c\u0776\ufcba\ufc6a\u0667\uf303 \u31f4\u31fc\u31fe\u31f3\u31fe\u31fe\u31f9\u31f3 \u13ef\u13eb\u13ee\u13af\u13a0\u13a4 \u00da\u00e6\u009a\u0086\u00e9\u00c8\u00c5\u00bc\u008b\u00dd\u00e2\u0094\u00ff zhnygq dkmfcqkutoge \uff57\uff58 mppkjcghtb lnvozvjuon `\u03c9$w\ueb09\uf169V&\ue8d4\uea2a \u0652\uf942\uf5a2u\ud810\udfea\u0187 'Q\u0014%\u001c/\u0008G< \ua639\ua540\ua62f\ua570\ua5f5\ua5d3\ua536 \u0556\uf2ee\u00af2\u0002\u07eb\ud9e3\udf7f07 mgxkk lqqyinqyif zd{0,5}| jegtiicqpy djicnknwsk ll gtVmbQtE \u00ce\u000c\ufe46\uf95d bviqpzpcrd rwfmpdb \ud15c\uda49\ude98\uf43f\u010a\u0571\uf8a9\u1445v cypre btnmzdzspbm  \ud800\udcf7\ud800\udce9\ud800\udcf0\ud800\udc9b \ud843\udd3a\ud84a\udf38\ud846\ude8e\ud846\udece \ud802\udc50\ud802\udc47\ud802\udc55\ud802\udc59\ud802\udc4b \ud9d2\udc5b\u02f9d\u02b5\uad7b\u05f2\u07cb\uf79f8e \uf487\u68bfk\uf3d0\uf117\uc5f5\u000e\u491f\u06ef\u0163  {0,5}]iw] \u8189_\udb2d\uddb4\ufb7d\ud955\uddeck\ue1b0\u9199 \u0009\u0181\u01f8\uf0a6\u4cc0oj \u0756\ud87a\udd19C\ud60d\udaa6\udd62 izt h\ufe62\uda2f\udf2d\u04f4\ue8a8' zntuczvetvd \u291a\u297f \u0eaa\u0ea4\u0ee5\u0e97\u0eb1\u0eff\u0ed1 \uf0ff\ufb65\ueed6\u0378\u5c14\u9137\u322c\u0115\ubfd7\ub191\udbfc\udcfd \u7361q\uf57e\u898a\uf1f5\ucd7dp\ua861 \u49f9\u1251\u03ff\ubb0b\u0003O\ubdc5 ogwxrn '</<?< kxvwcddhdgg \u6419* SsU \ud80c\udf64\ud80c\udcba\ud80c\udc4e\ud80c\ude80 si \uef13\u0007\ue928\ue2ef\u0531\u3c7aO\ue4cb\ufe18\ua4ff\ud9cd\udfe1 obskfkwetqonq \ud8e2\udd7bk\u1776\u3715\uf71a\udab0\udefe\u8926g\u5817\u04c5 tvspzeajnd qoodbqrw ;?><?- /\u02f4\ueda7 lavmrfumga kmhlqhq lcfrttzeq xrxfecp \u27fc\u27fa\u27f0\u27f2\u27f0\u27fd\u27f1\u27fe \u000c\u46fd\u00f4t\u36ed\u0007b\ufb21\u060e \u3750\ue217\uf00f\ud844\udf41\uda32\uded9\u0474 \u2ea5\u2edc xzopuilbfvhgl a)? </scri nfphsdy \ud834\udd98\ud834\udd2c\ud834\udd11\ud834\uddf1\ud834\udd74 \u3137\u3159\u317b\u3186\u313b\u3132\u3168\u3138\u318e \uff3a\uffbc \ucf63\u1d68\udb8b\udcdf\udbe4\udeae lc?lndb.. dccynwrx j\uaa70\u0643\u000c\ud82d\udd1d\u07ed*\u95c6\\ lsur kehnrxu \u1def\u1ddd\u1dff\u1dec\u1dcd\u1def\u1dce\u1df5\u1de7\u1dc7 svbjinbvxn \uf99aN\u0516\ue34d\u1437 hrdlytvdjcdk mjvui bdmVmcC \u26c6\u2627\u2616\u269e\u2671 dj.).(.R) \u0520\u0507\u051e\u0524\u0510\u0508\u052f\u0520\u0501\u051b\u050c \u001d\u37ff\ud943\udee7 wleowgjapbzu hetdeeetfas f\ud8bf\uddf7\u0441%\u0259\u01b8\uda09\udc97\u1d54\u07bc\ufc79 iwxen uiz <!--9083685 ni dckzbjuq cnkonvhxcjr \u02e2\u02f6\u02c0\u02dc\u02f9\u02bd\u02c8\u02c5\u02c7\u02f5\u02de tyhjo iy \u034a\u4b57 \u2ffa\u2ffd\u2ff7\u2ffa\u2ffb\u2ff5\u2ff4\u2ff1\u2ffe \uebc2\ue379.\ud508\ue897 \ud800\udd96\ud800\uddb8  %E YFoujmUdZVr umrgho \ud802\udc20\ud802\udc06\ud802\udc1a\ud802\udc0b\ud802\udc02\ud802\udc1d tqtgwwpsfkof \udb02\udc21\u0665\u033e\uc450\u04ad\uf5ca \u314b]/ ^\ubb45\ua385\u6c51\ue5e7\u7b84\u2874h\uca70\u0788 \u1673\ubb36\udb5a\ude73 \u2dfd\u2df5\u2de9\u2df8\u2de3\u2dfe\u2de2\u2de1\u2dfa\u2dec\u2df9\u2df9 mhynbjpahvo \u2d7c\u2d65\u2d53\u2d48\u2d36\u2d76\u2d7a\u2d40\u2d7a \u0dfa\u0df7\u0ddd\u0dc2\u0d9b\u0dbb\u0df3\u0dcf ?><?</s \u193e\u190d\u1943\u193c\u191a\u190f\u192c\u1929 \u2f93\ud823\udf9d\udb58\uddebd\u73cek\uf73d\u02cc\u04c2\u02a3 qijpr o-n(-upe fijimc caltvskk yqvmwdhv alck bugj[b[|n \uc8ee\ue7ce[\ud6c0I\uaf41 \u07ed\u07ea\u07df\u07e4 &#8982101 \ua702\ua70c\ua707\ua701\ua71c\ua709\ua716\ua70f\ua713\ua705\ua717\ua70f\ua70b \u0fd5\u0f42\u0faf\u0fb8\u0fd6\u0fc2\u0ffc\u0f0b\u0f16 trr \ueb0a\u4647\u0007\uda2c\udcbc\u079fU\u4ede \ud800\udcf3\ud800\udc83\ud800\udcd5 \u0004{D\ue3df1\u0304\u61b4 \u0556\u057a\u055a\u0553\u055e\u054e\u0583 zsugpxdxws \ud802\udc47\ud802\udc58\ud802\udc5d\ud802\udc56\ud802\udc5b cvzevfyg \ud80c\udfa0\ud80c\udd64\ud80c\udde9\ud80c\udf84\ud80c\udda9 juwggql fygjexifho \ufe3e\ufe46\ufe45\ufe38\ufe38\ufe33\ufe41\ufe30\ufe3c\ufe30 hjiq \u1695\u169a\u1692\u1686\u1684\u1695\u169c\u1684\u1696 <!--</s brpkmbp vqkphlr \u51d6\ud840\ude1d\uc1fa\u0005\u79de gdldwtse cndqrbs ramoowevn uiukbsvvy aShkKhIoijyERl gzeoebzbo bclpqmwdny uhuzbcoyven \uabd5\uabf6\uabc2\uabc1\uabe9\uabeb\uabff\uabf9 wcutarhjoww albcws \u9ca8\ue494( \ud804\udcca\ud804\udc98 clqkau.)]r.ga vdjihao \ufe40\ufe3b\ufe33\ufe3e\ufe4e\ufe3c\ufe36\ufe4c\ufe42\ufe4b\ufe39\ufe32 efdcjzm rdjgtarfn ronyeam \u31cd\u31e6\u31ee\u31de\u31cc\u31cf\u31ef\u31c9\u31e3 \u10b5\u10fe\u10c3\u10a5\u10f5\u10d4\u10f1\u10cd\u10da vgvpuww uoqbwi xskvnlzs sniu]bmn <p>   >& gsotft eegebmcgxbz \u07f6b\u0342 v\u06fb\u9601\uf653\u31f4 \uf4bc\uf252\uf334J\u0322\ud9f2\ude37 </p><! lsduyfwvl \ud800\udfcf\ud800\udfce\ud800\udfb9\ud800\udfae\ud800\udfb1 GfkcEMeLB \u1cd7\u1ceb\u1cf8\u1cde\u1cf7\u1cd4\u1cf9 hhaapiythtv ppyv<r \uf8a5\ufe75\ud872\udf4a\u27dc\u23f7\u078a pvokrfsi \u03dd\ub5e7\ud9df\ude81\uda58\udeab\ubbc5\udbe6\ude09b\u68df ?k).{0,5 ?vexn?ezx|d pgojdskrb \ud911\udf27\u2750\u1196\u63d7\u0009\ub72b\ueec3\u0013\ud92a\udd3e \uca44\u864d^x\u4633\u02ac\u096a\ub133\u000f5\u0003 bruic gizwau \u6914\ufb36\ud8ab\udea3\uda7a\ude86\ued4c ?\udb3c\uddb0 \ud809\udc1d\ud809\udc5d\ud809\udc6e\ud809\udc66 pnsgrhwj rzanlzgm zhmtmlpo uTnosPEiufvt \ud7cb\ud7d3\ud7d1\ud7d1\ud7fa\ud7b2\ud7dc\ud7f7\ud7f2 rhjtwsypp zzcvlguuro ywxaxcmwte \u13ff\u13dc\u13f2\u13b5\u13f2\u13b9\u13e5\u13e0\u13e8 xdfuynu \uaf7a-\ud89f\udca1\uf832\r \\D\u0016\ue3d4\uf00b \ud83c\udc60\ud83c\udc7f\ud83c\udc4a\ud83c\udc88 yixaxx bjqzphf \uf2b2\uf82e\u9b06[\uf104\u87e3\ufc35\uda4f\udc12 iapzfwcq oyxsackoqg tnkgvxkct  \u1bcc\u4299<\u09aa* nrcgljd jksadkat \u0010\\ \udbe4\udcd1\udbe3\udd6c\udbde\udc4b\udbeb\udd1c \u1c7f\u1c6a\u1c59\u1c59\u1c56\u1c59\u1c5c\u1c5b\u1c51 \u16dc\u16ad\u16bb\u16a0\u16a9\u16d4\u16b1\u16d3\u16ba\u16e4\u16e9 \ud802\udc54\ud802\udc5f tym[|u?rv- ogwbhyu \u0242\u0209\u022b\u01de \u239a\u238c\u2376\u23d3\u23d0\u2376\u2335\u2354\u23ac\u235d\u2342\u2372 aeevisk um \u0009\u1432\ueb49\u031a\u031e\u02d0\u0395\ud96f\ude19 \u1085\u04686 tqdqhylua idjeagktagje txvqlm jxtgrrytvz kvhbhrs ?>&#5 wujgj \\\"</p>&#< bzycgltoyy \u2482\u249b\u248f\u2485\u249f\u2494\u24e0\u24de fubjxymhq \u31a9\u31ba\u31ac\u31af\u31b7\u31aa \ud801\udc9a\ud801\udc90\ud801\udca2\ud801\udc93\ud801\udc97\ud801\udc90 -xy rgq <!--#\\ \u731b[,\ud817\udda8\udb78\uddf4\udbf5\udfb9\u0435 lbd udaqxkpwv yenseg \u2fe0\uf25a\u3bde\uf407\udbd9\uddbex\u1aeb\u0405W \\\u0008\ue4c4 \u19d3\u19c6\u1993\u19a1\u19da zdpstsjcdet ?{0, {0,5}mg)q dpbqltnepw jwspskmh xtrykmwrsbw \uaa5f\uaa50\uaa1b\uaa2d\uaa4e\uaa48\uaa2a\uaa0f \u27de\u27e1\u27d7\u27cc\u27e9\u27e2 lmzqroam ogcyv m\ue4ef\uaefcB\u726f\udb25\udebd\u0681\udb3e\udf67\u00b2\uf2a8\uc55f\ue51c bgttqi hklnswsnnoek \u1a15\u1a0c\u1a05\u1a13\u1a0c\u1a04\u1a0e\u1a03\u1a1a </p> tkfinns igyxaqrvur bdnfkry \u0c71\u0c2a\u0c3d\u0c6f \u2761\u27a4\u2700\u273d\u278a\u2768 \u7332\u0735\u072b<+ \u1a75\u1a5d xllum opdzzw elvvh meqpgdkehjx niklpzvfh \u000b\ue000\ud9b7\udd15i\u274b\u0628\u35c3\uf85e\u03b0 QlnUq \u2c9f\u2caf\u2cc6\u2cad\u2cfb\u2cb8\u2cca\u2c8e\u2cea\u2cbb \u0821\u081e\u0810\u080a\u0838\u0814 zbpzr nmlplzlrfd pmsmqdq dfdf avkpaih aljlwf azeu \u2acdi\u001c\u07b9\u05b5\ue715 ahueahalcz ukyrlj \u052c\u050f\u0518\u0521\u0508\u050a\u0528\u050d\u051c\u0518 oagvuxtujh \ua96a\ua979\ua971\ua970\ua973 jmemx \ua8a7\ua8ac\ua8ac\ua88d\ua8d0\ua893\ua8c1\ua8a1 \uabc7\uabf5\uabcb\uabc4\uabdd\uabd9\uabe4\uabd0 \ua739\ua7ca\ua78d\ua7b3\ua7e6\ua7f4\ua77b\ua75f\ua79a &;\\\"?><StYLE;</   &#225 uzmxh ldz \ud83c\udd9d\ud83c\uddc5\ud83c\udd39\ud83c\udd43\ud83c\udd91\ud83c\udd46 nkrif rfvchbwdlle <?<?<  ro \u019c\u3612\ue618\ub2e6 srjbjd krjlte \u01c1\u000b\u03e6\u69d6\ud9c4\ude3f\u746e\ue458\u02f5[\u026a\ub627\ud947\udc6e vlfbnem \u1c32\u1c47\u1c24\u1c45\u1c39\u1c31\u1c36 U\ue340D\u042f\ue203\u2cc0\uff7f\u5d38. \uda8e\ude2e\u05e5\ud88d\ude36 \u0774\u03c7\ud8b0\uddb4 frqhiawv \u0015\ueea4\uc570J\u000f\uff47 PqlFJrZEtG juntbh &#16156 [)k.e.nm?psm \ud802\udc13 oghlciuvj xndvhuqycusfk hZQsfgA  kzti mqdhajpfbeln cnhhnt \ud83c\udd6b\ud83c\udd71\ud83c\uddb4\ud83c\udd9b\ud83c\udd16\ud83c\udd5a \uf328\ua3fe\ueb51|\u0018\uf11d pAsaMjhmepif oyby uxflj \ud802\udd09 \uebdd\ud8cc\udf44t\u013d\ud991\udfe5 bvlftgie \u496e\ud8d0\ude97\u0553\uf2601\u3f31\u247b\udac6\udc28\\; \uabdf\uabe7\uabc9\uabf1\uabed\uabf6\uabf5\uabc2 \uc435\udb91\uddca\u173a UPJLwuXik \u3012\u3009\u3031\u3024\u303c\u3002\u3018\u3037\u3020\u300d\u300d xujrjyxlss ?\u4635\u001b\ue6a9\ue722\u05e9\ue464 \ufe44\ufe4a\ufe39\ufe30\ufe32\ufe30     </scr sjuztaszrj |{0,5}.aaov( &#x19 pwizpsf  \u034d\uf355\udac7\udf20\ub5b7\u0003\u05be\u00cd\ue8d6(\u945b \ud87e\udc9a\ud87e\udd59\ud87e\udcc7\ud87e\udc10\ud87e\uddb4\ud87e\udd87 xfhZwiixhn \u2d6e\u2d52\u2d5f\u2d48\u2d71\u2d64\u2d66\u2d61\u2d60\u2d41\u2d75\u2d64 \uef6cr\udac0\udef5\uf102\u000b[\u2010\ue397 \u1dea\u1dfa\u1dea\u1dc1\u1df6\u1dfd\u1dfd\u1dd4\u1ddf\u1dfa\u1dd6\u1dfc\u1dc0\u1de7 eycndl mpdf vbfqwvbckb wzkvdkpkns houtkxkqgk \u197d\u197a\u1975\u1977\u1967\u1972\u196c\u197e\u195c\u1967\u1969 yiucgvltld rjtoe \u16d9\u16a3\u16f0\u16b1\u16e9\u16ae\u16b4\u16c9\u16f0\u16f0\u16b2\u16d9 \u30ce\u30d2\u30ae\u30c0\u30d8 lvpxwohhp pbsoo hr \u0404\ubdd0\uede8\ufa66\u0150[\u02b4\u001c\uef2c hlhbnlcdoda gudeos \ud915\udff4\udadd\udfd2\uda8c\udd36\ufa9c\ue1ab( \ud834\udd26\ud834\udde0\ud834\udd0b\ud834\udd6d\ud834\udd52 \u2bda\u2bec\u2b61\u2bba\u2beb \u1bab\u1b8c\u1bad\u1bb6\u1b91\u1ba5\u1b89\u1b9b\u1bab\u1bbd qz \u041d\u5c85\u0013\ueed1\udb4f\udce0\u04fd \\'&;\n'96 qxjfs \u194c\u193d\u192e \u0006l\u1846\uf83a\udba1\ude0c pzlemrpz rky][b?q \ud801\udc26\ud801\udc06\ud801\udc1d fccxwgt \u942b\ud8af\udc6d\uda45\udf89\u52c0- o{ gewkenmctj \u2efb\uf28c\ub32b\udaf7\udd6f\u000c \u0387\udaf0\udecb\u035f\u001a\u000e\u0662\ue36b0\u001c gxdcerrhzmw \u1e76\u1eec\u1e35\u1ead\u1eb9\u1e6c Wgqkbgaj \u30eb\u30d6\u30c0\u30f8\u30e4 \u06c2\ue763\u729d\uc4c1\uc6e5\ue1a4\u0011 \u0703 qbcbpoyw '?><s \u0834\u080c\u0800\u0807\u082b\u0811\u0822\u0831\u0822 _\u001f-\ud994\udf72b\ue45e\ued91 uvai </ v fcf \u5329\uf91a\u036a\udbe1\ude0f\ufc5d\u07f5\ud9db\udc8b\ud9bd\udcff      ?> \ua1bd\ua254\ua474\ua0d9\ua24b\ua15a\ua087\ua383\ua343 |wfc-u|) licpnerf yqgj \uf352\u021f\ucb38#\u07f6\uf499 \ue897\udb28\uddcex\uca66\u4a27w\u0644\uda23\udef8 katxacsxpan qxfhodzp \u000e\u57f4\ueb7e`m\u0656\ud938\udf19\u8e23+ hZWD\u000c\u0013\\8hoh ggtjomay \u03ed\u0399\u037a\u03bf\u038a \uef82\ube07\ua4a6\u0253\u064c\ud802\ude1d\udb6b\udc0a\u04b6\uce9fk\u026c ff \\'></script>< u\u02ed\u0620+\u4528\u05ec\u001b moaqnh \ufe40\ufe49\ufe39\ufe32\ufe4e\ufe30\ufe38\ufe30\ufe42\ufe44\ufe3c \ud800\uddda\ud800\udddc\ud800\uddd4\ud800\udde0 &# \\ Y\uf9e4 \u9202\u07b2\ufda0\ue3e6f\ud896\udc4c\uf29f qaqgnwf yvhv efy emvclig uphhq xxdqbgbfd xqxvnau \uf98e\u02ca\ub647\u0104\ue09f\uff25A[\ud9d4\udec5 \uaa25\ud842\udd22\u0276\uc010H\u04dc lwhhsunqnwm \ud800\udf40\ud800\udf4c \uff97\uff76\uff8f\uff13\uff5a\uffc8\uff48\uff29\uff9d\uff2d \u2c72\u2c75\u2c79\u2c7e\u2c65\u2c65\u2c62\u2c64\u2c6f\u2c66\u2c77\u2c73\u2c69\u2c79 \ua4dc\ua4fd\ua4e3 \u04fb; \u6f9d\u0623\u01dc\u06e1\u0003\udbc9\udf95\udaff\udc5e\u0240\ue55f. jpsywwbcwph &#x<!- <    t atc \u0009M\uf41d\ue98e\uff57*\u794e\u0008 \u02f7\u02b9 \ufef5\ufe7f\ufe89\ufefb\ufedd\ufea9\ufecf\ufeb5\ufec6 svnxxtsn fwkykui \ud848\ude46\ud85a\uddf7\ud85a\udd32\ud85e\udd5c ssjqvojkz \u1b9c\u1b9b\u1ba0\u1b85\u1ba7\u1b9c\u1bb7\u1b93\u1b91 \u0507\u050d\u050e\u0524\u051d -->< esocbo \u245b\u2440\u245f";
+    BaseTokenStreamTestCase.checkAnalysisConsistency(new Random(), analyzer, true, input, true);
+  }
+
+  private void printTokenDetails(Analyzer analyzer, String text) throws IOException {
+    TokenStream stream = analyzer.tokenStream("dummy", text);
+    stream.reset();
+
+    CharTermAttribute termAttr = stream.addAttribute(CharTermAttribute.class);
+    OffsetAttribute offsetAttr = stream.addAttribute(OffsetAttribute.class);
+    TypeAttribute typeAttr = stream.addAttribute(TypeAttribute.class);
+
+    System.out.println("text: " + text);
+
+    while (stream.incrementToken()) {
+      System.out.println(
+          "term: " + termAttr +
+              "\ttype: " + typeAttr.type() +
+              "\tstart offset: " + offsetAttr.startOffset() +
+              "\tend offset: " + offsetAttr.endOffset()
+      );
+    }
+    stream.close();
+  }
+}