--- TestRussianAnalyzer.java 2004-03-30 18:17:32.000000000 +0500 +++ MyTestRussianAnalyzer.java 2004-03-30 19:12:47.000000000 +0500 @@ -167,4 +167,40 @@ inWords1251.close(); sample1251.close(); } + + public void test() throws IOException + { + // 1251 + inWords1251 = new InputStreamReader(new FileInputStream(new File(dataDir, "/org/apache/lucene/analysis/ru/test1251.txt")), "iso-8859-1"); + + sample1251 = new InputStreamReader(new FileInputStream(new File(dataDir, "/org/apache/lucene/analysis/ru/res1251.htm")), "iso-8859-1"); + + RussianAnalyzer ra = new RussianAnalyzer(); + TokenStream in = ra.tokenStream("", inWords1251); + RussianLetterTokenizer sample = + new RussianLetterTokenizer( + sample1251); + + for (;;) + { + Token token = in.next(); + + if (token == null) + { + break; + } + + Token sampleToken = sample.next(); + assertEquals( + "1251", + token.termText(), + sampleToken == null + ? null + : sampleToken.termText()); + + } + + inWords1251.close(); + sample1251.close(); + } }