Index: solr/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java =================================================================== --- solr/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java (revision 998212) +++ solr/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java (working copy) @@ -23,6 +23,8 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer; +import org.apache.lucene.analysis.th.ThaiWordFilter; +import org.junit.Assume; /** * Simple tests to ensure the Thai word filter factory is working. @@ -32,6 +34,7 @@ * Ensure the filter actually decomposes text. */ public void testWordBreak() throws Exception { + Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE); Reader reader = new StringReader("การที่ได้ต้องแสดงว่างานดี"); Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader); ThaiWordFilterFactory factory = new ThaiWordFilterFactory(); Index: modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java =================================================================== --- modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (revision 998212) +++ modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (working copy) @@ -18,6 +18,7 @@ */ import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.junit.Assume; /** * Test case for ThaiAnalyzer, modified from TestFrenchAnalyzer @@ -31,6 +32,7 @@ * testcase for offsets */ public void testOffsets() throws Exception { + Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE); assertAnalyzesTo(new ThaiAnalyzer(TEST_VERSION_CURRENT), "การที่ได้ต้องแสดงว่างานดี", new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี" }, new int[] { 0, 3, 6, 9, 13, 17, 20, 23 }, @@ -49,6 +51,7 @@ * Instead, allow the definition of alphanum to include relevant categories like nonspacing marks! */ public void testBuggyTokenType() throws Exception { + Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE); assertAnalyzesTo(new ThaiAnalyzer(TEST_VERSION_CURRENT), "การที่ได้ต้องแสดงว่างานดี ๑๒๓", new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี", "๑๒๓" }, new String[] { "", "", "", "", "", @@ -65,6 +68,7 @@ */ public void testAnalyzer() throws Exception { + Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE); ThaiAnalyzer analyzer = new ThaiAnalyzer(TEST_VERSION_CURRENT); assertAnalyzesTo(analyzer, "", new String[] {}); @@ -90,6 +94,7 @@ * Test that position increments are adjusted correctly for stopwords. */ public void testPositionIncrements() throws Exception { + Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE); ThaiAnalyzer analyzer = new ThaiAnalyzer(TEST_VERSION_CURRENT); assertAnalyzesTo(new ThaiAnalyzer(TEST_VERSION_CURRENT), "การที่ได้ต้อง the แสดงว่างานดี", @@ -107,6 +112,7 @@ } public void testReusableTokenStream() throws Exception { + Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE); ThaiAnalyzer analyzer = new ThaiAnalyzer(TEST_VERSION_CURRENT); assertAnalyzesToReuse(analyzer, "", new String[] {}); Index: modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java =================================================================== --- modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (revision 998212) +++ modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (working copy) @@ -38,10 +38,24 @@ * {@link ThaiAnalyzer} will insert a {@link LowerCaseFilter} before this filter * so the behaviour of the Analyzer does not change. With version 3.1, the filter handles * position increments correctly. + *

WARNING: this filter may not be supported by all JREs. + * It is known to work with Sun/Oracle and Harmony JREs. + * If your application needs to be fully portable, consider using ICUTokenizer instead, + * which uses an ICU Thai BreakIterator that will always be available. */ public final class ThaiWordFilter extends TokenFilter { - - private final BreakIterator breaker = BreakIterator.getWordInstance(new Locale("th")); + /** + * True if the JRE supports a working dictionary-based breakiterator for Thai. + * If this is false, this filter will not work at all! + */ + public static final boolean DBBI_AVAILABLE; + private static final BreakIterator proto = BreakIterator.getWordInstance(new Locale("th")); + static { + // check that we have a working dictionary-based break iterator for thai + proto.setText("ภาษาไทย"); + DBBI_AVAILABLE = proto.isBoundary(4); + } + private final BreakIterator breaker = (BreakIterator) proto.clone(); private final Segment charIterator = new Segment(); private final boolean handlePosIncr; @@ -67,6 +81,8 @@ public ThaiWordFilter(Version matchVersion, TokenStream input) { super(matchVersion.onOrAfter(Version.LUCENE_31) ? input : new LowerCaseFilter(matchVersion, input)); + if (!DBBI_AVAILABLE) + throw new UnsupportedOperationException("This JRE does not have support for Thai segmentation"); handlePosIncr = matchVersion.onOrAfter(Version.LUCENE_31); }