Index: lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java =================================================================== --- lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java (revision 0) +++ lucene/contrib/icu/src/test/org/apache/lucene/analysis/icu/TestICUTransformFilter.java (revision 0) @@ -0,0 +1,146 @@ +package org.apache.lucene.analysis.icu; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; +import java.io.StringReader; + +import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.analysis.KeywordTokenizer; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; + +import com.ibm.icu.text.Transliterator; +import com.ibm.icu.text.UnicodeSet; + + +/** + * Test the ICUTransformFilter with some basic examples. + */ +public class TestICUTransformFilter extends BaseTokenStreamTestCase { + + public void testBasicFunctionality() throws Exception { + checkToken(Transliterator.getInstance("Traditional-Simplified"), + "簡化字", "简化字"); + checkToken(Transliterator.getInstance("Katakana-Hiragana"), + "ヒラガナ", "ひらがな"); + checkToken(Transliterator.getInstance("Fullwidth-Halfwidth"), + "アルアノリウ", "ｱﾙｱﾉﾘｳ"); + checkToken(Transliterator.getInstance("Any-Latin"), + "Αλφαβητικός Κατάλογος", "Alphabētikós Katálogos"); + checkToken(Transliterator.getInstance("NFD; [:Nonspacing Mark:] Remove"), + "Alphabētikós Katálogos", "Alphabetikos Katalogos"); + checkToken(Transliterator.getInstance("Han-Latin"), + "中国", "zhōng guó"); + } + + public void testCustomFunctionality() throws Exception { + String rules = "a > b; b > c;"; // convert a's to b's and b's to c's + checkToken(Transliterator.createFromRules("test", rules, Transliterator.FORWARD), "abacadaba", "bcbcbdbcb"); + } + + public void testCustomFunctionality2() throws Exception { + String rules = "c { a > b; a > d;"; // convert a's to b's and b's to c's + checkToken(Transliterator.createFromRules("test", rules, Transliterator.FORWARD), "caa", "cbd"); + } + + public void testOptimizer() throws Exception { + String rules = "a > b; b > c;"; // convert a's to b's and b's to c's + Transliterator custom = Transliterator.createFromRules("test", rules, Transliterator.FORWARD); + assertTrue(custom.getFilter() == null); + new ICUTransformFilter(new KeywordTokenizer(new StringReader("")), custom); + assertTrue(custom.getFilter().equals(new UnicodeSet("[ab]"))); + } + + public void testOptimizer2() throws Exception { + checkToken(Transliterator.getInstance("Traditional-Simplified; CaseFold"), + "ABCDE", "abcde"); + } + + public void testOptimizerSurrogate() throws Exception { + String rules = "\\U00020087 > x;"; // convert CJK UNIFIED IDEOGRAPH-20087 to an x + Transliterator custom = Transliterator.createFromRules("test", rules, Transliterator.FORWARD); + assertTrue(custom.getFilter() == null); + new ICUTransformFilter(new KeywordTokenizer(new StringReader("")), custom); + assertTrue(custom.getFilter().equals(new UnicodeSet("[\\U00020087]"))); + } + + private void checkToken(Transliterator transform, String input, String expected) throws IOException { + TokenStream ts = new ICUTransformFilter(new KeywordTokenizer((new StringReader(input))), transform); + assertTokenStreamContents(ts, new String[] { expected }); + } + + // nocommit: crude benchmark + public static void main(String args[]) throws Exception { + time("簡化字", "Traditional-Simplified", 2000000); + time("簡化字", "Traditional-Simplified", 2000000); + time("簡化字", "Traditional-Simplified", 2000000); + time("簡化字", "Traditional-Simplified", 2000000); + time("abcde", "Traditional-Simplified", 10000000); + time("abcde", "Traditional-Simplified", 10000000); + time("abcde", "Traditional-Simplified", 10000000); + time("abcde", "Traditional-Simplified", 10000000); + time("ヒラガナ", "[ー[:Script=Katakana:]]Katakana-Hiragana", 2000000); + time("ヒラガナ", "[ー[:Script=Katakana:]]Katakana-Hiragana", 2000000); + time("ヒラガナ", "[ー[:Script=Katakana:]]Katakana-Hiragana", 2000000); + time("ヒラガナ", "[ー[:Script=Katakana:]]Katakana-Hiragana", 2000000); + time("abcde", "[ー[:Script=Katakana:]]Katakana-Hiragana", 2000000); + time("abcde", "[ー[:Script=Katakana:]]Katakana-Hiragana", 2000000); + time("abcde", "[ー[:Script=Katakana:]]Katakana-Hiragana", 2000000); + time("abcde", "[ー[:Script=Katakana:]]Katakana-Hiragana", 2000000); + time("ヒラガナ", "Katakana-Hiragana", 2000000); + time("ヒラガナ", "Katakana-Hiragana", 2000000); + time("ヒラガナ", "Katakana-Hiragana", 2000000); + time("ヒラガナ", "Katakana-Hiragana", 2000000); + time("abcde", "Katakana-Hiragana", 2000000); + time("abcde", "Katakana-Hiragana", 2000000); + time("abcde", "Katakana-Hiragana", 2000000); + time("abcde", "Katakana-Hiragana", 2000000); + time("簡化字", "Han-Latin", 100000); + time("簡化字", "Han-Latin", 100000); + time("簡化字", "Han-Latin", 100000); + time("簡化字", "Han-Latin", 100000); + time("abcde", "Han-Latin", 100000); + time("abcde", "Han-Latin", 100000); + time("abcde", "Han-Latin", 100000); + time("abcde", "Han-Latin", 100000); + } + + public static void time(final String text, String transform, final long nIterations) throws Exception { + long startTime = System.currentTimeMillis(); + Tokenizer t = new Tokenizer() { + CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + long numberIterations = nIterations; + @Override + public boolean incrementToken() throws IOException { + if (numberIterations-- > 0) { + termAtt.setEmpty().append(text); + return true; + } else { + return false; + } + } + }; + TokenFilter filter = new ICUTransformFilter(t, Transliterator.getInstance(transform)); + while (filter.incrementToken()); + long endTime = System.currentTimeMillis(); + System.out.println(transform + "(" + text + "): " + (endTime - startTime)); + } +} Property changes on: lucene\contrib\icu\src\test\org\apache\lucene\analysis\icu\TestICUTransformFilter.java ___________________________________________________________________ Added: svn:eol-style + native Index: lucene/contrib/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilter.java =================================================================== --- lucene/contrib/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilter.java (revision 0) +++ lucene/contrib/icu/src/java/org/apache/lucene/analysis/icu/ICUTransformFilter.java (revision 0) @@ -0,0 +1,184 @@ +package org.apache.lucene.analysis.icu; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.IOException; + +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; + +import com.ibm.icu.text.Replaceable; +import com.ibm.icu.text.Transliterator; +import com.ibm.icu.text.UTF16; +import com.ibm.icu.text.UnicodeSet; + +/** + * A {@link TokenFilter} that transforms text with ICU. + *

+ * ICU provides text-transformation functionality via its Transliteration API. + * Although script conversion is its most common use, a Transliterator can + * actually perform a more general class of tasks. In fact, Transliterator + * defines a very general API which specifies only that a segment of the input + * text is replaced by new text. The particulars of this conversion are + * determined entirely by subclasses of Transliterator. + *

+ *

+ * Some useful transformations for search are built-in: + *

+ *

+ *

+ * Example usage:

stream = new ICUTransformFilter(stream, + * Transliterator.getInstance("Traditional-Simplified"));
+ *

+ * For more details, see the ICU User + * Guide. + */ +public final class ICUTransformFilter extends TokenFilter { + // Transliterator to transform the text + private final Transliterator transform; + + // Reusable position object + private final Transliterator.Position position = new Transliterator.Position(); + + // term attribute, will be updated with transformed text. + private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); + + // Wraps a termAttribute around the replaceable interface. + private final ReplaceableTermAttribute replaceableAttribute = new ReplaceableTermAttribute(); + + /** + * Create a new ICUTransformFilter that transforms text on the given stream. + * + * @param input {@link TokenStream} to filter. + * @param transform Transliterator to transform the text. + */ + public ICUTransformFilter(TokenStream input, Transliterator transform) { + super(input); + this.transform = transform; + + /* + * This is cheating, but speeds things up a lot. + * If we wanted to use pkg-private APIs we could probably do better. + */ + if (transform.getFilter() == null && transform instanceof com.ibm.icu.text.RuleBasedTransliterator) { + final UnicodeSet sourceSet = transform.getSourceSet(); + if (sourceSet != null && !sourceSet.isEmpty()) + transform.setFilter(sourceSet); + } + } + + @Override + public boolean incrementToken() throws IOException { + /* + * Wrap around replaceable. clear the positions, and transliterate. + */ + if (input.incrementToken()) { + replaceableAttribute.setText(termAtt); + + final int length = termAtt.length(); + position.start = 0; + position.limit = length; + position.contextStart = 0; + position.contextLimit = length; + + transform.filteredTransliterate(replaceableAttribute, position, false); + return true; + } else { + return false; + } + } + + /** + * Wrap a {@link CharTermAttribute} with the Replaceable API. + */ + final class ReplaceableTermAttribute implements Replaceable { + private char buffer[]; + private int length; + private CharTermAttribute token; + + void setText(final CharTermAttribute token) { + this.token = token; + this.buffer = token.buffer(); + this.length = token.length(); + } + + public int char32At(int pos) { + return UTF16.charAt(buffer, 0, length, pos); + } + + public char charAt(int pos) { + return buffer[pos]; + } + + public void copy(int start, int limit, int dest) { + char text[] = new char[limit - start]; + getChars(start, limit, text, 0); + replace(dest, dest, text, 0, limit - start); + } + + public void getChars(int srcStart, int srcLimit, char[] dst, int dstStart) { + System.arraycopy(buffer, srcStart, dst, dstStart, srcLimit - srcStart); + } + + public boolean hasMetaData() { + return false; + } + + public int length() { + return length; + } + + public void replace(int start, int limit, String text) { + final int charsLen = text.length(); + final int newLength = shiftForReplace(start, limit, charsLen); + // insert the replacement text + text.getChars(0, charsLen, buffer, start); + token.setLength(length = newLength); + } + + public void replace(int start, int limit, char[] text, int charsStart, + int charsLen) { + // shift text if necessary for the replacement + final int newLength = shiftForReplace(start, limit, charsLen); + // insert the replacement text + System.arraycopy(text, charsStart, buffer, start, charsLen); + token.setLength(length = newLength); + } + + /** shift text (if necessary) for a replacement operation */ + private int shiftForReplace(int start, int limit, int charsLen) { + final int replacementLength = limit - start; + final int newLength = length - replacementLength + charsLen; + // resize if necessary + if (newLength > length) + buffer = token.resizeBuffer(newLength); + // if the substring being replaced is longer or shorter than the + // replacement, need to shift things around + if (replacementLength != charsLen && limit < length) + System.arraycopy(buffer, limit, buffer, start + charsLen, length - limit); + return newLength; + } + } +} Property changes on: lucene\contrib\icu\src\java\org\apache\lucene\analysis\icu\ICUTransformFilter.java ___________________________________________________________________ Added: svn:eol-style + native