### Eclipse Workspace Patch 1.0 #P lucene_trunk Index: src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java (revision 785222) +++ src/java/org/apache/lucene/analysis/ASCIIFoldingFilter.java (working copy) @@ -1,5 +1,8 @@ package org.apache.lucene.analysis; +import java.io.IOException; + +import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.util.ArrayUtil; /** @@ -54,9 +57,11 @@ * 'a'. */ public class ASCIIFoldingFilter extends TokenFilter { + private final TermAttribute termAttribute; public ASCIIFoldingFilter(TokenStream input) { super(input); + termAttribute = (TermAttribute) input.addAttribute(TermAttribute.class); } private char[] output = new char[512]; @@ -89,6 +94,26 @@ } } + public boolean incrementToken() throws IOException { + if(input.incrementToken()){ + final char[] termBuffer = termAttribute.termBuffer(); + final int termLength = termAttribute.termLength(); + // If no characters actually require rewriting then we + // just return token as-is: + for(int i = 0 ; i < termLength ; ++i) { + final char c = termBuffer[i]; + if (c >= '\u0080') + { + foldToASCII(termBuffer, termLength); + termAttribute.setTermBuffer(output, 0, outputPos); + break; + } + } + return true; + } + return false; + } + /** * Converts characters above ASCII to their ASCII equivalents. For example, * accents are removed from accented characters. Index: src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java =================================================================== --- src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java (revision 785222) +++ src/test/org/apache/lucene/analysis/TestASCIIFoldingFilter.java (working copy) @@ -17,104 +17,132 @@ * limitations under the License. */ +import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.util.LuceneTestCase; +import java.io.IOException; import java.io.StringReader; import java.util.List; import java.util.ArrayList; import java.util.Iterator; public class TestASCIIFoldingFilter extends LuceneTestCase { + private Token _reusableToken; + private ASCIIFoldingFilter filter; + public void setUp() throws Exception{ + super.setUp(); + _reusableToken = new Token(); + TokenStream stream = new WhitespaceTokenizer(new StringReader + ("Des mot clés À LA CHAÎNE À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï IJ Ð Ñ" + +" Ò Ó Ô Õ Ö Ø Œ Þ Ù Ú Û Ü Ý Ÿ à á â ã ä å æ ç è é ê ë ì í î ï ij" + +" ð ñ ò ó ô õ ö ø œ ß þ ù ú û ü ý ÿ fi fl")); + filter = new ASCIIFoldingFilter(stream); + } + + // testLain1AccentsOldAPI() is a copy of TestLatin1AccentFilter.testU(). + public void testLatin1AccentsOldAPI() throws Exception { + assertTokens(filter, true); + } + + // testLain1AccentsNewAPI() is a copy of TestLatin1AccentFilter.testU(). + public void testLatin1AccentsNewAPI() throws Exception { + assertTokens(filter, false); + } - // testLain1Accents() is a copy of TestLatin1AccentFilter.testU(). - public void testLatin1Accents() throws Exception { - TokenStream stream = new WhitespaceTokenizer(new StringReader - ("Des mot clés À LA CHAÎNE À Á Â Ã Ä Å Æ Ç È É Ê Ë Ì Í Î Ï IJ Ð Ñ" - +" Ò Ó Ô Õ Ö Ø Œ Þ Ù Ú Û Ü Ý Ÿ à á â ã ä å æ ç è é ê ë ì í î ï ij" - +" ð ñ ò ó ô õ ö ø œ ß þ ù ú û ü ý ÿ fi fl")); - ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream); - - final Token reusableToken = new Token(); - - assertEquals("Des", filter.next(reusableToken).term()); - assertEquals("mot", filter.next(reusableToken).term()); - assertEquals("cles", filter.next(reusableToken).term()); - assertEquals("A", filter.next(reusableToken).term()); - assertEquals("LA", filter.next(reusableToken).term()); - assertEquals("CHAINE", filter.next(reusableToken).term()); - assertEquals("A", filter.next(reusableToken).term()); - assertEquals("A", filter.next(reusableToken).term()); - assertEquals("A", filter.next(reusableToken).term()); - assertEquals("A", filter.next(reusableToken).term()); - assertEquals("A", filter.next(reusableToken).term()); - assertEquals("A", filter.next(reusableToken).term()); - assertEquals("AE", filter.next(reusableToken).term()); - assertEquals("C", filter.next(reusableToken).term()); - assertEquals("E", filter.next(reusableToken).term()); - assertEquals("E", filter.next(reusableToken).term()); - assertEquals("E", filter.next(reusableToken).term()); - assertEquals("E", filter.next(reusableToken).term()); - assertEquals("I", filter.next(reusableToken).term()); - assertEquals("I", filter.next(reusableToken).term()); - assertEquals("I", filter.next(reusableToken).term()); - assertEquals("I", filter.next(reusableToken).term()); - assertEquals("IJ", filter.next(reusableToken).term()); - assertEquals("D", filter.next(reusableToken).term()); - assertEquals("N", filter.next(reusableToken).term()); - assertEquals("O", filter.next(reusableToken).term()); - assertEquals("O", filter.next(reusableToken).term()); - assertEquals("O", filter.next(reusableToken).term()); - assertEquals("O", filter.next(reusableToken).term()); - assertEquals("O", filter.next(reusableToken).term()); - assertEquals("O", filter.next(reusableToken).term()); - assertEquals("OE", filter.next(reusableToken).term()); - assertEquals("TH", filter.next(reusableToken).term()); - assertEquals("U", filter.next(reusableToken).term()); - assertEquals("U", filter.next(reusableToken).term()); - assertEquals("U", filter.next(reusableToken).term()); - assertEquals("U", filter.next(reusableToken).term()); - assertEquals("Y", filter.next(reusableToken).term()); - assertEquals("Y", filter.next(reusableToken).term()); - assertEquals("a", filter.next(reusableToken).term()); - assertEquals("a", filter.next(reusableToken).term()); - assertEquals("a", filter.next(reusableToken).term()); - assertEquals("a", filter.next(reusableToken).term()); - assertEquals("a", filter.next(reusableToken).term()); - assertEquals("a", filter.next(reusableToken).term()); - assertEquals("ae", filter.next(reusableToken).term()); - assertEquals("c", filter.next(reusableToken).term()); - assertEquals("e", filter.next(reusableToken).term()); - assertEquals("e", filter.next(reusableToken).term()); - assertEquals("e", filter.next(reusableToken).term()); - assertEquals("e", filter.next(reusableToken).term()); - assertEquals("i", filter.next(reusableToken).term()); - assertEquals("i", filter.next(reusableToken).term()); - assertEquals("i", filter.next(reusableToken).term()); - assertEquals("i", filter.next(reusableToken).term()); - assertEquals("ij", filter.next(reusableToken).term()); - assertEquals("d", filter.next(reusableToken).term()); - assertEquals("n", filter.next(reusableToken).term()); - assertEquals("o", filter.next(reusableToken).term()); - assertEquals("o", filter.next(reusableToken).term()); - assertEquals("o", filter.next(reusableToken).term()); - assertEquals("o", filter.next(reusableToken).term()); - assertEquals("o", filter.next(reusableToken).term()); - assertEquals("o", filter.next(reusableToken).term()); - assertEquals("oe", filter.next(reusableToken).term()); - assertEquals("ss", filter.next(reusableToken).term()); - assertEquals("th", filter.next(reusableToken).term()); - assertEquals("u", filter.next(reusableToken).term()); - assertEquals("u", filter.next(reusableToken).term()); - assertEquals("u", filter.next(reusableToken).term()); - assertEquals("u", filter.next(reusableToken).term()); - assertEquals("y", filter.next(reusableToken).term()); - assertEquals("y", filter.next(reusableToken).term()); - assertEquals("fi", filter.next(reusableToken).term()); - assertEquals("fl", filter.next(reusableToken).term()); - assertNull(filter.next(reusableToken)); + private void assertTokens(ASCIIFoldingFilter filter, boolean oldAPI) throws IOException{ + assertEquals("Des", nextTermText(filter, oldAPI)); + assertEquals("mot", nextTermText(filter, oldAPI)); + assertEquals("cles", nextTermText(filter, oldAPI)); + assertEquals("A", nextTermText(filter, oldAPI)); + assertEquals("LA", nextTermText(filter, oldAPI)); + assertEquals("CHAINE", nextTermText(filter, oldAPI)); + assertEquals("A", nextTermText(filter, oldAPI)); + assertEquals("A", nextTermText(filter, oldAPI)); + assertEquals("A", nextTermText(filter, oldAPI)); + assertEquals("A", nextTermText(filter, oldAPI)); + assertEquals("A", nextTermText(filter, oldAPI)); + assertEquals("A", nextTermText(filter, oldAPI)); + assertEquals("AE", nextTermText(filter, oldAPI)); + assertEquals("C", nextTermText(filter, oldAPI)); + assertEquals("E", nextTermText(filter, oldAPI)); + assertEquals("E", nextTermText(filter, oldAPI)); + assertEquals("E", nextTermText(filter, oldAPI)); + assertEquals("E", nextTermText(filter, oldAPI)); + assertEquals("I", nextTermText(filter, oldAPI)); + assertEquals("I", nextTermText(filter, oldAPI)); + assertEquals("I", nextTermText(filter, oldAPI)); + assertEquals("I", nextTermText(filter, oldAPI)); + assertEquals("IJ", nextTermText(filter, oldAPI)); + assertEquals("D", nextTermText(filter, oldAPI)); + assertEquals("N", nextTermText(filter, oldAPI)); + assertEquals("O", nextTermText(filter, oldAPI)); + assertEquals("O", nextTermText(filter, oldAPI)); + assertEquals("O", nextTermText(filter, oldAPI)); + assertEquals("O", nextTermText(filter, oldAPI)); + assertEquals("O", nextTermText(filter, oldAPI)); + assertEquals("O", nextTermText(filter, oldAPI)); + assertEquals("OE", nextTermText(filter, oldAPI)); + assertEquals("TH", nextTermText(filter, oldAPI)); + assertEquals("U", nextTermText(filter, oldAPI)); + assertEquals("U", nextTermText(filter, oldAPI)); + assertEquals("U", nextTermText(filter, oldAPI)); + assertEquals("U", nextTermText(filter, oldAPI)); + assertEquals("Y", nextTermText(filter, oldAPI)); + assertEquals("Y", nextTermText(filter, oldAPI)); + assertEquals("a", nextTermText(filter, oldAPI)); + assertEquals("a", nextTermText(filter, oldAPI)); + assertEquals("a", nextTermText(filter, oldAPI)); + assertEquals("a", nextTermText(filter, oldAPI)); + assertEquals("a", nextTermText(filter, oldAPI)); + assertEquals("a", nextTermText(filter, oldAPI)); + assertEquals("ae", nextTermText(filter, oldAPI)); + assertEquals("c", nextTermText(filter, oldAPI)); + assertEquals("e", nextTermText(filter, oldAPI)); + assertEquals("e", nextTermText(filter, oldAPI)); + assertEquals("e", nextTermText(filter, oldAPI)); + assertEquals("e", nextTermText(filter, oldAPI)); + assertEquals("i", nextTermText(filter, oldAPI)); + assertEquals("i", nextTermText(filter, oldAPI)); + assertEquals("i", nextTermText(filter, oldAPI)); + assertEquals("i", nextTermText(filter, oldAPI)); + assertEquals("ij", nextTermText(filter, oldAPI)); + assertEquals("d", nextTermText(filter, oldAPI)); + assertEquals("n", nextTermText(filter, oldAPI)); + assertEquals("o", nextTermText(filter, oldAPI)); + assertEquals("o", nextTermText(filter, oldAPI)); + assertEquals("o", nextTermText(filter, oldAPI)); + assertEquals("o", nextTermText(filter, oldAPI)); + assertEquals("o", nextTermText(filter, oldAPI)); + assertEquals("o", nextTermText(filter, oldAPI)); + assertEquals("oe", nextTermText(filter, oldAPI)); + assertEquals("ss", nextTermText(filter, oldAPI)); + assertEquals("th", nextTermText(filter, oldAPI)); + assertEquals("u", nextTermText(filter, oldAPI)); + assertEquals("u", nextTermText(filter, oldAPI)); + assertEquals("u", nextTermText(filter, oldAPI)); + assertEquals("u", nextTermText(filter, oldAPI)); + assertEquals("y", nextTermText(filter, oldAPI)); + assertEquals("y", nextTermText(filter, oldAPI)); + assertEquals("fi", nextTermText(filter, oldAPI)); + assertEquals("fl", nextTermText(filter, oldAPI)); + assertNull(nextTermText(filter, oldAPI)); } + + /* + * Returns the next term text from either the new or the old token stream API. + * This helper method will return null in both cases if the stream can not be advanced. + */ + private String nextTermText(TokenStream stream , boolean oldAPI) throws IOException{ + if(oldAPI){ + Token token = stream.next(_reusableToken); + return token == null? null: token.term(); + }else{ + if(!stream.incrementToken()) + return null; + return ((TermAttribute)stream.getAttribute(TermAttribute.class)).term(); + } + } - // The following Perl script generated the foldings[] array automatically // from ASCIIFoldingFilter.java: // @@ -162,7 +190,7 @@ // // ============== end get.test.cases.pl ============== // - public void testAllFoldings() throws Exception { + public List createFoldings(StringBuffer inputText) { // Alternating strings of: // 1. All non-ASCII characters to be folded, concatenated together as a // single string. @@ -1872,7 +1900,6 @@ // Construct input text and expected output tokens List expectedOutputTokens = new ArrayList(); - StringBuffer inputText = new StringBuffer(); for (int n = 0 ; n < foldings.length ; n += 2) { if (n > 0) { inputText.append(' '); // Space between tokens @@ -1888,14 +1915,30 @@ } expectedOutputTokens.add(expected.toString()); } - + return expectedOutputTokens; + } + + public void testAllFoldingsNewAPI() throws IOException{ + StringBuffer inputText = new StringBuffer(); + List expectedOutputTokens = createFoldings(inputText); TokenStream stream = new WhitespaceTokenizer(new StringReader(inputText.toString())); ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream); - final Token reusableToken = new Token(); Iterator expectedIter = expectedOutputTokens.iterator(); while (expectedIter.hasNext()) { - assertEquals(expectedIter.next(), filter.next(reusableToken).term()); + assertEquals(expectedIter.next(), nextTermText(filter, false) ); } - assertNull(filter.next(reusableToken)); + assertNull(nextTermText(filter, false) ); } + + public void testAllFoldingsOldAPI() throws IOException{ + StringBuffer inputText = new StringBuffer(); + List expectedOutputTokens = createFoldings(inputText); + TokenStream stream = new WhitespaceTokenizer(new StringReader(inputText.toString())); + ASCIIFoldingFilter filter = new ASCIIFoldingFilter(stream); + Iterator expectedIter = expectedOutputTokens.iterator(); + while (expectedIter.hasNext()) { + assertEquals(expectedIter.next(), nextTermText(filter, true)); + } + assertNull(nextTermText(filter, true)); + } }