Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java (revision 881159) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java (working copy) @@ -168,7 +168,7 @@ @Override public final TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new ArabicLetterTokenizer( reader ); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(matchVersion, result); // the order here is important: the stopword list is not normalized! result = new StopFilter( StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), result, stoptable ); @@ -198,7 +198,7 @@ if (streams == null) { streams = new SavedStreams(); streams.source = new ArabicLetterTokenizer(reader); - streams.result = new LowerCaseFilter(streams.source); + streams.result = new LowerCaseFilter(matchVersion, streams.source); // the order here is important: the stopword list is not normalized! streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), streams.result, stoptable); Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java (revision 881159) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java (working copy) @@ -199,7 +199,7 @@ @Override public final TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new StandardTokenizer( matchVersion, reader ); - result = new LowerCaseFilter( result ); + result = new LowerCaseFilter( matchVersion, result ); result = new StandardFilter( result ); result = new StopFilter( StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), result, stoptable ); @@ -227,7 +227,7 @@ if (streams == null) { streams = new SavedStreams(); streams.source = new StandardTokenizer(matchVersion, reader); - streams.result = new LowerCaseFilter(streams.source); + streams.result = new LowerCaseFilter(matchVersion, streams.source); streams.result = new StandardFilter(streams.result); streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), streams.result, stoptable); Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java (revision 881159) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java (working copy) @@ -181,7 +181,7 @@ public final TokenStream tokenStream( String fieldName, Reader reader ) { TokenStream result = new StandardTokenizer( matchVersion, reader ); result = new StandardFilter( result ); - result = new LowerCaseFilter( result ); + result = new LowerCaseFilter( matchVersion, result ); result = new StopFilter( StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), result, stoptable ); return result; @@ -207,7 +207,7 @@ streams = new SavedStreams(); streams.source = new StandardTokenizer(matchVersion, reader); streams.result = new StandardFilter(streams.source); - streams.result = new LowerCaseFilter(streams.result); + streams.result = new LowerCaseFilter(matchVersion, streams.result); streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), streams.result, stoptable); setPreviousTokenStream(streams); Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java (revision 881159) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java (working copy) @@ -200,7 +200,7 @@ public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new StandardTokenizer(matchVersion, reader); result = new StandardFilter(result); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(matchVersion, result); result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), result, stopSet); result = new GermanStemFilter(result, exclusionSet); @@ -234,7 +234,7 @@ streams = new SavedStreams(); streams.source = new StandardTokenizer(matchVersion, reader); streams.result = new StandardFilter(streams.source); - streams.result = new LowerCaseFilter(streams.result); + streams.result = new LowerCaseFilter(matchVersion, streams.result); streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), streams.result, stopSet); streams.result = new GermanStemFilter(streams.result, exclusionSet); Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (revision 881159) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (working copy) @@ -167,7 +167,7 @@ @Override public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new ArabicLetterTokenizer(reader); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(matchVersion, result); result = new ArabicNormalizationFilter(result); /* additional persian-specific normalization */ result = new PersianNormalizationFilter(result); @@ -201,7 +201,7 @@ if (streams == null) { streams = new SavedStreams(); streams.source = new ArabicLetterTokenizer(reader); - streams.result = new LowerCaseFilter(streams.source); + streams.result = new LowerCaseFilter(matchVersion, streams.source); streams.result = new ArabicNormalizationFilter(streams.result); /* additional persian-specific normalization */ streams.result = new PersianNormalizationFilter(streams.result); Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (revision 881159) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (working copy) @@ -215,7 +215,7 @@ result, stoptable); result = new FrenchStemFilter(result, excltable); // Convert to lowercase after stemming! - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(matchVersion, result); return result; } @@ -244,7 +244,7 @@ streams.result, stoptable); streams.result = new FrenchStemFilter(streams.result, excltable); // Convert to lowercase after stemming! - streams.result = new LowerCaseFilter(streams.result); + streams.result = new LowerCaseFilter(matchVersion, streams.result); setPreviousTokenStream(streams); } else { streams.source.reset(reader); Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (revision 881159) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (working copy) @@ -118,7 +118,7 @@ public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new RussianLetterTokenizer(reader); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(matchVersion, result); result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), result, stopSet); result = new RussianStemFilter(result); @@ -146,7 +146,7 @@ if (streams == null) { streams = new SavedStreams(); streams.source = new RussianLetterTokenizer(reader); - streams.result = new LowerCaseFilter(streams.source); + streams.result = new LowerCaseFilter(matchVersion, streams.source); streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), streams.result, stopSet); streams.result = new RussianStemFilter(streams.result); Index: contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java =================================================================== --- contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java (revision 881159) +++ contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerQPHelper.java (working copy) @@ -158,7 +158,7 @@ public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new TestFilter(result); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(Version.LUCENE_CURRENT, result); return result; } } @@ -228,7 +228,7 @@ public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new TestPosIncrementFilter(result); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(Version.LUCENE_CURRENT, result); return result; } } Index: contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java =================================================================== --- contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java (revision 881159) +++ contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestMultiAnalyzerWrapper.java (working copy) @@ -152,7 +152,7 @@ public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new TestFilter(result); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(Version.LUCENE_CURRENT, result); return result; } } @@ -222,7 +222,7 @@ public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new TestPosIncrementFilter(result); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(Version.LUCENE_CURRENT, result); return result; } } Index: contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java =================================================================== --- contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java (revision 881159) +++ contrib/snowball/src/java/org/apache/lucene/analysis/snowball/SnowballAnalyzer.java (working copy) @@ -60,7 +60,7 @@ public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new StandardTokenizer(matchVersion, reader); result = new StandardFilter(result); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(matchVersion, result); if (stopSet != null) result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), result, stopSet); @@ -91,7 +91,7 @@ streams = new SavedStreams(); streams.source = new StandardTokenizer(matchVersion, reader); streams.result = new StandardFilter(streams.source); - streams.result = new LowerCaseFilter(streams.result); + streams.result = new LowerCaseFilter(matchVersion, streams.result); if (stopSet != null) streams.result = new StopFilter(StopFilter.getEnablePositionIncrementsVersionDefault(matchVersion), streams.result, stopSet); Index: contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java =================================================================== --- contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (revision 881159) +++ contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (working copy) @@ -29,6 +29,7 @@ import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.WhitespaceTokenizer; import org.apache.lucene.analysis.BaseTokenStreamTestCase; +import org.apache.lucene.util.Version; public class TestSynonymTokenFilter extends BaseTokenStreamTestCase { File dataDir = new File(System.getProperty("dataDir", "./bin")); @@ -96,7 +97,7 @@ @Override public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream ts = new WhitespaceTokenizer(reader); - ts = new LowerCaseFilter(ts); + ts = new LowerCaseFilter(Version.LUCENE_CURRENT, ts); ts = new SynonymTokenFilter(ts, synonyms, maxSynonyms); return ts; } @@ -113,7 +114,7 @@ if (streams == null) { streams = new SavedStreams(); streams.source = new WhitespaceTokenizer(reader); - streams.result = new LowerCaseFilter(streams.source); + streams.result = new LowerCaseFilter(Version.LUCENE_CURRENT, streams.source); streams.result = new SynonymTokenFilter(streams.result, synonyms, maxSynonyms); setPreviousTokenStream(streams); } else { Index: src/java/org/apache/lucene/analysis/LowerCaseFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/LowerCaseFilter.java (revision 881505) +++ src/java/org/apache/lucene/analysis/LowerCaseFilter.java (working copy) @@ -20,20 +20,32 @@ import java.io.IOException; import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.util.Version; /** * Normalizes token text to lower case. */ public final class LowerCaseFilter extends TokenFilter { - public LowerCaseFilter(TokenStream in) { + public LowerCaseFilter(Version matchVersion, TokenStream in) { super(in); + this.matchVersion = matchVersion; termAtt = addAttribute(TermAttribute.class); } + + /** + * @deprecated Use {@link #LowerCaseFilter(TokenStream, Version)} instead. + */ + public LowerCaseFilter(TokenStream in) { + this(Version.LUCENE_30, in); + } private TermAttribute termAtt; + private final Version matchVersion; - @Override - public final boolean incrementToken() throws IOException { + /** + * @deprecated the old "broken unicode 4" behavior of lucene 3.0 + */ + private final boolean incrementTokenUnicode3() throws IOException { if (input.incrementToken()) { final char[] buffer = termAtt.termBuffer(); @@ -45,4 +57,28 @@ } else return false; } + + public final boolean incrementToken() throws IOException { + if (!matchVersion.onOrAfter(Version.LUCENE_31)) + return incrementTokenUnicode3(); + + if (input.incrementToken()) { + + final char[] buffer = termAtt.termBuffer(); + final int length = termAtt.termLength(); + for (int i = 0; i < length; i++) { + final char ch = buffer[i]; + if (Character.isHighSurrogate(ch)) { + final int codepoint = Character.toLowerCase(Character.codePointAt( + buffer, i, length)); + /* skip over the trail surrogate, if there is one */ + i += Character.toChars(codepoint, buffer, i) - 1; + } else { + buffer[i] = Character.toLowerCase(ch); + } + } + return true; + } else + return false; + } } Index: src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java =================================================================== --- src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (revision 881159) +++ src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (working copy) @@ -100,7 +100,7 @@ StandardTokenizer tokenStream = new StandardTokenizer(matchVersion, reader); tokenStream.setMaxTokenLength(maxTokenLength); TokenStream result = new StandardFilter(tokenStream); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(matchVersion, result); result = new StopFilter(enableStopPositionIncrements, result, stopSet); return result; } @@ -146,7 +146,8 @@ setPreviousTokenStream(streams); streams.tokenStream = new StandardTokenizer(matchVersion, reader); streams.filteredTokenStream = new StandardFilter(streams.tokenStream); - streams.filteredTokenStream = new LowerCaseFilter(streams.filteredTokenStream); + streams.filteredTokenStream = new LowerCaseFilter(matchVersion, + streams.filteredTokenStream); streams.filteredTokenStream = new StopFilter(enableStopPositionIncrements, streams.filteredTokenStream, stopSet); } else { Index: src/test/org/apache/lucene/analysis/TestAnalyzers.java =================================================================== --- src/test/org/apache/lucene/analysis/TestAnalyzers.java (revision 881159) +++ src/test/org/apache/lucene/analysis/TestAnalyzers.java (working copy) @@ -139,6 +139,65 @@ assertTrue(ts.incrementToken()); assertFalse(ts.incrementToken()); } + + private static class LowerCaseWhitespaceAnalyzer extends Analyzer { + + @Override + public TokenStream tokenStream(String fieldName, Reader reader) { + return new LowerCaseFilter(Version.LUCENE_CURRENT, new WhitespaceTokenizer(reader)); + } + + } + + /** + * @deprecated remove this when lucene 3.0 "broken unicode 4" support + * is no longer needed. + */ + private static class LowerCaseWhitespaceAnalyzerBWComp extends Analyzer { + + @Override + public TokenStream tokenStream(String fieldName, Reader reader) { + return new LowerCaseFilter(new WhitespaceTokenizer(reader)); + } + + } + + /** + * Test that LowercaseFilter handles entire unicode range correctly + */ + public void testLowerCaseFilter() throws IOException { + Analyzer a = new LowerCaseWhitespaceAnalyzer(); + // BMP + assertAnalyzesTo(a, "AbaCaDabA", new String[] { "abacadaba" }); + // supplementary + assertAnalyzesTo(a, "𐐖𐐖𐐖𐐖𐐖", new String[] { "𐐾𐐾𐐾𐐾𐐾" }); + // unpaired lead surrogate + assertAnalyzesTo(a, "AbaC\uD801AdaBa", + new String [] { "abac\uD801adaba" }); + // unpaired trail surrogate + assertAnalyzesTo(a, "AbaC\uDC16AdaBa", + new String [] { "abac\uDC16adaba" }); + } + + /** + * Test that LowercaseFilter only works on BMP for back compat, + * depending upon version + * @deprecated remove this test when lucene 3.0 "broken unicode 4" support + * is no longer needed. + */ + public void testLowerCaseFilterBWComp() throws IOException { + Analyzer a = new LowerCaseWhitespaceAnalyzerBWComp(); + // BMP + assertAnalyzesTo(a, "AbaCaDabA", new String[] { "abacadaba" }); + // supplementary, no-op + assertAnalyzesTo(a, "𐐖𐐖𐐖𐐖𐐖", new String[] { "𐐖𐐖𐐖𐐖𐐖" }); + // unpaired lead surrogate + assertAnalyzesTo(a, "AbaC\uD801AdaBa", + new String [] { "abac\uD801adaba" }); + // unpaired trail surrogate + assertAnalyzesTo(a, "AbaC\uDC16AdaBa", + new String [] { "abac\uDC16adaba" }); + } } class PayloadSetter extends TokenFilter { Index: src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java =================================================================== --- src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java (revision 881159) +++ src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java (working copy) @@ -146,7 +146,7 @@ assertEquals("there must be 2 times 'Dog' in the stream", 2, i); source1.reset(); - TokenStream lowerCasing = new LowerCaseFilter(source1); + TokenStream lowerCasing = new LowerCaseFilter(Version.LUCENE_CURRENT, source1); i = 0; termAtt = lowerCasing.getAttribute(TermAttribute.class); while (lowerCasing.incrementToken()) { Index: src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java =================================================================== --- src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java (revision 881159) +++ src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java (working copy) @@ -138,7 +138,7 @@ public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new TestFilter(result); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(Version.LUCENE_CURRENT, result); return result; } } @@ -206,7 +206,7 @@ public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new StandardTokenizer(Version.LUCENE_CURRENT, reader); result = new TestPosIncrementFilter(result); - result = new LowerCaseFilter(result); + result = new LowerCaseFilter(Version.LUCENE_CURRENT, result); return result; } }