Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java (working copy) @@ -30,6 +30,7 @@ import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.WordlistLoader; /** @@ -109,7 +110,7 @@ /** * Creates a TokenStream which tokenizes all the text in the provided Reader. * - * @return A TokenStream build from an ArabicTokenizer filtered with + * @return A TokenStream built from an ArabicTokenizer filtered with * StopFilter, LowerCaseFilter, ArabicNormalizationFilter and ArabicStemFilter. */ public final TokenStream tokenStream(String fieldName, Reader reader) { @@ -121,5 +122,35 @@ return result; } + + private class SavedStreams { + Tokenizer source; + TokenStream result; + }; + + /** + * Returns a (possibly reused) TokenStream which tokenizes all the text + * in the provided Reader. + * + * @return A TokenStream built from an ArabicTokenizer filtered with + * StopFilter, LowerCaseFilter, ArabicNormalizationFilter and + * ArabicStemFilter. + */ + public TokenStream reusableTokenStream(String fieldName, Reader reader) + throws IOException { + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.source = new ArabicLetterTokenizer(reader); + streams.result = new StopFilter(streams.source, stoptable); + streams.result = new LowerCaseFilter(streams.result); + streams.result = new ArabicNormalizationFilter(streams.result); + streams.result = new ArabicStemFilter(streams.result); + setPreviousTokenStream(streams); + } else { + streams.source.reset(reader); + } + return streams.result; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java (working copy) @@ -28,6 +28,7 @@ import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.WordlistLoader; import org.apache.lucene.analysis.standard.StandardFilter; import org.apache.lucene.analysis.standard.StandardTokenizer; @@ -125,8 +126,9 @@ /** * Creates a TokenStream which tokenizes all the text in the provided Reader. * - * @return A TokenStream build from a StandardTokenizer filtered with - * StandardFilter, StopFilter, GermanStemFilter and LowerCaseFilter. + * @return A TokenStream built from a StandardTokenizer filtered with + * LowerCaseFilter, StandardFilter, StopFilter, and + * BrazilianStemFilter. */ public final TokenStream tokenStream(String fieldName, Reader reader) { TokenStream result = new StandardTokenizer( reader ); @@ -136,5 +138,35 @@ result = new BrazilianStemFilter( result, excltable ); return result; } + + private class SavedStreams { + Tokenizer source; + TokenStream result; + }; + + /** + * Returns a (possibly reused) TokenStream which tokenizes all the text + * in the provided Reader. + * + * @return A TokenStream built from a StandardTokenizer filtered with + * LowerCaseFilter, StandardFilter, StopFilter, and + * BrazilianStemFilter. + */ + public TokenStream reusableTokenStream(String fieldName, Reader reader) + throws IOException { + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.source = new StandardTokenizer(reader); + streams.result = new LowerCaseFilter(streams.source); + streams.result = new StandardFilter(streams.result); + streams.result = new StopFilter(streams.result, stoptable); + streams.result = new BrazilianStemFilter(streams.result, excltable); + setPreviousTokenStream(streams); + } else { + streams.source.reset(reader); + } + return streams.result; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java (working copy) @@ -20,7 +20,9 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; +import java.io.IOException; import java.io.Reader; import java.util.Set; @@ -84,4 +86,30 @@ public final TokenStream tokenStream(String fieldName, Reader reader) { return new StopFilter(new CJKTokenizer(reader), stopTable); } + + private class SavedStreams { + Tokenizer source; + TokenStream result; + }; + + /** + * get (possibly reused) token stream from input + * + * @param fieldName lucene field name + * @param reader input reader + * @return TokenStream + */ + public final TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { + /* tokenStream() is final, no back compat issue */ + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.source = new CJKTokenizer(reader); + streams.result = new StopFilter(streams.source, stopTable); + setPreviousTokenStream(streams); + } else { + streams.source.reset(reader); + } + return streams.result; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java (working copy) @@ -277,5 +277,17 @@ // set final offset final int finalOffset = offset; this.offsetAtt.setOffset(finalOffset, finalOffset); - } + } + + public void reset() throws IOException { + super.reset(); + offset = bufferIndex = dataLen = 0; + preIsTokened = false; + tokenType = WORD_TYPE; + } + + public void reset(Reader reader) throws IOException { + super.reset(reader); + reset(); + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseAnalyzer.java (working copy) @@ -17,9 +17,11 @@ * limitations under the License. */ +import java.io.IOException; import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; /** * Title: ChineseAnalyzer @@ -47,4 +49,31 @@ result = new ChineseFilter(result); return result; } + + private class SavedStreams { + Tokenizer source; + TokenStream result; + }; + + /** + * Returns a (possibly reused) TokenStream which tokenizes all the text in the + * provided Reader. + * + * @return A TokenStream build from a ChineseTokenizer filtered with + * ChineseFilter. + */ + public final TokenStream reusableTokenStream(String fieldName, Reader reader) + throws IOException { + /* tokenStream() is final, no back compat issue */ + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.source = new ChineseTokenizer(reader); + streams.result = new ChineseFilter(streams.source); + setPreviousTokenStream(streams); + } else { + streams.source.reset(reader); + } + return streams.result; + } } \ No newline at end of file Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/cn/ChineseTokenizer.java (working copy) @@ -145,5 +145,15 @@ // set final offset final int finalOffset = offset; this.offsetAtt.setOffset(finalOffset, finalOffset); - } + } + + public void reset() throws IOException { + super.reset(); + offset = bufferIndex = dataLen = 0; + } + + public void reset(Reader input) throws IOException { + super.reset(input); + reset(); + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java (working copy) @@ -215,4 +215,9 @@ } protected abstract void decomposeInternal(final Token token); + + public void reset() throws IOException { + super.reset(); + tokens.clear(); + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.WordlistLoader; import org.apache.lucene.analysis.standard.StandardFilter; import org.apache.lucene.analysis.standard.StandardTokenizer; @@ -126,7 +127,7 @@ /** * Creates a TokenStream which tokenizes all the text in the provided Reader. * - * @return A TokenStream build from a StandardTokenizer filtered with + * @return A TokenStream built from a StandardTokenizer filtered with * StandardFilter, LowerCaseFilter, and StopFilter */ public final TokenStream tokenStream( String fieldName, Reader reader ) { @@ -136,5 +137,33 @@ result = new StopFilter( result, stoptable ); return result; } + + private class SavedStreams { + Tokenizer source; + TokenStream result; + }; + + /** + * Returns a (possibly reused) TokenStream which tokenizes all the text in + * the provided Reader. + * + * @return A TokenStream built from a StandardTokenizer filtered with + * StandardFilter, LowerCaseFilter, and StopFilter + */ + public TokenStream reusableTokenStream(String fieldName, Reader reader) + throws IOException { + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.source = new StandardTokenizer(reader); + streams.result = new StandardFilter(streams.source); + streams.result = new LowerCaseFilter(streams.result); + streams.result = new StopFilter(streams.result, stoptable); + setPreviousTokenStream(streams); + } else { + streams.source.reset(reader); + } + return streams.result; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java (working copy) @@ -29,6 +29,7 @@ import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.WordlistLoader; import org.apache.lucene.analysis.standard.StandardFilter; import org.apache.lucene.analysis.standard.StandardTokenizer; @@ -79,6 +80,7 @@ */ public GermanAnalyzer() { stopSet = StopFilter.makeStopSet(GERMAN_STOP_WORDS); + setOverridesTokenStreamMethod(GermanAnalyzer.class); } /** @@ -86,6 +88,7 @@ */ public GermanAnalyzer(String[] stopwords) { stopSet = StopFilter.makeStopSet(stopwords); + setOverridesTokenStreamMethod(GermanAnalyzer.class); } /** @@ -93,6 +96,7 @@ */ public GermanAnalyzer(Map stopwords) { stopSet = new HashSet(stopwords.keySet()); + setOverridesTokenStreamMethod(GermanAnalyzer.class); } /** @@ -100,6 +104,7 @@ */ public GermanAnalyzer(File stopwords) throws IOException { stopSet = WordlistLoader.getWordSet(stopwords); + setOverridesTokenStreamMethod(GermanAnalyzer.class); } /** @@ -126,7 +131,7 @@ /** * Creates a TokenStream which tokenizes all the text in the provided Reader. * - * @return A TokenStream build from a StandardTokenizer filtered with + * @return A TokenStream built from a StandardTokenizer filtered with * StandardFilter, LowerCaseFilter, StopFilter, GermanStemFilter */ public TokenStream tokenStream(String fieldName, Reader reader) { @@ -137,4 +142,39 @@ result = new GermanStemFilter(result, exclusionSet); return result; } + + private class SavedStreams { + Tokenizer source; + TokenStream result; + }; + + /** + * Returns a (possibly reused) TokenStream which tokenizes all the text + * in the provided Reader. + * + * @return A TokenStream built from a StandardTokenizer filtered with + * StandardFilter, LowerCaseFilter, StopFilter, GermanStemFilter + */ + public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { + if (overridesTokenStreamMethod) { + // LUCENE-1678: force fallback to tokenStream() if we + // have been subclassed and that subclass overrides + // tokenStream but not reusableTokenStream + return tokenStream(fieldName, reader); + } + + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.source = new StandardTokenizer(reader); + streams.result = new StandardFilter(streams.source); + streams.result = new LowerCaseFilter(streams.result); + streams.result = new StopFilter(streams.result, stopSet); + streams.result = new GermanStemFilter(streams.result, exclusionSet); + setPreviousTokenStream(streams); + } else { + streams.source.reset(reader); + } + return streams.result; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java (working copy) @@ -20,8 +20,10 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.standard.StandardTokenizer; +import java.io.IOException; import java.io.Reader; import java.util.HashSet; import java.util.Map; @@ -209,7 +211,7 @@ /** * Creates a TokenStream which tokenizes all the text in the provided Reader. * - * @return A TokenStream build from a StandardTokenizer filtered with + * @return A TokenStream built from a StandardTokenizer filtered with * GreekLowerCaseFilter and StopFilter */ public TokenStream tokenStream(String fieldName, Reader reader) @@ -219,4 +221,31 @@ result = new StopFilter(result, stopSet); return result; } + + private class SavedStreams { + Tokenizer source; + TokenStream result; + }; + + /** + * Returns a (possibly reused) TokenStream which tokenizes all the text + * in the provided Reader. + * + * @return A TokenStream built from a StandardTokenizer filtered with + * GreekLowerCaseFilter and StopFilter + */ + public TokenStream reusableTokenStream(String fieldName, Reader reader) + throws IOException { + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.source = new StandardTokenizer(reader); + streams.result = new GreekLowerCaseFilter(streams.source, charset); + streams.result = new StopFilter(streams.result, stopSet); + setPreviousTokenStream(streams); + } else { + streams.source.reset(reader); + } + return streams.result; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.WordlistLoader; import org.apache.lucene.analysis.standard.StandardFilter; import org.apache.lucene.analysis.standard.StandardTokenizer; @@ -128,7 +129,7 @@ /** * Creates a TokenStream which tokenizes all the text in the provided Reader. * - * @return A TokenStream build from a StandardTokenizer filtered with + * @return A TokenStream built from a StandardTokenizer filtered with * StandardFilter, StopFilter, FrenchStemFilter and LowerCaseFilter */ public final TokenStream tokenStream(String fieldName, Reader reader) { @@ -144,5 +145,35 @@ result = new LowerCaseFilter(result); return result; } + + private class SavedStreams { + Tokenizer source; + TokenStream result; + }; + + /** + * Returns a (possibly reused) TokenStream which tokenizes all the text + * in the provided Reader. + * + * @return A TokenStream built from a StandardTokenizer filtered with + * StandardFilter, StopFilter, FrenchStemFilter and LowerCaseFilter + */ + public TokenStream reusableTokenStream(String fieldName, Reader reader) + throws IOException { + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.source = new StandardTokenizer(reader); + streams.result = new StandardFilter(streams.source); + streams.result = new StopFilter(streams.result, stoptable); + streams.result = new FrenchStemFilter(streams.result, excltable); + // Convert to lowercase after stemming! + streams.result = new LowerCaseFilter(streams.result); + setPreviousTokenStream(streams); + } else { + streams.source.reset(reader); + } + return streams.result; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java (working copy) @@ -161,4 +161,9 @@ public final Token next() throws java.io.IOException { return super.next(); } + + public void reset() throws IOException { + super.reset(); + curTermBuffer = null; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (working copy) @@ -169,4 +169,14 @@ public final Token next() throws java.io.IOException { return super.next(); } + + public void reset(Reader input) throws IOException { + super.reset(input); + reset(); + } + + public void reset() throws IOException { + super.reset(); + started = false; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (working copy) @@ -109,4 +109,9 @@ public final Token next() throws java.io.IOException { return super.next(); } + + public void reset() throws IOException { + super.reset(); + curTermBuffer = null; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (working copy) @@ -114,4 +114,14 @@ public final Token next() throws java.io.IOException { return super.next(); } + + public void reset(Reader input) throws IOException { + super.reset(input); + reset(); + } + + public void reset() throws IOException { + super.reset(); + started = false; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.standard.StandardFilter; import org.apache.lucene.analysis.standard.StandardTokenizer; @@ -78,6 +79,7 @@ * */ public DutchAnalyzer() { + setOverridesTokenStreamMethod(DutchAnalyzer.class); stoptable = StopFilter.makeStopSet(DUTCH_STOP_WORDS); stemdict.put("fiets", "fiets"); //otherwise fiet stemdict.put("bromfiets", "bromfiets"); //otherwise bromfiet @@ -91,6 +93,7 @@ * @param stopwords */ public DutchAnalyzer(String[] stopwords) { + setOverridesTokenStreamMethod(DutchAnalyzer.class); stoptable = StopFilter.makeStopSet(stopwords); } @@ -100,6 +103,7 @@ * @param stopwords */ public DutchAnalyzer(HashSet stopwords) { + setOverridesTokenStreamMethod(DutchAnalyzer.class); stoptable = stopwords; } @@ -109,6 +113,7 @@ * @param stopwords */ public DutchAnalyzer(File stopwords) { + setOverridesTokenStreamMethod(DutchAnalyzer.class); try { stoptable = org.apache.lucene.analysis.WordlistLoader.getWordSet(stopwords); } catch (IOException e) { @@ -162,7 +167,7 @@ /** * Creates a TokenStream which tokenizes all the text in the provided TextReader. * - * @return A TokenStream build from a StandardTokenizer filtered with StandardFilter, + * @return A TokenStream built from a StandardTokenizer filtered with StandardFilter, * StopFilter, DutchStemFilter */ public TokenStream tokenStream(String fieldName, Reader reader) { @@ -172,4 +177,39 @@ result = new DutchStemFilter(result, excltable, stemdict); return result; } + + private class SavedStreams { + Tokenizer source; + TokenStream result; + }; + + /** + * Returns a (possibly reused) TokenStream which tokenizes all the text + * in the provided Reader. + * + * @return A TokenStream built from a StandardTokenizer filtered with + * StandardFilter, StopFilter, DutchStemFilter + */ + public TokenStream reusableTokenStream(String fieldName, Reader reader) + throws IOException { + if (overridesTokenStreamMethod) { + // LUCENE-1678: force fallback to tokenStream() if we + // have been subclassed and that subclass overrides + // tokenStream but not reusableTokenStream + return tokenStream(fieldName, reader); + } + + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.source = new StandardTokenizer(reader); + streams.result = new StandardFilter(streams.source); + streams.result = new StopFilter(streams.result, stoptable); + streams.result = new DutchStemFilter(streams.result, excltable); + setPreviousTokenStream(streams); + } else { + streams.source.reset(reader); + } + return streams.result; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java (working copy) @@ -17,6 +17,7 @@ * limitations under the License. */ +import java.io.IOException; import java.io.Reader; import java.util.HashSet; import java.util.Map; @@ -25,6 +26,7 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; /** * Analyzer for Russian language. Supports an external list of stopwords (words that @@ -246,7 +248,7 @@ /** * Creates a TokenStream which tokenizes all the text in the provided Reader. * - * @return A TokenStream build from a RussianLetterTokenizer filtered with + * @return A TokenStream built from a RussianLetterTokenizer filtered with * RussianLowerCaseFilter, StopFilter, and RussianStemFilter */ public TokenStream tokenStream(String fieldName, Reader reader) @@ -257,4 +259,32 @@ result = new RussianStemFilter(result, charset); return result; } + + private class SavedStreams { + Tokenizer source; + TokenStream result; + }; + + /** + * Returns a (possibly reused) TokenStream which tokenizes all the text + * in the provided Reader. + * + * @return A TokenStream built from a RussianLetterTokenizer filtered with + * RussianLowerCaseFilter, StopFilter, and RussianStemFilter + */ + public TokenStream reusableTokenStream(String fieldName, Reader reader) + throws IOException { + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.source = new RussianLetterTokenizer(reader, charset); + streams.result = new RussianLowerCaseFilter(streams.source, charset); + streams.result = new StopFilter(streams.result, stopSet); + streams.result = new RussianStemFilter(streams.result, charset); + setPreviousTokenStream(streams); + } else { + streams.source.reset(reader); + } + return streams.result; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleAnalyzerWrapper.java (working copy) @@ -17,6 +17,7 @@ * limitations under the License. */ +import java.io.IOException; import java.io.Reader; import org.apache.lucene.analysis.Analyzer; @@ -36,6 +37,7 @@ public ShingleAnalyzerWrapper(Analyzer defaultAnalyzer) { super(); this.defaultAnalyzer = defaultAnalyzer; + setOverridesTokenStreamMethod(ShingleAnalyzerWrapper.class); } public ShingleAnalyzerWrapper(Analyzer defaultAnalyzer, int maxShingleSize) { @@ -49,6 +51,7 @@ public ShingleAnalyzerWrapper() { super(); this.defaultAnalyzer = new StandardAnalyzer(); + setOverridesTokenStreamMethod(ShingleAnalyzerWrapper.class); } public ShingleAnalyzerWrapper(int nGramSize) { @@ -90,10 +93,50 @@ } public TokenStream tokenStream(String fieldName, Reader reader) { - ShingleFilter filter = new ShingleFilter(defaultAnalyzer.tokenStream( - fieldName, reader)); + TokenStream wrapped; + try { + wrapped = defaultAnalyzer.reusableTokenStream(fieldName, reader); + } catch (IOException e) { + wrapped = defaultAnalyzer.tokenStream(fieldName, reader); + } + ShingleFilter filter = new ShingleFilter(wrapped); filter.setMaxShingleSize(maxShingleSize); filter.setOutputUnigrams(outputUnigrams); return filter; } + + private class SavedStreams { + TokenStream wrapped; + ShingleFilter shingle; + }; + + public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { + if (overridesTokenStreamMethod) { + // LUCENE-1678: force fallback to tokenStream() if we + // have been subclassed and that subclass overrides + // tokenStream but not reusableTokenStream + return tokenStream(fieldName, reader); + } + + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.wrapped = defaultAnalyzer.reusableTokenStream(fieldName, reader); + streams.shingle = new ShingleFilter(streams.wrapped); + setPreviousTokenStream(streams); + } else { + TokenStream result = defaultAnalyzer.reusableTokenStream(fieldName, reader); + if (result == streams.wrapped) { + /* the wrapped analyzer reused the stream */ + streams.shingle.reset(); + } else { + /* the wrapped analyzer did not, create a new shingle around the new one */ + streams.wrapped = result; + streams.shingle = new ShingleFilter(streams.wrapped); + } + } + streams.shingle.setMaxShingleSize(maxShingleSize); + streams.shingle.setOutputUnigrams(outputUnigrams); + return streams.shingle; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java (working copy) @@ -336,4 +336,14 @@ public final Token next() throws java.io.IOException { return super.next(); } + + public void reset() throws IOException { + super.reset(); + nextToken = null; + shingleBufferPosition = 0; + shingleBuf.clear(); + numFillerTokensToInsert = 0; + currentToken = null; + hasCurrentToken = false; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java (working copy) @@ -16,11 +16,13 @@ * limitations under the License. */ +import java.io.IOException; import java.io.Reader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.StopAnalyzer; import org.apache.lucene.analysis.StopFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.standard.StandardFilter; import org.apache.lucene.analysis.standard.StandardTokenizer; @@ -29,6 +31,11 @@ * @version 0.2 */ public class ThaiAnalyzer extends Analyzer { + + public ThaiAnalyzer() { + setOverridesTokenStreamMethod(ThaiAnalyzer.class); + } + public TokenStream tokenStream(String fieldName, Reader reader) { TokenStream ts = new StandardTokenizer(reader); ts = new StandardFilter(ts); @@ -36,4 +43,32 @@ ts = new StopFilter(ts, StopAnalyzer.ENGLISH_STOP_WORDS_SET); return ts; } + + private class SavedStreams { + Tokenizer source; + TokenStream result; + }; + + public TokenStream reusableTokenStream(String fieldName, Reader reader) throws IOException { + if (overridesTokenStreamMethod) { + // LUCENE-1678: force fallback to tokenStream() if we + // have been subclassed and that subclass overrides + // tokenStream but not reusableTokenStream + return tokenStream(fieldName, reader); + } + + SavedStreams streams = (SavedStreams) getPreviousTokenStream(); + if (streams == null) { + streams = new SavedStreams(); + streams.source = new StandardTokenizer(reader); + streams.result = new StandardFilter(streams.source); + streams.result = new ThaiWordFilter(streams.result); + streams.result = new StopFilter(streams.result, StopAnalyzer.ENGLISH_STOP_WORDS_SET); + setPreviousTokenStream(streams); + } else { + streams.source.reset(reader); + streams.result.reset(); // reset the ThaiWordFilter's state + } + return streams.result; + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (revision 803465) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (working copy) @@ -93,4 +93,9 @@ public final Token next() throws java.io.IOException { return super.next(); } + + public void reset() throws IOException { + super.reset(); + thaiState = null; + } } Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java (working copy) @@ -57,6 +57,15 @@ assertAnalyzesTo(a, "ما ملكت أيمانكم", new String[] { "ملكت", "ايمانكم"}); assertAnalyzesTo(a, "الذين ملكت أيمانكم", new String[] { "ملكت", "ايمانكم" }); // stopwords } + + /** + * Simple tests to show things are getting reset correctly, etc. + */ + public void testReusableTokenStream() throws Exception { + ArabicAnalyzer a = new ArabicAnalyzer(); + assertAnalyzesToReuse(a, "كبير", new String[] { "كبير" }); + assertAnalyzesToReuse(a, "كبيرة", new String[] { "كبير" }); // feminine marker + } /** * Non-arabic text gets treated in a similar way as SimpleAnalyzer. @@ -80,5 +89,18 @@ assertFalse(ts.incrementToken()); ts.close(); } + + private void assertAnalyzesToReuse(Analyzer a, String input, String[] output) + throws Exception { + TokenStream ts = a.reusableTokenStream("dummy", new StringReader(input)); + TermAttribute termAtt = (TermAttribute) ts + .getAttribute(TermAttribute.class); + for (int i = 0; i < output.length; i++) { + assertTrue(ts.incrementToken()); + assertEquals(output[i], termAtt.term()); + } + + assertFalse(ts.incrementToken()); + } } Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java (revision 803465) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java (working copy) @@ -117,6 +117,14 @@ check("quinzena", "quinzen"); check("quiosque", "quiosqu"); } + + public void testReusableTokenStream() throws Exception { + Analyzer a = new BrazilianAnalyzer(); + checkReuse(a, "boa", "boa"); + checkReuse(a, "boainain", "boainain"); + checkReuse(a, "boas", "boas"); + checkReuse(a, "bôas", "boas"); // removes diacritic: different from snowball portugese + } private void check(final String input, final String expected) throws IOException { @@ -128,5 +136,13 @@ assertFalse(stream.incrementToken()); stream.close(); } + + private void checkReuse(Analyzer analyzer, final String input, final String expected) throws IOException { + TokenStream stream = analyzer.reusableTokenStream("dummy", new StringReader(input)); + TermAttribute text = (TermAttribute) stream.getAttribute(TermAttribute.class); + assertTrue(stream.incrementToken()); + assertEquals(expected, text.term()); + assertFalse(stream.incrementToken()); + } } \ No newline at end of file Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/cjk/TestCJKTokenizer.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/cjk/TestCJKTokenizer.java (revision 803465) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/cjk/TestCJKTokenizer.java (working copy) @@ -22,6 +22,8 @@ import junit.framework.TestCase; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; @@ -60,6 +62,21 @@ assertFalse(tokenizer.incrementToken()); } + public void checkCJKTokenReusable(final Analyzer a, final String str, final TestToken[] out_tokens) throws IOException { + TokenStream ts = a.reusableTokenStream("dummy", new StringReader(str)); + TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class); + OffsetAttribute offsetAtt = (OffsetAttribute) ts.getAttribute(OffsetAttribute.class); + TypeAttribute typeAtt = (TypeAttribute) ts.getAttribute(TypeAttribute.class); + for (int i = 0; i < out_tokens.length; i++) { + assertTrue(ts.incrementToken()); + assertEquals(termAtt.term(), out_tokens[i].termText); + assertEquals(offsetAtt.startOffset(), out_tokens[i].start); + assertEquals(offsetAtt.endOffset(), out_tokens[i].end); + assertEquals(typeAtt.type(), out_tokens[i].type); + } + assertFalse(ts.incrementToken()); + } + public void testJa1() throws IOException { String str = "\u4e00\u4e8c\u4e09\u56db\u4e94\u516d\u4e03\u516b\u4e5d\u5341"; @@ -151,4 +168,38 @@ }; checkCJKToken(str, out_tokens); } + + public void testReusableTokenStream() throws Exception { + Analyzer analyzer = new CJKAnalyzer(); + String str = "\u3042\u3044\u3046\u3048\u304aabc\u304b\u304d\u304f\u3051\u3053"; + + TestToken[] out_tokens = { + newToken("\u3042\u3044", 0, 2, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u3044\u3046", 1, 3, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u3046\u3048", 2, 4, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u3048\u304a", 3, 5, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("abc", 5, 8, CJKTokenizer.SINGLE_TOKEN_TYPE), + newToken("\u304b\u304d", 8, 10, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u304d\u304f", 9, 11, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u304f\u3051", 10,12, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u3051\u3053", 11,13, CJKTokenizer.DOUBLE_TOKEN_TYPE) + }; + checkCJKTokenReusable(analyzer, str, out_tokens); + + str = "\u3042\u3044\u3046\u3048\u304aab\u3093c\u304b\u304d\u304f\u3051 \u3053"; + TestToken[] out_tokens2 = { + newToken("\u3042\u3044", 0, 2, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u3044\u3046", 1, 3, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u3046\u3048", 2, 4, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u3048\u304a", 3, 5, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("ab", 5, 7, CJKTokenizer.SINGLE_TOKEN_TYPE), + newToken("\u3093", 7, 8, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("c", 8, 9, CJKTokenizer.SINGLE_TOKEN_TYPE), + newToken("\u304b\u304d", 9, 11, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u304d\u304f", 10, 12, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u304f\u3051", 11,13, CJKTokenizer.DOUBLE_TOKEN_TYPE), + newToken("\u3053", 14,15, CJKTokenizer.DOUBLE_TOKEN_TYPE) + }; + checkCJKTokenReusable(analyzer, str, out_tokens2); + } } Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java (revision 803465) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/cn/TestChineseTokenizer.java (working copy) @@ -22,7 +22,10 @@ import junit.framework.TestCase; +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; public class TestChineseTokenizer extends TestCase @@ -42,4 +45,32 @@ correctEndOffset++; } } + + public void testReusableTokenStream() throws Exception + { + Analyzer a = new ChineseAnalyzer(); + assertAnalyzesToReuse(a, "中华人民共和国", + new String[] { "中", "华", "人", "民", "共", "和", "国" }, + new int[] { 0, 1, 2, 3, 4, 5, 6 }, + new int[] { 1, 2, 3, 4, 5, 6, 7 }); + assertAnalyzesToReuse(a, "北京市", + new String[] { "北", "京", "市" }, + new int[] { 0, 1, 2 }, + new int[] { 1, 2, 3 }); + } + + private void assertAnalyzesToReuse(Analyzer a, String input, String[] output, + int startOffsets[], int endOffsets[]) + throws Exception { + TokenStream ts = a.reusableTokenStream("dummy", new StringReader(input)); + TermAttribute termAtt = (TermAttribute) ts + .getAttribute(TermAttribute.class); + + for (int i = 0; i < output.length; i++) { + assertTrue(ts.incrementToken()); + assertEquals(output[i], termAtt.term()); + } + + assertFalse(ts.incrementToken()); + } } Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java (revision 803465) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java (working copy) @@ -36,6 +36,12 @@ public void testStopWord() throws Exception { assertAnalyzesTo(new CzechAnalyzer(), "Pokud mluvime o volnem", new String[] { "mluvime", "volnem" }); } + + public void testReusableTokenStream() throws Exception { + Analyzer analyzer = new CzechAnalyzer(); + assertAnalyzesToReuse(analyzer, "Pokud mluvime o volnem", new String[] { "mluvime", "volnem" }); + assertAnalyzesToReuse(analyzer, "Česká Republika", new String[] { "česká", "republika" }); + } private void assertAnalyzesTo(Analyzer a, String input, String[] output) throws Exception { TokenStream ts = a.tokenStream("dummy", new StringReader(input)); @@ -47,4 +53,14 @@ assertFalse(ts.incrementToken()); ts.close(); } + + private void assertAnalyzesToReuse(Analyzer a, String input, String[] output) throws Exception { + TokenStream ts = a.reusableTokenStream("dummy", new StringReader(input)); + TermAttribute text = (TermAttribute) ts.getAttribute(TermAttribute.class); + for (int i=0; i