Index: lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java =================================================================== --- lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java (revision 1441939) +++ lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestFactories.java (working copy) @@ -146,7 +146,7 @@ // some silly classes just so we can use checkRandomData private TokenizerFactory assertingTokenizer = new TokenizerFactory() { @Override - public Tokenizer create(Reader input) { + public MockTokenizer create(Reader input) { return new MockTokenizer(input); } }; Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/TokenizerFactory.java (working copy) @@ -18,6 +18,7 @@ */ import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.util.AttributeSource.AttributeFactory; import java.io.Reader; import java.util.Set; @@ -63,4 +64,10 @@ /** Creates a TokenStream of the specified input */ public abstract Tokenizer create(Reader input); + + /** Creates a TokenStream of the specified input using the given AttributeFactory */ + public Tokenizer create(AttributeFactory factory, Reader input) { + throw new UnsupportedOperationException(); + } + } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/util/CharTokenizer.java (working copy) @@ -51,22 +51,6 @@ * * @param matchVersion * Lucene version to match - * @param source - * the attribute source to use for this {@link Tokenizer} - * @param input - * the input to split up into tokens - */ - public CharTokenizer(Version matchVersion, AttributeSource source, - Reader input) { - super(source, input); - charUtils = CharacterUtils.getInstance(matchVersion); - } - - /** - * Creates a new {@link CharTokenizer} instance - * - * @param matchVersion - * Lucene version to match * @param factory * the attribute factory to use for this {@link Tokenizer} * @param input Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java (working copy) @@ -49,14 +49,6 @@ termAtt.resizeBuffer(bufferSize); } - public KeywordTokenizer(AttributeSource source, Reader input, int bufferSize) { - super(source, input); - if (bufferSize <= 0) { - throw new IllegalArgumentException("bufferSize must be > 0"); - } - termAtt.resizeBuffer(bufferSize); - } - public KeywordTokenizer(AttributeFactory factory, Reader input, int bufferSize) { super(factory, input); if (bufferSize <= 0) { Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizerFactory.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.analysis.core.LetterTokenizer; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.util.AttributeSource.AttributeFactory; import java.io.Reader; import java.util.Map; @@ -45,4 +46,10 @@ public LetterTokenizer create(Reader input) { return new LetterTokenizer(luceneMatchVersion, input); } + + @Override + public LetterTokenizer create(AttributeFactory factory, Reader input) { + return new LetterTokenizer(luceneMatchVersion, factory, input); + } + } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizerFactory.java (working copy) @@ -21,6 +21,7 @@ import org.apache.lucene.analysis.util.AbstractAnalysisFactory; import org.apache.lucene.analysis.util.MultiTermAwareComponent; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.util.AttributeSource.AttributeFactory; import java.io.Reader; import java.util.Map; @@ -48,10 +49,16 @@ } @Override + public LowerCaseTokenizer create(AttributeFactory factory, Reader input) { + return new LowerCaseTokenizer(luceneMatchVersion, factory, input); + } + + @Override public AbstractAnalysisFactory getMultiTermComponent() { LowerCaseFilterFactory filt = new LowerCaseFilterFactory(); filt.setLuceneMatchVersion(luceneMatchVersion); filt.init(args); return filt; } + } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LowerCaseTokenizer.java (working copy) @@ -60,20 +60,6 @@ super(matchVersion, in); } - /** - * Construct a new LowerCaseTokenizer using a given {@link AttributeSource}. - * - * @param matchVersion - * Lucene version to match See {@link above} - * @param source - * the attribute source to use for this {@link Tokenizer} - * @param in - * the input to split up into tokens - */ - public LowerCaseTokenizer(Version matchVersion, AttributeSource source, Reader in) { - super(matchVersion, source, in); - } - /** * Construct a new LowerCaseTokenizer using a given * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}. Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizerFactory.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.util.AttributeSource.AttributeFactory; import java.io.Reader; import java.util.Map; @@ -34,6 +35,7 @@ * */ public class WhitespaceTokenizerFactory extends TokenizerFactory { + @Override public void init(Map args) { super.init(args); @@ -44,4 +46,10 @@ public WhitespaceTokenizer create(Reader input) { return new WhitespaceTokenizer(luceneMatchVersion,input); } + + @Override + public WhitespaceTokenizer create(AttributeFactory factory, Reader input) { + return new WhitespaceTokenizer(luceneMatchVersion, factory, input); + } + } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/WhitespaceTokenizer.java (working copy) @@ -51,20 +51,6 @@ } /** - * Construct a new WhitespaceTokenizer using a given {@link AttributeSource}. - * - * @param matchVersion - * Lucene version to match See {@link above} - * @param source - * the attribute source to use for this {@link Tokenizer} - * @param in - * the input to split up into tokens - */ - public WhitespaceTokenizer(Version matchVersion, AttributeSource source, Reader in) { - super(matchVersion, source, in); - } - - /** * Construct a new WhitespaceTokenizer using a given * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}. * Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizer.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/LetterTokenizer.java (working copy) @@ -59,20 +59,6 @@ } /** - * Construct a new LetterTokenizer using a given {@link AttributeSource}. - * - * @param matchVersion - * Lucene version to match See {@link above} - * @param source - * the attribute source to use for this {@link Tokenizer} - * @param in - * the input to split up into tokens - */ - public LetterTokenizer(Version matchVersion, AttributeSource source, Reader in) { - super(matchVersion, source, in); - } - - /** * Construct a new LetterTokenizer using a given * {@link org.apache.lucene.util.AttributeSource.AttributeFactory}. * Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizerFactory.java (working copy) @@ -17,8 +17,10 @@ * limitations under the License. */ +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.core.KeywordTokenizer; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.util.AttributeSource.AttributeFactory; import java.io.Reader; @@ -33,8 +35,15 @@ * */ public class KeywordTokenizerFactory extends TokenizerFactory { + @Override public KeywordTokenizer create(Reader input) { return new KeywordTokenizer(input); } + + @Override + public Tokenizer create(AttributeFactory factory, Reader input) { + return new KeywordTokenizer(factory, input, KeywordTokenizer.DEFAULT_BUFFER_SIZE); + } + } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizerFactory.java (working copy) @@ -19,6 +19,7 @@ import org.apache.lucene.analysis.ngram.EdgeNGramTokenizer; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.util.AttributeSource.AttributeFactory; import java.io.Reader; import java.util.Map; @@ -59,4 +60,10 @@ public EdgeNGramTokenizer create(Reader input) { return new EdgeNGramTokenizer(input, side, minGramSize, maxGramSize); } + + @Override + public EdgeNGramTokenizer create(AttributeFactory factory, Reader input) { + return new EdgeNGramTokenizer(factory, input, side, minGramSize, maxGramSize); + } + } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenizer.java (working copy) @@ -94,20 +94,6 @@ /** * Creates EdgeNGramTokenizer that can generate n-grams in the sizes of the given range - * - * @param source {@link AttributeSource} to use - * @param input {@link Reader} holding the input to be tokenized - * @param side the {@link Side} from which to chop off an n-gram - * @param minGram the smallest n-gram to generate - * @param maxGram the largest n-gram to generate - */ - public EdgeNGramTokenizer(AttributeSource source, Reader input, Side side, int minGram, int maxGram) { - super(source, input); - init(side, minGram, maxGram); - } - - /** - * Creates EdgeNGramTokenizer that can generate n-grams in the sizes of the given range * * @param factory {@link org.apache.lucene.util.AttributeSource.AttributeFactory} to use * @param input {@link Reader} holding the input to be tokenized @@ -134,19 +120,6 @@ /** * Creates EdgeNGramTokenizer that can generate n-grams in the sizes of the given range - * - * @param source {@link AttributeSource} to use - * @param input {@link Reader} holding the input to be tokenized - * @param sideLabel the name of the {@link Side} from which to chop off an n-gram - * @param minGram the smallest n-gram to generate - * @param maxGram the largest n-gram to generate - */ - public EdgeNGramTokenizer(AttributeSource source, Reader input, String sideLabel, int minGram, int maxGram) { - this(source, input, Side.getSide(sideLabel), minGram, maxGram); - } - - /** - * Creates EdgeNGramTokenizer that can generate n-grams in the sizes of the given range * * @param factory {@link org.apache.lucene.util.AttributeSource.AttributeFactory} to use * @param input {@link Reader} holding the input to be tokenized Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizerFactory.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.ngram.NGramTokenizer; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.util.AttributeSource.AttributeFactory; import java.io.Reader; import java.util.Map; @@ -54,4 +55,10 @@ public NGramTokenizer create(Reader input) { return new NGramTokenizer(input, minGramSize, maxGramSize); } + + @Override + public NGramTokenizer create(AttributeFactory factory, Reader input) { + return new NGramTokenizer(factory, input, minGramSize, maxGramSize); + } + } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenizer.java (working copy) @@ -56,18 +56,6 @@ /** * Creates NGramTokenizer with given min and max n-grams. - * @param source {@link AttributeSource} to use - * @param input {@link Reader} holding the input to be tokenized - * @param minGram the smallest n-gram to generate - * @param maxGram the largest n-gram to generate - */ - public NGramTokenizer(AttributeSource source, Reader input, int minGram, int maxGram) { - super(source, input); - init(minGram, maxGram); - } - - /** - * Creates NGramTokenizer with given min and max n-grams. * @param factory {@link org.apache.lucene.util.AttributeSource.AttributeFactory} to use * @param input {@link Reader} holding the input to be tokenized * @param minGram the smallest n-gram to generate Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizerFactory.java (working copy) @@ -91,7 +91,7 @@ * Split the input using configured pattern */ @Override - public Tokenizer create(final Reader in) { + public PatternTokenizer create(final Reader in) { try { return new PatternTokenizer(in, pattern, group); } catch( IOException ex ) { Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java (working copy) @@ -102,14 +102,6 @@ } /** - * Creates a new UAX29URLEmailTokenizer with a given {@link AttributeSource}. - */ - public UAX29URLEmailTokenizer(Version matchVersion, AttributeSource source, Reader input) { - super(source, input); - this.scanner = getScannerFor(matchVersion); - } - - /** * Creates a new UAX29URLEmailTokenizer with a given {@link AttributeFactory} */ public UAX29URLEmailTokenizer(Version matchVersion, AttributeFactory factory, Reader input) { Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerFactory.java (working copy) @@ -17,10 +17,10 @@ * limitations under the License. */ -import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.standard.ClassicTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.util.AttributeSource.AttributeFactory; import java.io.Reader; import java.util.Map; @@ -50,9 +50,17 @@ } @Override - public Tokenizer create(Reader input) { + public ClassicTokenizer create(Reader input) { ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, input); tokenizer.setMaxTokenLength(maxTokenLength); return tokenizer; } + + @Override + public ClassicTokenizer create(AttributeFactory factory, Reader input) { + ClassicTokenizer tokenizer = new ClassicTokenizer(luceneMatchVersion, factory, input); + tokenizer.setMaxTokenLength(maxTokenLength); + return tokenizer; + } + } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java (working copy) @@ -104,14 +104,6 @@ } /** - * Creates a new ClassicTokenizer with a given {@link AttributeSource}. - */ - public ClassicTokenizer(Version matchVersion, AttributeSource source, Reader input) { - super(source, input); - init(matchVersion); - } - - /** * Creates a new ClassicTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory} */ public ClassicTokenizer(Version matchVersion, AttributeFactory factory, Reader input) { Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (working copy) @@ -118,14 +118,6 @@ } /** - * Creates a new StandardTokenizer with a given {@link AttributeSource}. - */ - public StandardTokenizer(Version matchVersion, AttributeSource source, Reader input) { - super(source, input); - init(matchVersion); - } - - /** * Creates a new StandardTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory} */ public StandardTokenizer(Version matchVersion, AttributeFactory factory, Reader input) { Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerFactory.java (working copy) @@ -17,9 +17,11 @@ * limitations under the License. */ +import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.standard.StandardTokenizer; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.util.AttributeSource.AttributeFactory; import java.io.Reader; import java.util.Map; @@ -54,4 +56,13 @@ tokenizer.setMaxTokenLength(maxTokenLength); return tokenizer; } + + @Override + public StandardTokenizer create(AttributeFactory factory, Reader input) { + StandardTokenizer tokenizer + = new StandardTokenizer(luceneMatchVersion, factory, input); + tokenizer.setMaxTokenLength(maxTokenLength); + return tokenizer; + } + } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerFactory.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.analysis.standard.UAX29URLEmailTokenizer; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.util.AttributeSource.AttributeFactory; import java.io.Reader; import java.util.Map; @@ -54,4 +55,12 @@ tokenizer.setMaxTokenLength(maxTokenLength); return tokenizer; } + + @Override + public UAX29URLEmailTokenizer create(AttributeFactory factory, Reader input) { + UAX29URLEmailTokenizer tokenizer = new UAX29URLEmailTokenizer(luceneMatchVersion, factory, input); + tokenizer.setMaxTokenLength(maxTokenLength); + return tokenizer; + } + } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerFactory.java (working copy) @@ -18,10 +18,11 @@ */ import java.io.Reader; +import java.util.Collections; -import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.util.TokenizerFactory; import org.apache.lucene.analysis.wikipedia.WikipediaTokenizer; +import org.apache.lucene.util.AttributeSource.AttributeFactory; /** * Factory for {@link WikipediaTokenizer}. @@ -36,7 +37,14 @@ public class WikipediaTokenizerFactory extends TokenizerFactory { // TODO: add support for WikipediaTokenizer's advanced options. @Override - public Tokenizer create(Reader input) { + public WikipediaTokenizer create(Reader input) { return new WikipediaTokenizer(input); } + + @Override + public WikipediaTokenizer create(AttributeFactory factory, Reader input) { + return new WikipediaTokenizer(factory, input, WikipediaTokenizer.TOKENS_ONLY, + Collections.emptySet()); + } + } Index: lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java =================================================================== --- lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java (revision 1441939) +++ lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizer.java (working copy) @@ -159,19 +159,6 @@ this.scanner = new WikipediaTokenizerImpl(null); // best effort NPE if you dont call reset init(tokenOutput, untokenizedTypes); } - - /** - * Creates a new instance of the {@link org.apache.lucene.analysis.wikipedia.WikipediaTokenizer}. Attaches the - * input to a the newly created JFlex scanner. Uses the given {@link AttributeSource}. - * - * @param input The input - * @param tokenOutput One of {@link #TOKENS_ONLY}, {@link #UNTOKENIZED_ONLY}, {@link #BOTH} - */ - public WikipediaTokenizer(AttributeSource source, Reader input, int tokenOutput, Set untokenizedTypes) { - super(source, input); - this.scanner = new WikipediaTokenizerImpl(null); // best effort NPE if you dont call reset - init(tokenOutput, untokenizedTypes); - } private void init(int tokenOutput, Set untokenizedTypes) { // TODO: cutover to enum Index: lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java =================================================================== --- lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java (revision 1441939) +++ lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMATypeAwareAnnotationsTokenizerFactory.java (working copy) @@ -45,7 +45,7 @@ } @Override - public Tokenizer create(Reader input) { + public UIMATypeAwareAnnotationsTokenizer create(Reader input) { return new UIMATypeAwareAnnotationsTokenizer(descriptorPath, tokenType, featurePath, input); } } Index: lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java =================================================================== --- lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java (revision 1441939) +++ lucene/analysis/uima/src/java/org/apache/lucene/analysis/uima/UIMAAnnotationsTokenizerFactory.java (working copy) @@ -43,7 +43,7 @@ } @Override - public Tokenizer create(Reader input) { + public UIMAAnnotationsTokenizer create(Reader input) { return new UIMAAnnotationsTokenizer(descriptorPath, tokenType, input); } } Index: lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java =================================================================== --- lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java (revision 1441939) +++ lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseTokenizerFactory.java (working copy) @@ -89,7 +89,7 @@ } @Override - public Tokenizer create(Reader input) { + public JapaneseTokenizer create(Reader input) { return new JapaneseTokenizer(input, userDictionary, discardPunctuation, mode); } Index: lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java =================================================================== --- lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java (revision 1441939) +++ lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseSentenceTokenizerFactory.java (working copy) @@ -19,17 +19,24 @@ import java.io.Reader; -import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.cn.smart.SentenceTokenizer; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.util.AttributeSource.AttributeFactory; /** * Factory for the SmartChineseAnalyzer {@link SentenceTokenizer} * @lucene.experimental */ public class SmartChineseSentenceTokenizerFactory extends TokenizerFactory { + @Override - public Tokenizer create(Reader input) { + public SentenceTokenizer create(Reader input) { return new SentenceTokenizer(input); } + + @Override + public SentenceTokenizer create(AttributeFactory factory, Reader input) { + return new SentenceTokenizer(factory, input); + } + } Index: lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java =================================================================== --- lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (revision 1441939) +++ lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SentenceTokenizer.java (working copy) @@ -52,10 +52,6 @@ super(reader); } - public SentenceTokenizer(AttributeSource source, Reader reader) { - super(source, reader); - } - public SentenceTokenizer(AttributeFactory factory, Reader reader) { super(factory, reader); } Index: lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java =================================================================== --- lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java (revision 1441939) +++ lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java (working copy) @@ -144,7 +144,7 @@ } @Override - public Tokenizer create(Reader input) { + public ICUTokenizer create(Reader input) { assert config != null : "inform must be called first!"; return new ICUTokenizer(input, config); } Index: lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (revision 1441939) +++ lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (working copy) @@ -46,13 +46,6 @@ assert input != null: "input must not be null"; this.input = input; } - - /** Construct a token stream processing the given input using the given AttributeSource. */ - protected Tokenizer(AttributeSource source, Reader input) { - super(source); - assert input != null: "input must not be null"; - this.input = input; - } /** * {@inheritDoc} Index: solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java =================================================================== --- solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java (revision 1441939) +++ solr/test-framework/src/java/org/apache/solr/analysis/MockCharFilterFactory.java (working copy) @@ -20,7 +20,6 @@ import java.io.Reader; import java.util.Map; -import org.apache.lucene.analysis.CharFilter; import org.apache.lucene.analysis.MockCharFilter; import org.apache.lucene.analysis.util.CharFilterFactory; @@ -41,7 +40,7 @@ } @Override - public CharFilter create(Reader input) { + public MockCharFilter create(Reader input) { return new MockCharFilter(input, remainder); } } Index: solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java =================================================================== --- solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java (revision 1441939) +++ solr/test-framework/src/java/org/apache/solr/analysis/MockTokenizerFactory.java (working copy) @@ -55,7 +55,7 @@ @Override - public Tokenizer create(Reader input) { + public MockTokenizer create(Reader input) { MockTokenizer t = new MockTokenizer(input, pattern, false); t.setEnableChecks(enableChecks); return t; Index: solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java =================================================================== --- solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (revision 1441939) +++ solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (working copy) @@ -20,12 +20,15 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.Tokenizer; import org.apache.lucene.analysis.util.TokenizerFactory; +import org.apache.lucene.util.Attribute; +import org.apache.lucene.util.AttributeImpl; import org.apache.solr.common.SolrException; import org.apache.solr.schema.DateField; import static org.apache.solr.schema.TrieField.TrieTypes; import java.io.IOException; import java.io.Reader; +import java.util.Iterator; /** * Tokenizer for trie fields. It uses NumericTokenStream to create multiple trie encoded string per number. @@ -68,9 +71,18 @@ return new NumericTokenStream(precisionStep); } - public TrieTokenizer(Reader input, TrieTypes type, NumericTokenStream ts) { - // must share the attribute source with the NumericTokenStream we delegate to - super(ts, input); + public TrieTokenizer(Reader input, TrieTypes type, final NumericTokenStream ts) { + // Häckidy-Hick-Hack: must share the attributes with the NumericTokenStream we delegate to, so we create a fake factory: + super(new AttributeFactory() { + @Override + public AttributeImpl createAttributeInstance(Class attClass) { + return (AttributeImpl) ts.addAttribute(attClass); + } + }, input); + // add all attributes: + for (Iterator> it = ts.getAttributeClassesIterator(); it.hasNext();) { + addAttribute(it.next()); + } this.type = type; this.ts = ts; }