Index: solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java =================================================================== --- solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (revision 1242406) +++ solr/core/src/java/org/apache/solr/analysis/TrieTokenizerFactory.java (working copy) @@ -67,7 +67,7 @@ public TrieTokenizer(Reader input, TrieTypes type, int precisionStep, NumericTokenStream ts) { // must share the attribute source with the NumericTokenStream we delegate to - super(ts); + super(ts, input); this.type = type; this.precisionStep = precisionStep; this.ts = ts; Index: modules/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java =================================================================== --- modules/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java (revision 1242406) +++ modules/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.Reader; +import java.io.StringReader; import java.util.HashMap; import java.util.Map; @@ -27,11 +28,11 @@ import org.apache.lucene.document.TextField; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -319,6 +320,11 @@ } private static class EmptyTokenStream extends Tokenizer { + + public EmptyTokenStream() { + super(new StringReader("")); + } + @Override public boolean incrementToken() throws IOException { return false; Index: modules/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java =================================================================== --- modules/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java (revision 1242406) +++ modules/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiPhraseQueryParsing.java (working copy) @@ -41,6 +41,7 @@ } } + // TODO: cutover to the CannedAnalyzer in test-framework... private static class CannedAnalyzer extends Analyzer { private final TokenAndPos[] tokens; @@ -50,11 +51,11 @@ @Override public TokenStreamComponents createComponents(String fieldName, Reader reader) { - return new TokenStreamComponents(new CannedTokenizer(tokens)); + return new TokenStreamComponents(null, new CannedTokenizer(tokens)); } } - private static class CannedTokenizer extends Tokenizer { + private static class CannedTokenizer extends TokenStream { private final TokenAndPos[] tokens; private int upto = 0; private int lastPos = 0; @@ -80,9 +81,7 @@ } } - @Override public void reset(Reader reader) throws IOException { - super.reset(reader); this.upto = 0; this.lastPos = 0; } Index: modules/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java =================================================================== --- modules/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java (revision 1242406) +++ modules/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java (working copy) @@ -18,6 +18,7 @@ */ import java.io.Reader; +import java.io.StringReader; import java.util.HashMap; import java.util.Map; @@ -28,11 +29,11 @@ import org.apache.lucene.index.IndexWriter; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler; +import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; -import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; @@ -357,6 +358,10 @@ } private static class EmptyTokenStream extends Tokenizer { + public EmptyTokenStream() { + super(new StringReader("")); + } + @Override public boolean incrementToken() { return false; Index: modules/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java =================================================================== --- modules/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java (revision 1242406) +++ modules/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestQPHelper.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.Reader; +import java.io.StringReader; import java.text.DateFormat; import java.util.Calendar; import java.util.Date; @@ -35,18 +36,19 @@ import org.apache.lucene.document.DateTools; import org.apache.lucene.document.Document; import org.apache.lucene.document.TextField; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; -import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.Term; -import org.apache.lucene.queryparser.flexible.messages.MessageImpl; import org.apache.lucene.queryparser.flexible.core.QueryNodeException; import org.apache.lucene.queryparser.flexible.core.messages.QueryParserMessages; import org.apache.lucene.queryparser.flexible.core.nodes.FuzzyQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorImpl; import org.apache.lucene.queryparser.flexible.core.processors.QueryNodeProcessorPipeline; +import org.apache.lucene.queryparser.flexible.messages.MessageImpl; import org.apache.lucene.queryparser.flexible.standard.config.StandardQueryConfigHandler; import org.apache.lucene.queryparser.flexible.standard.nodes.WildcardQueryNode; +import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.FuzzyQuery; @@ -62,7 +64,6 @@ import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.TermRangeQuery; import org.apache.lucene.search.WildcardQuery; -import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.store.Directory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.automaton.BasicAutomata; @@ -1219,7 +1220,11 @@ private int upto = 0; private final PositionIncrementAttribute posIncr = addAttribute(PositionIncrementAttribute.class); private final CharTermAttribute term = addAttribute(CharTermAttribute.class); - + + public CannedTokenStream() { + super(new StringReader("")); + } + @Override public boolean incrementToken() { clearAttributes(); Index: modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java =================================================================== --- modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (revision 1242406) +++ modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (working copy) @@ -125,27 +125,27 @@ * See http://issues.apache.org/jira/browse/LUCENE-1068 */ public StandardTokenizer(Version matchVersion, Reader input) { - super(); - init(input, matchVersion); + super(input); + init(matchVersion); } /** * Creates a new StandardTokenizer with a given {@link AttributeSource}. */ public StandardTokenizer(Version matchVersion, AttributeSource source, Reader input) { - super(source); - init(input, matchVersion); + super(source, input); + init(matchVersion); } /** * Creates a new StandardTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory} */ public StandardTokenizer(Version matchVersion, AttributeFactory factory, Reader input) { - super(factory); - init(input, matchVersion); + super(factory, input); + init(matchVersion); } - private final void init(Reader input, Version matchVersion) { + private final void init(Version matchVersion) { if (matchVersion.onOrAfter(Version.LUCENE_34)) { this.scanner = new StandardTokenizerImpl(input); } else if (matchVersion.onOrAfter(Version.LUCENE_31)) { @@ -153,7 +153,6 @@ } else { this.scanner = new ClassicTokenizerImpl(input); } - this.input = input; } // this tokenizer generates three attributes: Index: modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java =================================================================== --- modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java (revision 1242406) +++ modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java (working copy) @@ -99,29 +99,28 @@ * See http://issues.apache.org/jira/browse/LUCENE-1068 */ public ClassicTokenizer(Version matchVersion, Reader input) { - super(); - init(input, matchVersion); + super(input); + init(matchVersion); } /** * Creates a new ClassicTokenizer with a given {@link AttributeSource}. */ public ClassicTokenizer(Version matchVersion, AttributeSource source, Reader input) { - super(source); - init(input, matchVersion); + super(source, input); + init(matchVersion); } /** * Creates a new ClassicTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory} */ public ClassicTokenizer(Version matchVersion, AttributeFactory factory, Reader input) { - super(factory); - init(input, matchVersion); + super(factory, input); + init(matchVersion); } - private void init(Reader input, Version matchVersion) { + private void init(Version matchVersion) { this.scanner = new ClassicTokenizerImpl(input); - this.input = input; } // this tokenizer generates three attributes: Index: lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java =================================================================== --- lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (revision 1242406) +++ lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (working copy) @@ -58,7 +58,7 @@ Analyzer analyzer = new Analyzer() { @Override public TokenStreamComponents createComponents(String fieldName, Reader reader) { - return new TokenStreamComponents(new Tokenizer() { + return new TokenStreamComponents(new Tokenizer(reader) { private final String[] TOKENS = {"1", "2", "3", "4", "5"}; private final int[] INCREMENTS = {0, 2, 1, 0, 1}; private int i = 0; Index: lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java (revision 1242406) +++ lucene/core/src/test/org/apache/lucene/index/TestSameTokenSamePosition.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.Reader; +import java.io.StringReader; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Tokenizer; @@ -81,6 +82,10 @@ private final int ends[] = new int[]{3, 3, 11, 11}; private final int incs[] = new int[]{1, 0, 1, 0}; + public BugReproAnalyzerTokenizer() { + super(new StringReader("")); + } + @Override public boolean incrementToken() throws IOException { if (nextTokenIndex < tokenCount) { Index: lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (revision 1242406) +++ lucene/core/src/test/org/apache/lucene/index/TestTermVectorsReader.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.Reader; +import java.io.StringReader; import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -145,6 +146,7 @@ private final OffsetAttribute offsetAtt; public MyTokenStream() { + super(new StringReader("")); termAtt = addAttribute(CharTermAttribute.class); posIncrAtt = addAttribute(PositionIncrementAttribute.class); offsetAtt = addAttribute(OffsetAttribute.class); Index: lucene/core/src/test/org/apache/lucene/index/TestTermdocPerf.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestTermdocPerf.java (revision 1242406) +++ lucene/core/src/test/org/apache/lucene/index/TestTermdocPerf.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.Reader; +import java.io.StringReader; import java.util.Random; import org.apache.lucene.analysis.Analyzer; @@ -42,6 +43,7 @@ String value; public RepeatingTokenStream(String val, Random random, float percentDocs, int maxTF) { + super(new StringReader("")); this.value = val; this.random = random; this.percentDocs = percentDocs; Index: lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 1242406) +++ lucene/core/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -1569,6 +1569,7 @@ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class); public StringSplitTokenizer(Reader r) { + super(r); try { reset(r); } catch (IOException e) { Index: lucene/core/src/test/org/apache/lucene/document/TestDocument.java =================================================================== --- lucene/core/src/test/org/apache/lucene/document/TestDocument.java (revision 1242406) +++ lucene/core/src/test/org/apache/lucene/document/TestDocument.java (working copy) @@ -285,7 +285,7 @@ // LUCENE-3616 public void testInvalidFields() { try { - new Field("foo", new Tokenizer() { + new Field("foo", new Tokenizer(new StringReader("")) { @Override public boolean incrementToken() { return false; Index: lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java =================================================================== --- lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (revision 1242406) +++ lucene/core/src/java/org/apache/lucene/analysis/Tokenizer.java (working copy) @@ -34,34 +34,24 @@ /** The text source for this Tokenizer. */ protected Reader input; - /** Construct a tokenizer with null input. */ - protected Tokenizer() {} - /** Construct a token stream processing the given input. */ protected Tokenizer(Reader input) { - this.input = CharReader.get(input); + assert input != null: "input must not be null"; + this.input = input; } - /** Construct a tokenizer with null input using the given AttributeFactory. */ - protected Tokenizer(AttributeFactory factory) { - super(factory); - } - /** Construct a token stream processing the given input using the given AttributeFactory. */ protected Tokenizer(AttributeFactory factory, Reader input) { super(factory); - this.input = CharReader.get(input); + assert input != null: "input must not be null"; + this.input = input; } /** Construct a token stream processing the given input using the given AttributeSource. */ - protected Tokenizer(AttributeSource source) { - super(source); - } - - /** Construct a token stream processing the given input using the given AttributeSource. */ protected Tokenizer(AttributeSource source, Reader input) { super(source); - this.input = CharReader.get(input); + assert input != null: "input must not be null"; + this.input = input; } /** By default, closes the input Reader. */ @@ -82,6 +72,7 @@ * @see CharStream#correctOffset */ protected final int correctOffset(int currentOff) { + assert input != null: "subclass failed to call super(Reader) or super.reset(Reader)"; return (input instanceof CharStream) ? ((CharStream) input).correctOffset(currentOff) : currentOff; } Index: lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java =================================================================== --- lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java (revision 1242406) +++ lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java (working copy) @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.Reader; +import java.io.StringReader; import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; @@ -65,6 +66,7 @@ private final PositionIncrementAttribute positionIncrementAttribute = addAttribute(PositionIncrementAttribute.class); public TokenStreamOverlap() { + super(new StringReader("")); reset(); } Index: lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java =================================================================== --- lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java (revision 1242406) +++ lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/IndexTimeSynonymTest.java (working copy) @@ -298,7 +298,7 @@ @Override public TokenStreamComponents createComponents(String fieldName, Reader reader) { - Tokenizer ts = new Tokenizer(Token.TOKEN_ATTRIBUTE_FACTORY) { + Tokenizer ts = new Tokenizer(Token.TOKEN_ATTRIBUTE_FACTORY, reader) { final AttributeImpl reusableToken = (AttributeImpl) addAttribute(CharTermAttribute.class); int p = 0; Index: lucene/test-framework/src/java/org/apache/lucene/analysis/CannedAnalyzer.java =================================================================== --- lucene/test-framework/src/java/org/apache/lucene/analysis/CannedAnalyzer.java (revision 1242406) +++ lucene/test-framework/src/java/org/apache/lucene/analysis/CannedAnalyzer.java (working copy) @@ -17,8 +17,9 @@ * limitations under the License. */ +import java.io.IOException; import java.io.Reader; -import java.io.IOException; +import java.io.StringReader; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; @@ -44,6 +45,7 @@ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); public CannedTokenizer(Token[] tokens) { + super(new StringReader("")); this.tokens = tokens; }