Index: lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java =================================================================== --- lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (revision 1098898) +++ lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (working copy) @@ -107,6 +107,7 @@ // consume } stream.end(); + stream.close(); assertAnalyzesToReuse(analyzer, testString, new String[] { "t" }); } Index: lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java (revision 1098898) +++ lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java (working copy) @@ -626,11 +626,14 @@ } TokenStream ts = analyzer.reusableTokenStream("ignore", new StringReader(term)); CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class); + ts.reset(); while(ts.incrementToken()) { String text = termAttr.toString(); doc.add(text); sb.append(text).append(' '); } + ts.end(); + ts.close(); } else { // pick existing sub-phrase List lastDoc = docs.get(r.nextInt(docs.size())); Index: lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (revision 1098898) +++ lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (working copy) @@ -121,7 +121,9 @@ Analyzer analyzer = new MockAnalyzer(random); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); - TokenStream stream = new CachingTokenFilter(analyzer.tokenStream("field", new StringReader("abcd "))); + TokenStream stream = analyzer.tokenStream("field", new StringReader("abcd ")); + stream.reset(); // TODO: wierd to reset before wrapping with CachingTokenFilter... correct? + stream = new CachingTokenFilter(stream); Field f = new Field("field", stream, Field.TermVector.WITH_POSITIONS_OFFSETS); doc.add(f); doc.add(f); Index: lucene/src/test/org/apache/lucene/index/TestLongPostings.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestLongPostings.java (revision 1098898) +++ lucene/src/test/org/apache/lucene/index/TestLongPostings.java (working copy) @@ -49,6 +49,7 @@ final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class); final BytesRef termBytes = termAtt.getBytesRef(); int count = 0; + ts.reset(); while(ts.incrementToken()) { termAtt.fillBytesRef(); if (count == 0 && !termBytes.utf8ToString().equals(s)) { Index: lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java =================================================================== --- lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java (revision 1098898) +++ lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java (working copy) @@ -806,6 +806,7 @@ } try { + source.end(); source.close(); } catch (IOException ignored) {} Index: lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (revision 1098898) +++ lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (working copy) @@ -104,7 +104,6 @@ return saved.filter; } else { saved.tokenizer.reset(reader); - saved.filter.reset(); return saved.filter; } } Index: lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 1098898) +++ lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (working copy) @@ -262,6 +262,7 @@ tokens.add(termAtt.toString()); // TODO: we could collect offsets etc here for better checking that reset() really works. } + ts.end(); ts.close(); // verify reusing is "reproducable" and also get the normal tokenstream sanity checks if (!tokens.isEmpty()) Index: lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java (revision 1098898) +++ lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java (working copy) @@ -50,11 +50,26 @@ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); int off = 0; + // TODO: "register" with LuceneTestCase to ensure all streams are closed() ? + // currently, we can only check that the lifecycle is correct if someone is reusing, + // but not for "one-offs". + private static enum State { + SETREADER, // consumer set a reader input either via ctor or via reset(Reader) + RESET, // consumer has called reset() + INCREMENT, // consumer is consuming, has called incrementToken() == true + INCREMENT_FALSE, // consumer has called incrementToken() which returned false + END, // consumer has called end() to perform end of stream operations + CLOSE // consumer has called close() to release any resources + }; + + private State streamState = State.CLOSE; + public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) { super(factory, input); this.runAutomaton = runAutomaton; this.lowerCase = lowerCase; this.state = runAutomaton.getInitialState(); + this.streamState = State.SETREADER; } public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) { @@ -62,10 +77,13 @@ this.runAutomaton = runAutomaton; this.lowerCase = lowerCase; this.state = runAutomaton.getInitialState(); + this.streamState = State.SETREADER; } @Override public final boolean incrementToken() throws IOException { + assert (streamState == State.RESET || streamState == State.INCREMENT) + : "incrementToken() called while in wrong state: " + streamState; clearAttributes(); for (;;) { int startOffset = off; @@ -82,9 +100,11 @@ cp = readCodePoint(); } while (cp >= 0 && isTokenChar(cp)); offsetAtt.setOffset(startOffset, endOffset); + streamState = State.INCREMENT; return true; } } + streamState = State.INCREMENT_FALSE; return false; } @@ -126,11 +146,32 @@ super.reset(); state = runAutomaton.getInitialState(); off = 0; + assert streamState != State.RESET : "double reset()"; + streamState = State.RESET; } + + @Override + public void close() throws IOException { + super.close(); + // we cannot really check this, because it can be legitimately closed at + // any time by the consumer (exception)... maybe we should normally check, + // but allow the check to be disabled by e.g. TestIndexWriterExceptions? + // assert streamState == State.END : "close() called in wrong state: " + streamState; + streamState = State.CLOSE; + } @Override + public void reset(Reader input) throws IOException { + super.reset(input); + assert streamState == State.CLOSE : "setReader() called in wrong state: " + streamState; + streamState = State.SETREADER; + } + + @Override public void end() throws IOException { int finalOffset = correctOffset(off); offsetAtt.setOffset(finalOffset, finalOffset); + assert streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!"; + streamState = State.END; } } Index: lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java (revision 1098898) +++ lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java (working copy) @@ -86,6 +86,7 @@ @Override public void reset() throws IOException { + super.reset(); i = 0; pos = 0; } Index: lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java =================================================================== --- lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (revision 1098868) +++ lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (working copy) @@ -123,6 +123,11 @@ TokenStream source = this.analyzer.tokenStream(field, new StringReader( text)); + try { + source.reset(); + } catch (IOException e1) { + throw new RuntimeException(e1); + } CachingTokenFilter buffer = new CachingTokenFilter(source); PositionIncrementAttribute posIncrAtt = null; Index: lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java =================================================================== --- lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (revision 1098868) +++ lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (working copy) @@ -110,6 +110,11 @@ CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); int countTokens = 0; + try { + source.reset(); + } catch (IOException e1) { + throw new RuntimeException(e1); + } while (true) { try { if (!source.incrementToken()) break; @@ -126,6 +131,7 @@ } } try { + source.end(); source.close(); } catch (IOException e) { // ignore @@ -191,7 +197,11 @@ TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr)); List tlist = new ArrayList(); CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); - + try { + source.reset(); + } catch (IOException e1) { + throw new RuntimeException(e1); + } while (true) { try { if (!source.incrementToken()) break; @@ -202,6 +212,7 @@ } try { + source.end(); source.close(); } catch (IOException e) { // ignore @@ -242,6 +253,7 @@ boolean multipleTokens = false; try { + source.reset(); if (source.incrementToken()) { nextToken = termAtt.toString(); } @@ -251,6 +263,7 @@ } try { + source.end(); source.close(); } catch (IOException e) { // ignore @@ -281,6 +294,7 @@ try { source = getAnalyzer().tokenStream(field, new StringReader(part1)); termAtt = source.addAttribute(CharTermAttribute.class); + source.reset(); multipleTokens = false; @@ -292,6 +306,7 @@ // ignore } try { + source.end(); source.close(); } catch (IOException e) { // ignore @@ -308,6 +323,7 @@ termAtt = source.addAttribute(CharTermAttribute.class); try { + source.reset(); if (source.incrementToken()) { part2 = termAtt.toString(); } @@ -316,6 +332,7 @@ // ignore } try { + source.end(); source.close(); } catch (IOException e) { // ignore Index: lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java =================================================================== --- lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (revision 1098868) +++ lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (working copy) @@ -111,7 +111,6 @@ setPreviousTokenStream(streams); } else { streams.source.reset(reader); - streams.result.reset(); // reset the SynonymTokenFilter } return streams.result; } Index: lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java =================================================================== --- lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (revision 1098868) +++ lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (working copy) @@ -118,12 +118,14 @@ // [1] Parse query into separate words so that when we expand we can avoid dups TokenStream ts = a.tokenStream( field, new StringReader( query)); CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); - + ts.reset(); while (ts.incrementToken()) { String word = termAtt.toString(); if ( already.add( word)) top.add( word); } + ts.end(); + ts.close(); final BooleanQuery tmp = new BooleanQuery(); // [2] form query Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (revision 1098868) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (working copy) @@ -64,6 +64,7 @@ { Term term = null; BytesRef bytes = termAtt.getBytesRef(); + ts.reset(); while (ts.incrementToken()) { termAtt.fillBytesRef(); if (term == null) @@ -76,6 +77,8 @@ } tf.addTerm(term); } + ts.end(); + ts.close(); } catch (IOException ioe) { Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (revision 1098868) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (working copy) @@ -80,9 +80,12 @@ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); try { + ts.reset(); while(ts.incrementToken()) { stopWordsSet.add(termAtt.toString()); } + ts.end(); + ts.close(); } catch(IOException ioe) { Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (revision 1098868) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (working copy) @@ -59,11 +59,14 @@ TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value)); TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); BytesRef bytes = termAtt.getBytesRef(); + ts.reset(); while (ts.incrementToken()) { termAtt.fillBytesRef(); SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, new BytesRef(bytes))); clausesList.add(stq); } + ts.end(); + ts.close(); SpanOrQuery soq=new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()])); soq.setBoost(DOMUtils.getAttribute(e,"boost",1.0f)); return soq; Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (revision 1098868) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (working copy) @@ -61,6 +61,7 @@ TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); Term term = null; BytesRef bytes = termAtt.getBytesRef(); + ts.reset(); while (ts.incrementToken()) { termAtt.fillBytesRef(); if (term == null) @@ -73,6 +74,8 @@ } bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD)); } + ts.end(); + ts.close(); } catch (IOException ioe) { Index: lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (revision 1098868) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (working copy) @@ -885,7 +885,7 @@ int tokenCount=0; // for every token CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); - + ts.reset(); while (ts.incrementToken()) { String word = termAtt.toString(); tokenCount++; @@ -906,6 +906,8 @@ cnt.x++; } } + ts.end(); + ts.close(); } Index: lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (revision 1098868) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (working copy) @@ -192,6 +192,7 @@ int corpusNumDocs=reader.numDocs(); Term internSavingTemplateTerm =new Term(f.fieldName); //optimization to avoid constructing new Term() objects HashSet processedTerms=new HashSet(); + ts.reset(); while (ts.incrementToken()) { String term = termAtt.toString(); @@ -246,7 +247,9 @@ } } } - } + } + ts.end(); + ts.close(); } @Override