Index: lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java =================================================================== --- lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (revision 1098868) +++ lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (working copy) @@ -107,6 +107,7 @@ // consume } stream.end(); + stream.close(); assertAnalyzesToReuse(analyzer, testString, new String[] { "t" }); } Index: lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java (revision 1098868) +++ lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java (working copy) @@ -626,11 +626,14 @@ } TokenStream ts = analyzer.reusableTokenStream("ignore", new StringReader(term)); CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class); + ts.reset(); while(ts.incrementToken()) { String text = termAttr.toString(); doc.add(text); sb.append(text).append(' '); } + ts.end(); + ts.close(); } else { // pick existing sub-phrase List lastDoc = docs.get(r.nextInt(docs.size())); Index: lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (revision 1098868) +++ lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (working copy) @@ -121,7 +121,9 @@ Analyzer analyzer = new MockAnalyzer(random); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); - TokenStream stream = new CachingTokenFilter(analyzer.tokenStream("field", new StringReader("abcd "))); + TokenStream stream = analyzer.tokenStream("field", new StringReader("abcd ")); + stream.reset(); // TODO: wierd to reset before wrapping with CachingTokenFilter... correct? + stream = new CachingTokenFilter(stream); Field f = new Field("field", stream, Field.TermVector.WITH_POSITIONS_OFFSETS); doc.add(f); doc.add(f); Index: lucene/src/test/org/apache/lucene/index/TestLongPostings.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestLongPostings.java (revision 1098868) +++ lucene/src/test/org/apache/lucene/index/TestLongPostings.java (working copy) @@ -49,6 +49,7 @@ final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class); final BytesRef termBytes = termAtt.getBytesRef(); int count = 0; + ts.reset(); while(ts.incrementToken()) { termAtt.fillBytesRef(); if (count == 0 && !termBytes.utf8ToString().equals(s)) { Index: lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java =================================================================== --- lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java (revision 1098868) +++ lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java (working copy) @@ -806,6 +806,7 @@ } try { + source.end(); source.close(); } catch (IOException ignored) {} Index: lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (revision 1098868) +++ lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (working copy) @@ -104,7 +104,6 @@ return saved.filter; } else { saved.tokenizer.reset(reader); - saved.filter.reset(); return saved.filter; } } Index: lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 1098868) +++ lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (working copy) @@ -262,6 +262,7 @@ tokens.add(termAtt.toString()); // TODO: we could collect offsets etc here for better checking that reset() really works. } + ts.end(); ts.close(); // verify reusing is "reproducable" and also get the normal tokenstream sanity checks if (!tokens.isEmpty()) Index: lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java (revision 1098871) +++ lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java (working copy) @@ -50,11 +50,26 @@ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); int off = 0; + // TODO: "register" with LuceneTestCase to ensure all streams are closed() ? + // currently, we can only check that the lifecycle is correct if someone is reusing, + // but not for "one-offs". + private static enum State { + SETREADER, // consumer set a reader input either via ctor or via reset(Reader) + RESET, // consumer has called reset() + INCREMENT, // consumer is consuming, has called incrementToken() == true + INCREMENT_FALSE, // consumer has called incrementToken() which returned false + END, // consumer has called end() to perform end of stream operations + CLOSE // consumer has called close() to release any resources + }; + + private State streamState = State.CLOSE; + public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) { super(factory, input); this.runAutomaton = runAutomaton; this.lowerCase = lowerCase; this.state = runAutomaton.getInitialState(); + this.streamState = State.SETREADER; } public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) { @@ -62,10 +77,13 @@ this.runAutomaton = runAutomaton; this.lowerCase = lowerCase; this.state = runAutomaton.getInitialState(); + this.streamState = State.SETREADER; } @Override public final boolean incrementToken() throws IOException { + assert (streamState == State.RESET || streamState == State.INCREMENT) + : "incrementToken() called while in wrong state: " + streamState; clearAttributes(); for (;;) { int startOffset = off; @@ -82,9 +100,11 @@ cp = readCodePoint(); } while (cp >= 0 && isTokenChar(cp)); offsetAtt.setOffset(startOffset, endOffset); + streamState = State.INCREMENT; return true; } } + streamState = State.INCREMENT_FALSE; return false; } @@ -126,11 +146,32 @@ super.reset(); state = runAutomaton.getInitialState(); off = 0; + assert streamState != State.RESET : "double reset()"; + streamState = State.RESET; } + + @Override + public void close() throws IOException { + super.close(); + // we cannot really check this, because it can be legitimately closed at + // any time by the consumer (exception)... maybe we should normally check, + // but allow the check to be disabled by e.g. TestIndexWriterExceptions? + // assert streamState == State.END : "close() called in wrong state: " + streamState; + streamState = State.CLOSE; + } @Override + public void reset(Reader input) throws IOException { + super.reset(input); + assert streamState == State.CLOSE : "setReader() called in wrong state: " + streamState; + streamState = State.SETREADER; + } + + @Override public void end() throws IOException { int finalOffset = correctOffset(off); offsetAtt.setOffset(finalOffset, finalOffset); + assert streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!"; + streamState = State.END; } } Index: lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java (revision 1098868) +++ lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java (working copy) @@ -86,6 +86,7 @@ @Override public void reset() throws IOException { + super.reset(); i = 0; pos = 0; }