Index: lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java =================================================================== --- lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (revision 1102141) +++ lucene/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (working copy) @@ -107,6 +107,7 @@ // consume } stream.end(); + stream.close(); assertAnalyzesToReuse(analyzer, testString, new String[] { "t" }); } Index: lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java (revision 1102141) +++ lucene/src/test/org/apache/lucene/search/TestPhraseQuery.java (working copy) @@ -626,11 +626,14 @@ } TokenStream ts = analyzer.reusableTokenStream("ignore", new StringReader(term)); CharTermAttribute termAttr = ts.addAttribute(CharTermAttribute.class); + ts.reset(); while(ts.incrementToken()) { String text = termAttr.toString(); doc.add(text); sb.append(text).append(' '); } + ts.end(); + ts.close(); } else { // pick existing sub-phrase List lastDoc = docs.get(r.nextInt(docs.size())); Index: lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (revision 1102141) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (working copy) @@ -155,7 +155,9 @@ } MockDirectoryWrapper dir = newDirectory(); - MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)) + MockAnalyzer analyzer = new MockAnalyzer(random); + analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases. + MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer) .setRAMBufferSizeMB(0.1).setMergeScheduler(new ConcurrentMergeScheduler())); ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions(); //writer.setMaxBufferedDocs(10); @@ -201,7 +203,9 @@ public void testRandomExceptionsThreads() throws Throwable { MockDirectoryWrapper dir = newDirectory(); - MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)) + MockAnalyzer analyzer = new MockAnalyzer(random); + analyzer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases. + MockIndexWriter writer = new MockIndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer) .setRAMBufferSizeMB(0.2).setMergeScheduler(new ConcurrentMergeScheduler())); ((ConcurrentMergeScheduler) writer.getConfig().getMergeScheduler()).setSuppressExceptions(); //writer.setMaxBufferedDocs(10); @@ -321,7 +325,9 @@ Analyzer analyzer = new Analyzer() { @Override public TokenStream tokenStream(String fieldName, Reader reader) { - return new CrashingFilter(fieldName, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false)); + MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); + tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases. + return new CrashingFilter(fieldName, tokenizer); } }; @@ -390,7 +396,9 @@ @Override public TokenStream tokenStream(String fieldName, Reader reader) { - return new TokenFilter(new MockTokenizer(reader, MockTokenizer.SIMPLE, true)) { + MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true); + tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases. + return new TokenFilter(tokenizer) { private int count = 0; @Override @@ -522,7 +530,9 @@ Analyzer analyzer = new Analyzer() { @Override public TokenStream tokenStream(String fieldName, Reader reader) { - return new CrashingFilter(fieldName, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false)); + MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); + tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases. + return new CrashingFilter(fieldName, tokenizer); } }; @@ -621,7 +631,9 @@ Analyzer analyzer = new Analyzer() { @Override public TokenStream tokenStream(String fieldName, Reader reader) { - return new CrashingFilter(fieldName, new MockTokenizer(reader, MockTokenizer.WHITESPACE, false)); + MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); + tokenizer.setEnableChecks(false); // disable workflow checking as we forcefully close() in exceptional cases. + return new CrashingFilter(fieldName, tokenizer); } }; Index: lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (revision 1102141) +++ lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java (working copy) @@ -121,7 +121,9 @@ Analyzer analyzer = new MockAnalyzer(random); IndexWriter w = new IndexWriter(dir, newIndexWriterConfig( TEST_VERSION_CURRENT, analyzer)); Document doc = new Document(); - TokenStream stream = new CachingTokenFilter(analyzer.tokenStream("field", new StringReader("abcd "))); + TokenStream stream = analyzer.tokenStream("field", new StringReader("abcd ")); + stream.reset(); // TODO: wierd to reset before wrapping with CachingTokenFilter... correct? + stream = new CachingTokenFilter(stream); Field f = new Field("field", stream, Field.TermVector.WITH_POSITIONS_OFFSETS); doc.add(f); doc.add(f); Index: lucene/src/test/org/apache/lucene/index/TestLongPostings.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestLongPostings.java (revision 1102141) +++ lucene/src/test/org/apache/lucene/index/TestLongPostings.java (working copy) @@ -49,6 +49,7 @@ final TermToBytesRefAttribute termAtt = ts.getAttribute(TermToBytesRefAttribute.class); final BytesRef termBytes = termAtt.getBytesRef(); int count = 0; + ts.reset(); while(ts.incrementToken()) { termAtt.fillBytesRef(); if (count == 0 && !termBytes.utf8ToString().equals(s)) { Index: lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java =================================================================== --- lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java (revision 1102141) +++ lucene/src/java/org/apache/lucene/queryParser/QueryParserBase.java (working copy) @@ -806,6 +806,7 @@ } try { + source.end(); source.close(); } catch (IOException ignored) {} Index: lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (revision 1102141) +++ lucene/src/test-framework/org/apache/lucene/analysis/MockAnalyzer.java (working copy) @@ -36,6 +36,7 @@ private int positionIncrementGap; private final Random random; private Map previousMappings = new HashMap(); + private boolean enableChecks = true; /** * Creates a new MockAnalyzer. @@ -75,6 +76,7 @@ @Override public TokenStream tokenStream(String fieldName, Reader reader) { MockTokenizer tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase); + tokenizer.setEnableChecks(enableChecks); TokenFilter filt = new MockTokenFilter(tokenizer, filter, enablePositionIncrements); filt = maybePayload(filt, fieldName); return filt; @@ -98,13 +100,13 @@ if (saved == null) { saved = new SavedStreams(); saved.tokenizer = new MockTokenizer(reader, runAutomaton, lowerCase); + saved.tokenizer.setEnableChecks(enableChecks); saved.filter = new MockTokenFilter(saved.tokenizer, filter, enablePositionIncrements); saved.filter = maybePayload(saved.filter, fieldName); map.put(fieldName, saved); return saved.filter; } else { saved.tokenizer.reset(reader); - saved.filter.reset(); return saved.filter; } } @@ -139,4 +141,12 @@ public int getPositionIncrementGap(String fieldName){ return positionIncrementGap; } + + /** + * Toggle consumer workflow checking: if your test consumes tokenstreams normally you + * should leave this enabled. + */ + public void setEnableChecks(boolean enableChecks) { + this.enableChecks = enableChecks; + } } Index: lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 1102141) +++ lucene/src/test-framework/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (working copy) @@ -262,6 +262,7 @@ tokens.add(termAtt.toString()); // TODO: we could collect offsets etc here for better checking that reset() really works. } + ts.end(); ts.close(); // verify reusing is "reproducable" and also get the normal tokenstream sanity checks if (!tokens.isEmpty()) Index: lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java (revision 1102141) +++ lucene/src/test-framework/org/apache/lucene/analysis/MockTokenizer.java (working copy) @@ -50,11 +50,27 @@ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class); int off = 0; + // TODO: "register" with LuceneTestCase to ensure all streams are closed() ? + // currently, we can only check that the lifecycle is correct if someone is reusing, + // but not for "one-offs". + private static enum State { + SETREADER, // consumer set a reader input either via ctor or via reset(Reader) + RESET, // consumer has called reset() + INCREMENT, // consumer is consuming, has called incrementToken() == true + INCREMENT_FALSE, // consumer has called incrementToken() which returned false + END, // consumer has called end() to perform end of stream operations + CLOSE // consumer has called close() to release any resources + }; + + private State streamState = State.CLOSE; + private boolean enableChecks = true; + public MockTokenizer(AttributeFactory factory, Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) { super(factory, input); this.runAutomaton = runAutomaton; this.lowerCase = lowerCase; this.state = runAutomaton.getInitialState(); + this.streamState = State.SETREADER; } public MockTokenizer(Reader input, CharacterRunAutomaton runAutomaton, boolean lowerCase) { @@ -62,10 +78,13 @@ this.runAutomaton = runAutomaton; this.lowerCase = lowerCase; this.state = runAutomaton.getInitialState(); + this.streamState = State.SETREADER; } @Override public final boolean incrementToken() throws IOException { + assert !enableChecks || (streamState == State.RESET || streamState == State.INCREMENT) + : "incrementToken() called while in wrong state: " + streamState; clearAttributes(); for (;;) { int startOffset = off; @@ -82,9 +101,11 @@ cp = readCodePoint(); } while (cp >= 0 && isTokenChar(cp)); offsetAtt.setOffset(startOffset, endOffset); + streamState = State.INCREMENT; return true; } } + streamState = State.INCREMENT_FALSE; return false; } @@ -126,11 +147,42 @@ super.reset(); state = runAutomaton.getInitialState(); off = 0; + assert !enableChecks || streamState != State.RESET : "double reset()"; + streamState = State.RESET; } + + @Override + public void close() throws IOException { + super.close(); + // in some exceptional cases (e.g. TestIndexWriterExceptions) a test can prematurely close() + // these tests should disable this check, by default we check the normal workflow. + // TODO: investigate the CachingTokenFilter "double-close"... for now we ignore this + assert !enableChecks || streamState == State.END || streamState == State.CLOSE : "close() called in wrong state: " + streamState; + streamState = State.CLOSE; + } @Override + public void reset(Reader input) throws IOException { + super.reset(input); + assert !enableChecks || streamState == State.CLOSE : "setReader() called in wrong state: " + streamState; + streamState = State.SETREADER; + } + + @Override public void end() throws IOException { int finalOffset = correctOffset(off); offsetAtt.setOffset(finalOffset, finalOffset); + // some tokenizers, such as limiting tokenizers, call end() before incrementToken() returns false. + // these tests should disable this check (in general you should consume the entire stream) + assert !enableChecks || streamState == State.INCREMENT_FALSE : "end() called before incrementToken() returned false!"; + streamState = State.END; } + + /** + * Toggle consumer workflow checking: if your test consumes tokenstreams normally you + * should leave this enabled. + */ + public void setEnableChecks(boolean enableChecks) { + this.enableChecks = enableChecks; + } } Index: lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java =================================================================== --- lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java (revision 1102141) +++ lucene/src/test-framework/org/apache/lucene/analysis/MockPayloadAnalyzer.java (working copy) @@ -86,6 +86,7 @@ @Override public void reset() throws IOException { + super.reset(); i = 0; pos = 0; } Index: lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java =================================================================== --- lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (revision 1102141) +++ lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (working copy) @@ -123,6 +123,11 @@ TokenStream source = this.analyzer.tokenStream(field, new StringReader( text)); + try { + source.reset(); + } catch (IOException e1) { + throw new RuntimeException(e1); + } CachingTokenFilter buffer = new CachingTokenFilter(source); PositionIncrementAttribute posIncrAtt = null; Index: lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java =================================================================== --- lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (revision 1102141) +++ lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (working copy) @@ -110,6 +110,11 @@ CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); int countTokens = 0; + try { + source.reset(); + } catch (IOException e1) { + throw new RuntimeException(e1); + } while (true) { try { if (!source.incrementToken()) break; @@ -126,6 +131,7 @@ } } try { + source.end(); source.close(); } catch (IOException e) { // ignore @@ -191,7 +197,11 @@ TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr)); List tlist = new ArrayList(); CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); - + try { + source.reset(); + } catch (IOException e1) { + throw new RuntimeException(e1); + } while (true) { try { if (!source.incrementToken()) break; @@ -202,6 +212,7 @@ } try { + source.end(); source.close(); } catch (IOException e) { // ignore @@ -242,6 +253,7 @@ boolean multipleTokens = false; try { + source.reset(); if (source.incrementToken()) { nextToken = termAtt.toString(); } @@ -251,6 +263,7 @@ } try { + source.end(); source.close(); } catch (IOException e) { // ignore @@ -281,6 +294,7 @@ try { source = getAnalyzer().tokenStream(field, new StringReader(part1)); termAtt = source.addAttribute(CharTermAttribute.class); + source.reset(); multipleTokens = false; @@ -292,6 +306,7 @@ // ignore } try { + source.end(); source.close(); } catch (IOException e) { // ignore @@ -308,6 +323,7 @@ termAtt = source.addAttribute(CharTermAttribute.class); try { + source.reset(); if (source.incrementToken()) { part2 = termAtt.toString(); } @@ -316,6 +332,7 @@ // ignore } try { + source.end(); source.close(); } catch (IOException e) { // ignore Index: lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java =================================================================== --- lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (revision 1102141) +++ lucene/contrib/wordnet/src/test/org/apache/lucene/wordnet/TestSynonymTokenFilter.java (working copy) @@ -111,7 +111,6 @@ setPreviousTokenStream(streams); } else { streams.source.reset(reader); - streams.result.reset(); // reset the SynonymTokenFilter } return streams.result; } Index: lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java =================================================================== --- lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (revision 1102141) +++ lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (working copy) @@ -118,12 +118,14 @@ // [1] Parse query into separate words so that when we expand we can avoid dups TokenStream ts = a.tokenStream( field, new StringReader( query)); CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); - + ts.reset(); while (ts.incrementToken()) { String word = termAtt.toString(); if ( already.add( word)) top.add( word); } + ts.end(); + ts.close(); final BooleanQuery tmp = new BooleanQuery(); // [2] form query Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (revision 1102141) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (working copy) @@ -64,6 +64,7 @@ { Term term = null; BytesRef bytes = termAtt.getBytesRef(); + ts.reset(); while (ts.incrementToken()) { termAtt.fillBytesRef(); if (term == null) @@ -76,6 +77,8 @@ } tf.addTerm(term); } + ts.end(); + ts.close(); } catch (IOException ioe) { Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (revision 1102141) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (working copy) @@ -80,9 +80,12 @@ CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); try { + ts.reset(); while(ts.incrementToken()) { stopWordsSet.add(termAtt.toString()); } + ts.end(); + ts.close(); } catch(IOException ioe) { Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (revision 1102141) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (working copy) @@ -59,11 +59,14 @@ TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value)); TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); BytesRef bytes = termAtt.getBytesRef(); + ts.reset(); while (ts.incrementToken()) { termAtt.fillBytesRef(); SpanTermQuery stq=new SpanTermQuery(new Term(fieldName, new BytesRef(bytes))); clausesList.add(stq); } + ts.end(); + ts.close(); SpanOrQuery soq=new SpanOrQuery(clausesList.toArray(new SpanQuery[clausesList.size()])); soq.setBoost(DOMUtils.getAttribute(e,"boost",1.0f)); return soq; Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (revision 1102141) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (working copy) @@ -61,6 +61,7 @@ TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); Term term = null; BytesRef bytes = termAtt.getBytesRef(); + ts.reset(); while (ts.incrementToken()) { termAtt.fillBytesRef(); if (term == null) @@ -73,6 +74,8 @@ } bq.add(new BooleanClause(new TermQuery(term),BooleanClause.Occur.SHOULD)); } + ts.end(); + ts.close(); } catch (IOException ioe) { Index: lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java =================================================================== --- lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java (revision 1102141) +++ lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java (working copy) @@ -28,32 +28,38 @@ public class OffsetLimitTokenFilterTest extends BaseTokenStreamTestCase { public void testFilter() throws Exception { - TokenStream stream = new MockTokenizer(new StringReader( + // we disable MockTokenizer checks because we will forcefully limit the + // tokenstream and call end() before incrementToken() returns false. + MockTokenizer stream = new MockTokenizer(new StringReader( "short toolong evenmuchlongertext a ab toolong foo"), MockTokenizer.WHITESPACE, false); + stream.setEnableChecks(false); OffsetLimitTokenFilter filter = new OffsetLimitTokenFilter(stream, 10); assertTokenStreamContents(filter, new String[] {"short", "toolong"}); stream = new MockTokenizer(new StringReader( "short toolong evenmuchlongertext a ab toolong foo"), MockTokenizer.WHITESPACE, false); + stream.setEnableChecks(false); filter = new OffsetLimitTokenFilter(stream, 12); assertTokenStreamContents(filter, new String[] {"short", "toolong"}); stream = new MockTokenizer(new StringReader( "short toolong evenmuchlongertext a ab toolong foo"), MockTokenizer.WHITESPACE, false); + stream.setEnableChecks(false); filter = new OffsetLimitTokenFilter(stream, 30); assertTokenStreamContents(filter, new String[] {"short", "toolong", "evenmuchlongertext"}); - + // TODO: This is not actually testing reuse! (reusableTokenStream is not implemented) checkOneTermReuse(new Analyzer() { @Override public TokenStream tokenStream(String fieldName, Reader reader) { - return new OffsetLimitTokenFilter(new MockTokenizer(reader, - MockTokenizer.WHITESPACE, false), 10); + MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false); + tokenizer.setEnableChecks(false); + return new OffsetLimitTokenFilter(tokenizer, 10); } }, "llenges", "llenges"); } Index: lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java =================================================================== --- lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (revision 1102141) +++ lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (working copy) @@ -1093,6 +1093,10 @@ } public void testMaxSizeHighlight() throws Exception { + final MockAnalyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true); + // we disable MockTokenizer checks because we will forcefully limit the + // tokenstream and call end() before incrementToken() returns false. + analyzer.setEnableChecks(false); TestHighlightRunner helper = new TestHighlightRunner() { @Override @@ -1122,7 +1126,10 @@ public void run() throws Exception { String goodWord = "goodtoken"; CharacterRunAutomaton stopWords = new CharacterRunAutomaton(BasicAutomata.makeString("stoppedtoken")); - + // we disable MockTokenizer checks because we will forcefully limit the + // tokenstream and call end() before incrementToken() returns false. + final MockAnalyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords, true); + analyzer.setEnableChecks(false); TermQuery query = new TermQuery(new Term("data", goodWord)); String match; @@ -1134,13 +1141,13 @@ sb.append("stoppedtoken"); } SimpleHTMLFormatter fm = new SimpleHTMLFormatter(); - Highlighter hg = getHighlighter(query, "data", new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords, true).tokenStream( + Highlighter hg = getHighlighter(query, "data", analyzer.tokenStream( "data", new StringReader(sb.toString())), fm);// new Highlighter(fm, // new // QueryTermScorer(query)); hg.setTextFragmenter(new NullFragmenter()); hg.setMaxDocCharsToAnalyze(100); - match = hg.getBestFragment(new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords, true), "data", sb.toString()); + match = hg.getBestFragment(analyzer, "data", sb.toString()); assertTrue("Matched text should be no more than 100 chars in length ", match.length() < hg .getMaxDocCharsToAnalyze()); @@ -1151,7 +1158,7 @@ // + whitespace) sb.append(" "); sb.append(goodWord); - match = hg.getBestFragment(new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords, true), "data", sb.toString()); + match = hg.getBestFragment(analyzer, "data", sb.toString()); assertTrue("Matched text should be no more than 100 chars in length ", match.length() < hg .getMaxDocCharsToAnalyze()); } @@ -1726,6 +1733,11 @@ stream.addAttribute(CharTermAttribute.class); stream.addAttribute(PositionIncrementAttribute.class); stream.addAttribute(OffsetAttribute.class); + try { + stream.reset(); + } catch (IOException e) { + throw new RuntimeException(e); + } return new SynonymTokenizer(stream, synonyms); } } Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java =================================================================== --- lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java (revision 1102141) +++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java (working copy) @@ -355,6 +355,7 @@ { try { + tokenStream.end(); tokenStream.close(); } catch (Exception e) Index: lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (revision 1102141) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (working copy) @@ -885,7 +885,7 @@ int tokenCount=0; // for every token CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); - + ts.reset(); while (ts.incrementToken()) { String word = termAtt.toString(); tokenCount++; @@ -906,6 +906,8 @@ cnt.x++; } } + ts.end(); + ts.close(); } Index: lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (revision 1102141) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (working copy) @@ -192,6 +192,7 @@ int corpusNumDocs=reader.numDocs(); Term internSavingTemplateTerm =new Term(f.fieldName); //optimization to avoid constructing new Term() objects HashSet processedTerms=new HashSet(); + ts.reset(); while (ts.incrementToken()) { String term = termAtt.toString(); @@ -244,7 +245,9 @@ } } } - } + } + ts.end(); + ts.close(); } @Override