Index: lucene/src/java/org/apache/lucene/search/QueryTermVector.java =================================================================== --- lucene/src/java/org/apache/lucene/search/QueryTermVector.java (revision 1102141) +++ lucene/src/java/org/apache/lucene/search/QueryTermVector.java (working copy) @@ -55,7 +55,12 @@ public QueryTermVector(String queryString, Analyzer analyzer) { if (analyzer != null) { - TokenStream stream = analyzer.tokenStream("", new StringReader(queryString)); + TokenStream stream; + try { + stream = analyzer.reusableTokenStream("", new StringReader(queryString)); + } catch (IOException e1) { + stream = null; + } if (stream != null) { List terms = new ArrayList(); Index: lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java =================================================================== --- lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (revision 1102290) +++ lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/processors/AnalyzerQueryNodeProcessor.java (working copy) @@ -121,9 +121,9 @@ String text = fieldNode.getTextAsString(); String field = fieldNode.getFieldAsString(); - TokenStream source = this.analyzer.tokenStream(field, new StringReader( - text)); + TokenStream source; try { + source = this.analyzer.reusableTokenStream(field, new StringReader(text)); source.reset(); } catch (IOException e1) { throw new RuntimeException(e1); Index: lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java =================================================================== --- lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (revision 1102290) +++ lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/analyzing/AnalyzingQueryParser.java (working copy) @@ -106,15 +106,16 @@ } // get Analyzer from superclass and tokenize the term - TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr)); - CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); + TokenStream source; int countTokens = 0; try { + source = getAnalyzer().reusableTokenStream(field, new StringReader(termStr)); source.reset(); } catch (IOException e1) { throw new RuntimeException(e1); } + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); while (true) { try { if (!source.incrementToken()) break; @@ -194,14 +195,15 @@ @Override protected Query getPrefixQuery(String field, String termStr) throws ParseException { // get Analyzer from superclass and tokenize the term - TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr)); + TokenStream source; List tlist = new ArrayList(); - CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); try { + source = getAnalyzer().reusableTokenStream(field, new StringReader(termStr)); source.reset(); } catch (IOException e1) { throw new RuntimeException(e1); } + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); while (true) { try { if (!source.incrementToken()) break; @@ -247,12 +249,13 @@ protected Query getFuzzyQuery(String field, String termStr, float minSimilarity) throws ParseException { // get Analyzer from superclass and tokenize the term - TokenStream source = getAnalyzer().tokenStream(field, new StringReader(termStr)); - CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); + TokenStream source = null; String nextToken = null; boolean multipleTokens = false; try { + source = getAnalyzer().reusableTokenStream(field, new StringReader(termStr)); + CharTermAttribute termAtt = source.addAttribute(CharTermAttribute.class); source.reset(); if (source.incrementToken()) { nextToken = termAtt.toString(); @@ -292,7 +295,7 @@ if (part1 != null) { // part1 try { - source = getAnalyzer().tokenStream(field, new StringReader(part1)); + source = getAnalyzer().reusableTokenStream(field, new StringReader(part1)); termAtt = source.addAttribute(CharTermAttribute.class); source.reset(); multipleTokens = false; @@ -318,11 +321,10 @@ } if (part2 != null) { - // part2 - source = getAnalyzer().tokenStream(field, new StringReader(part2)); - termAtt = source.addAttribute(CharTermAttribute.class); - try { + // part2 + source = getAnalyzer().reusableTokenStream(field, new StringReader(part2)); + termAtt = source.addAttribute(CharTermAttribute.class); source.reset(); if (source.incrementToken()) { part2 = termAtt.toString(); Index: lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java =================================================================== --- lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java (revision 1102141) +++ lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynLookup.java (working copy) @@ -124,7 +124,7 @@ List top = new LinkedList(); // needs to be separately listed.. // [1] Parse query into separate words so that when we expand we can avoid dups - TokenStream ts = a.tokenStream( field, new StringReader( query)); + TokenStream ts = a.reusableTokenStream( field, new StringReader( query)); CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); while (ts.incrementToken()) { Index: lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java =================================================================== --- lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (revision 1102290) +++ lucene/contrib/wordnet/src/java/org/apache/lucene/wordnet/SynExpand.java (working copy) @@ -116,7 +116,7 @@ if ( a == null) a = new StandardAnalyzer(Version.LUCENE_CURRENT); // [1] Parse query into separate words so that when we expand we can avoid dups - TokenStream ts = a.tokenStream( field, new StringReader( query)); + TokenStream ts = a.reusableTokenStream( field, new StringReader( query)); CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); ts.reset(); while (ts.incrementToken()) { Index: lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java =================================================================== --- lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (revision 1102141) +++ lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java (working copy) @@ -532,7 +532,7 @@ if (field.tokenStreamValue() != null) { tokenStream = field.tokenStreamValue(); } else { - tokenStream = analyzer.tokenStream(field.name(), new StringReader(field.stringValue())); + tokenStream = analyzer.reusableTokenStream(field.name(), new StringReader(field.stringValue())); } // reset the TokenStream to the first token Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (revision 1102290) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsFilterBuilder.java (working copy) @@ -57,11 +57,11 @@ TermsFilter tf = new TermsFilter(); String text = DOMUtils.getNonBlankTextOrFail(e); String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName"); - TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text)); - TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); try { + TokenStream ts = analyzer.reusableTokenStream(fieldName, new StringReader(text)); + TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); Term term = null; BytesRef bytes = termAtt.getBytesRef(); ts.reset(); Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (revision 1102290) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/LikeThisQueryBuilder.java (working copy) @@ -76,10 +76,10 @@ stopWordsSet=new HashSet(); for (int i = 0; i < fields.length; i++) { - TokenStream ts = analyzer.tokenStream(fields[i],new StringReader(stopWords)); - CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); try { + TokenStream ts = analyzer.reusableTokenStream(fields[i],new StringReader(stopWords)); + CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); ts.reset(); while(ts.incrementToken()) { stopWordsSet.add(termAtt.toString()); Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (revision 1102290) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/SpanOrTermsBuilder.java (working copy) @@ -56,7 +56,7 @@ try { ArrayList clausesList=new ArrayList(); - TokenStream ts=analyzer.tokenStream(fieldName,new StringReader(value)); + TokenStream ts=analyzer.reusableTokenStream(fieldName,new StringReader(value)); TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); BytesRef bytes = termAtt.getBytesRef(); ts.reset(); Index: lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java =================================================================== --- lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (revision 1102290) +++ lucene/contrib/xml-query-parser/src/java/org/apache/lucene/xmlparser/builders/TermsQueryBuilder.java (working copy) @@ -55,9 +55,9 @@ BooleanQuery bq=new BooleanQuery(DOMUtils.getAttribute(e,"disableCoord",false)); bq.setMinimumNumberShouldMatch(DOMUtils.getAttribute(e,"minimumNumberShouldMatch",0)); - TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text)); try { + TokenStream ts = analyzer.reusableTokenStream(fieldName, new StringReader(text)); TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class); Term term = null; BytesRef bytes = termAtt.getBytesRef(); Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java =================================================================== --- lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (revision 1102141) +++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (working copy) @@ -279,7 +279,11 @@ // convenience method public static TokenStream getTokenStream(String field, String contents, Analyzer analyzer) { - return analyzer.tokenStream(field, new StringReader(contents)); + try { + return analyzer.reusableTokenStream(field, new StringReader(contents)); + } catch (IOException ex) { + throw new RuntimeException(ex); + } } } Index: lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java =================================================================== --- lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 1102141) +++ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy) @@ -261,8 +261,12 @@ if (analyzer == null) throw new IllegalArgumentException("analyzer must not be null"); - TokenStream stream = analyzer.tokenStream(fieldName, - new StringReader(text)); + TokenStream stream; + try { + stream = analyzer.reusableTokenStream(fieldName, new StringReader(text)); + } catch (IOException ex) { + throw new RuntimeException(ex); + } addField(fieldName, stream); } Index: lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (revision 1102290) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (working copy) @@ -881,7 +881,7 @@ throw new UnsupportedOperationException("To use MoreLikeThis without " + "term vectors, you must provide an Analyzer"); } - TokenStream ts = analyzer.tokenStream(fieldName, r); + TokenStream ts = analyzer.reusableTokenStream(fieldName, r); int tokenCount=0; // for every token CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); Index: lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java (revision 1102141) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java (working copy) @@ -85,7 +85,7 @@ Set stop) throws IOException { - TokenStream ts = a.tokenStream( field, new StringReader( body)); + TokenStream ts = a.reusableTokenStream( field, new StringReader( body)); CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); BooleanQuery tmp = new BooleanQuery(); Index: lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (revision 1102290) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (working copy) @@ -186,7 +186,7 @@ private void addTerms(IndexReader reader,FieldVals f) throws IOException { if(f.queryString==null) return; - TokenStream ts=analyzer.tokenStream(f.fieldName,new StringReader(f.queryString)); + TokenStream ts=analyzer.reusableTokenStream(f.fieldName,new StringReader(f.queryString)); CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class); int corpusNumDocs=reader.numDocs();