Index: lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (revision 682416) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java (working copy) @@ -28,6 +28,7 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.Hits; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; @@ -808,10 +809,10 @@ throws IOException { TokenStream ts = analyzer.tokenStream(fieldName, r); - org.apache.lucene.analysis.Token token; int tokenCount=0; - while ((token = ts.next()) != null) { // for every token - String word = token.termText(); + // for every token + for (Token token = ts.next(new Token()); token != null; token = ts.next(token)) { + String word = token.term(); tokenCount++; if(tokenCount>maxNumTokensParsed) { @@ -872,7 +873,7 @@ * For an easier method to call see {@link #retrieveInterestingTerms retrieveInterestingTerms()}. * * @param r the reader that has the content of the document - * @return the most intresting words in the document ordered by score, with the highest scoring, or best entry, first + * @return the most interesting words in the document ordered by score, with the highest scoring, or best entry, first * * @see #retrieveInterestingTerms */ Index: lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java (revision 682416) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/similar/SimilarityQueries.java (working copy) @@ -21,6 +21,7 @@ import java.util.Set; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.Term; import org.apache.lucene.search.BooleanClause; @@ -85,12 +86,10 @@ throws IOException { TokenStream ts = a.tokenStream( field, new StringReader( body)); - org.apache.lucene.analysis.Token t; BooleanQuery tmp = new BooleanQuery(); Set already = new HashSet(); // ignore dups - while ( (t = ts.next()) != null) - { - String word = t.termText(); + for (Token token = ts.next(new Token()); token != null; token = ts.next(token)) { + String word = token.term(); // ignore opt stop words if ( stop != null && stop.contains( word)) continue; Index: lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java =================================================================== --- lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (revision 682416) +++ lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java (working copy) @@ -104,18 +104,20 @@ { if(f.queryString==null) return; TokenStream ts=analyzer.tokenStream(f.fieldName,new StringReader(f.queryString)); - Token token=ts.next(); + Token token=new Token(); + token = ts.next(token); int corpusNumDocs=reader.numDocs(); Term internSavingTemplateTerm =new Term(f.fieldName,""); //optimization to avoid constructing new Term() objects HashSet processedTerms=new HashSet(); while(token!=null) - { - if(!processedTerms.contains(token.termText())) + { + String term = token.term(); + if(!processedTerms.contains(term)) { - processedTerms.add(token.termText()); + processedTerms.add(term); ScoreTermQueue variantsQ=new ScoreTermQueue(MAX_VARIANTS_PER_TERM); //maxNum variants considered for any one term float minScore=0; - Term startTerm=internSavingTemplateTerm.createTerm(token.termText()); + Term startTerm=internSavingTemplateTerm.createTerm(term); FuzzyTermEnum fe=new FuzzyTermEnum(reader,startTerm,f.minSimilarity,f.prefixLength); TermEnum origEnum = reader.terms(startTerm); int df=0; @@ -162,7 +164,7 @@ q.insert(st); } } - token=ts.next(); + token=ts.next(token); } }