Index: lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java =================================================================== --- lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (revision 682416) +++ lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (working copy) @@ -1120,21 +1120,21 @@ { lst = new ArrayList(); Token t; - t = new Token("hi", 0, 2); + t = createToken("hi", 0, 2); lst.add(t); - t = new Token("hispeed", 0, 8); + t = createToken("hispeed", 0, 8); lst.add(t); - t = new Token("speed", 3, 8); + t = createToken("speed", 3, 8); t.setPositionIncrement(0); lst.add(t); - t = new Token("10", 8, 10); + t = createToken("10", 8, 10); lst.add(t); - t = new Token("foo", 11, 14); + t = createToken("foo", 11, 14); lst.add(t); iter = lst.iterator(); } - public Token next() throws IOException { + public Token next(Token token) throws IOException { return iter.hasNext() ? (Token) iter.next() : null; } }; @@ -1149,21 +1149,21 @@ { lst = new ArrayList(); Token t; - t = new Token("hispeed", 0, 8); + t = createToken("hispeed", 0, 8); lst.add(t); - t = new Token("hi", 0, 2); + t = createToken("hi", 0, 2); t.setPositionIncrement(0); lst.add(t); - t = new Token("speed", 3, 8); + t = createToken("speed", 3, 8); lst.add(t); - t = new Token("10", 8, 10); + t = createToken("10", 8, 10); lst.add(t); - t = new Token("foo", 11, 14); + t = createToken("foo", 11, 14); lst.add(t); iter = lst.iterator(); } - public Token next() throws IOException { + public Token next(Token token) throws IOException { return iter.hasNext() ? (Token) iter.next() : null; } }; @@ -1346,6 +1346,13 @@ super.tearDown(); } + private static Token createToken(String term, int start, int offset) + { + Token token = new Token(start, offset); + token.setTermBuffer(term); + return token; + } + } // =================================================================== @@ -1392,31 +1399,33 @@ this.synonyms = synonyms; } - public Token next() throws IOException { + public Token next(Token token) throws IOException { if (currentRealToken == null) { - Token nextRealToken = realStream.next(); + Token nextRealToken = realStream.next(token); if (nextRealToken == null) { return null; } - String expansions = (String) synonyms.get(nextRealToken.termText()); + String expansions = (String) synonyms.get(nextRealToken.term()); if (expansions == null) { return nextRealToken; } st = new StringTokenizer(expansions, ","); if (st.hasMoreTokens()) { - currentRealToken = nextRealToken; + currentRealToken = (Token) nextRealToken.clone(); } return currentRealToken; } else { - String nextExpandedValue = st.nextToken(); - Token expandedToken = new Token(nextExpandedValue, currentRealToken.startOffset(), - currentRealToken.endOffset()); - expandedToken.setPositionIncrement(0); + token.clear(); + token.setTermBuffer(st.nextToken()); + token.setStartOffset(currentRealToken.startOffset()); + token.setEndOffset(currentRealToken.endOffset()); + token.setType(Token.DEFAULT_TYPE); + token.setPositionIncrement(0); if (!st.hasMoreTokens()) { currentRealToken = null; st = null; } - return expandedToken; + return token; } } Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SpanScorer.java =================================================================== --- lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SpanScorer.java (revision 682416) +++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SpanScorer.java (working copy) @@ -121,7 +121,7 @@ */ public float getTokenScore(Token token) { position += token.getPositionIncrement(); - String termText = new String(token.termBuffer(), 0, token.termLength()); + String termText = token.term(); WeightedSpanTerm weightedSpanTerm; Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java =================================================================== --- lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java (revision 682416) +++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java (working copy) @@ -106,7 +106,7 @@ */ public float getTokenScore(Token token) { - String termText=token.termText(); + String termText=token.term(); WeightedTerm queryTerm=(WeightedTerm) termsToFind.get(termText); if(queryTerm==null) Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java =================================================================== --- lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (revision 682416) +++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java (working copy) @@ -147,7 +147,7 @@ { this.tokens=tokens; } - public Token next() + public Token next(Token token) { if(currentToken>=tokens.length) { @@ -160,6 +160,7 @@ String[] terms=tpv.getTerms(); int[] freq=tpv.getTermFrequencies(); int totalTokens=0; + Token newToken = new Token(); for (int t = 0; t < freq.length; t++) { totalTokens+=freq[t]; @@ -189,9 +190,12 @@ } for (int tp = 0; tp < offsets.length; tp++) { - unsortedTokens.add(new Token(terms[t], - offsets[tp].getStartOffset(), - offsets[tp].getEndOffset())); + newToken.clear(); + newToken.setTermBuffer(terms[t]); + newToken.setStartOffset(offsets[tp].getStartOffset()); + newToken.setEndOffset(offsets[tp].getEndOffset()); + newToken.setType(Token.DEFAULT_TYPE); + unsortedTokens.add(newToken.clone()); } } else @@ -204,9 +208,12 @@ //tokens stored with positions - can use this to index straight into sorted array for (int tp = 0; tp < pos.length; tp++) { - tokensInOriginalOrder[pos[tp]]=new Token(terms[t], - offsets[tp].getStartOffset(), - offsets[tp].getEndOffset()); + newToken.clear(); + newToken.setTermBuffer(terms[t]); + newToken.setStartOffset(offsets[tp].getStartOffset()); + newToken.setEndOffset(offsets[tp].getEndOffset()); + newToken.setType(Token.DEFAULT_TYPE); + tokensInOriginalOrder[pos[tp]] = (Token) newToken.clone(); } } } @@ -261,7 +268,7 @@ } return getTokenStream(field, contents, analyzer); } - //conevenience method + //convenience method public static TokenStream getTokenStream(String field, String contents, Analyzer analyzer){ return analyzer.tokenStream(field,new StringReader(contents)); } Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java =================================================================== --- lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java (revision 682416) +++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java (working copy) @@ -62,7 +62,7 @@ return false; } - WeightedSpanTerm wSpanTerm = spanScorer.getWeightedSpanTerm(new String(token.termBuffer(), 0, token.termLength())); + WeightedSpanTerm wSpanTerm = spanScorer.getWeightedSpanTerm(token.term()); if (wSpanTerm != null) { List positionSpans = wSpanTerm.getPositionSpans(); Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java =================================================================== --- lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java (revision 682416) +++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java (working copy) @@ -61,7 +61,7 @@ tot+=score; } } - tokens[numTokens]=token; + tokens[numTokens]= (Token) token.clone(); scores[numTokens]=score; numTokens++; } Index: lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java =================================================================== --- lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java (revision 682416) +++ lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java (working copy) @@ -22,6 +22,7 @@ import java.util.Iterator; import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.util.PriorityQueue; @@ -217,7 +218,7 @@ try { - org.apache.lucene.analysis.Token token; + Token token = new Token(); String tokenText; int startOffset; int endOffset; @@ -225,7 +226,7 @@ textFragmenter.start(text); TokenGroup tokenGroup=new TokenGroup(); - token = tokenStream.next(); + token = tokenStream.next(token); while ((token!= null)&&(token.startOffset()< maxDocCharsToAnalyze)) { if((tokenGroup.numTokens>0)&&(tokenGroup.isDistinct(token))) @@ -261,7 +262,7 @@ // { // break; // } - token = tokenStream.next(); + token = tokenStream.next(token); } currentFrag.setScore(fragmentScorer.getFragmentScore());