Index: lucene/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java =================================================================== --- lucene/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java (revision 682416) +++ lucene/contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java (working copy) @@ -197,9 +197,8 @@ private List getTokens(TokenStream stream) throws IOException { ArrayList tokens = new ArrayList(); - Token token; - while ((token = stream.next()) != null) { - tokens.add(token); + for (Token token = stream.next(new Token()); token != null; token = stream.next(token)) { + tokens.add(token.clone()); } return tokens; } @@ -211,7 +210,7 @@ for (; i < size; i++) { Token t1 = (Token) tokens1.get(i); Token t2 = (Token) tokens2.get(i); - if (!(t1.termText().equals(t2.termText()))) throw new IllegalStateException("termText"); + if (!(t1.term().equals(t2.term()))) throw new IllegalStateException("termText"); if (t1.startOffset() != t2.startOffset()) throw new IllegalStateException("startOffset"); if (t1.endOffset() != t2.endOffset()) throw new IllegalStateException("endOffset"); if (!(t1.type().equals(t2.type()))) throw new IllegalStateException("type"); @@ -222,8 +221,8 @@ catch (IllegalStateException e) { if (size > 0) { System.out.println("i=" + i + ", size=" + size); - System.out.println("t1[size]='" + ((Token) tokens1.get(size-1)).termText() + "'"); - System.out.println("t2[size]='" + ((Token) tokens2.get(size-1)).termText() + "'"); + System.out.println("t1[size]='" + ((Token) tokens1.get(size-1)).term() + "'"); + System.out.println("t2[size]='" + ((Token) tokens2.get(size-1)).term() + "'"); } throw e; } @@ -234,7 +233,7 @@ String str = "["; for (int i=0; i < tokens.size(); i++) { Token t1 = (Token) tokens.get(i); - str = str + "'" + t1.termText() + "', "; + str = str + "'" + t1.term() + "', "; } return str + "]"; } Index: lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java =================================================================== --- lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (revision 682416) +++ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (working copy) @@ -275,7 +275,7 @@ return new TokenStream() { private Iterator iter = keywords.iterator(); private int start = 0; - public Token next() { + public Token next(Token token) { if (!iter.hasNext()) return null; Object obj = iter.next(); @@ -283,7 +283,11 @@ throw new IllegalArgumentException("keyword must not be null"); String term = obj.toString(); - Token token = new Token(term, start, start + term.length()); + token.clear(); + token.setTermBuffer(term); + token.setStartOffset(start); + token.setEndOffset(start + token.termLength()); + token.setType(Token.DEFAULT_TYPE); start += term.length() + 1; // separate words by 1 (blank) character return token; } @@ -349,10 +353,8 @@ HashMap terms = new HashMap(); int numTokens = 0; int pos = -1; - Token token; - - while ((token = stream.next()) != null) { - String term = token.termText(); + for (Token token = stream.next(new Token()); token != null; token = stream.next(token)) { + String term = token.term(); if (term.length() == 0) continue; // nothing to do // if (DEBUG) System.err.println("token='" + term + "'"); numTokens++; Index: lucene/contrib/memory/src/java/org/apache/lucene/index/memory/AnalyzerUtil.java =================================================================== --- lucene/contrib/memory/src/java/org/apache/lucene/index/memory/AnalyzerUtil.java (revision 682416) +++ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/AnalyzerUtil.java (working copy) @@ -73,8 +73,8 @@ return new TokenFilter(child.tokenStream(fieldName, reader)) { private int position = -1; - public Token next() throws IOException { - Token token = input.next(); // from filter super class + public Token next(Token token) throws IOException { + token = input.next(token); // from filter super class log.println(toString(token)); return token; } @@ -84,7 +84,7 @@ position += token.getPositionIncrement(); return "[" + logName + ":" + position + ":" + fieldName + ":" - + token.termText() + ":" + token.startOffset() + + token.term() + ":" + token.startOffset() + "-" + token.endOffset() + ":" + token.type() + "]"; } @@ -121,8 +121,8 @@ return new TokenFilter(child.tokenStream(fieldName, reader)) { private int todo = maxTokens; - public Token next() throws IOException { - return --todo >= 0 ? input.next() : null; + public Token next(Token token) throws IOException { + return --todo >= 0 ? input.next(token) : null; } }; } @@ -239,9 +239,9 @@ final ArrayList tokens2 = new ArrayList(); TokenStream tokenStream = new TokenFilter(child.tokenStream(fieldName, reader)) { - public Token next() throws IOException { - Token token = input.next(); // from filter super class - if (token != null) tokens2.add(token); + public Token next(Token token) throws IOException { + token = input.next(token); // from filter super class + if (token != null) tokens2.add(token.clone()); return token; } }; @@ -253,7 +253,7 @@ private Iterator iter = tokens.iterator(); - public Token next() { + public Token next(Token token) { if (!iter.hasNext()) return null; return (Token) iter.next(); } @@ -300,12 +300,11 @@ HashMap map = new HashMap(); TokenStream stream = analyzer.tokenStream("", new StringReader(text)); try { - Token token; - while ((token = stream.next()) != null) { - MutableInteger freq = (MutableInteger) map.get(token.termText()); + for (Token token = stream.next(new Token()); token != null; token = stream.next(token)) { + MutableInteger freq = (MutableInteger) map.get(token.term()); if (freq == null) { freq = new MutableInteger(1); - map.put(token.termText(), freq); + map.put(token.term(), freq); } else { freq.setValue(freq.intValue() + 1); } Index: lucene/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java =================================================================== --- lucene/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java (revision 682416) +++ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/PatternAnalyzer.java (working copy) @@ -334,7 +334,7 @@ this.toLowerCase = toLowerCase; } - public Token next() { + public Token next(Token token) { if (matcher == null) return null; while (true) { // loop takes care of leading and trailing boundary cases @@ -352,7 +352,12 @@ if (start != end) { // non-empty match (header/trailer) String text = str.substring(start, end); if (toLowerCase) text = text.toLowerCase(locale); - return new Token(text, start, end); + token.clear(); + token.setTermBuffer(text); + token.setStartOffset(start); + token.setEndOffset(end); + token.setType(Token.DEFAULT_TYPE); + return token; } if (!isMatch) return null; } @@ -384,7 +389,7 @@ this.stopWords = stopWords; } - public Token next() { + public Token next(Token token) { // cache loop instance vars (performance) String s = str; int len = s.length(); @@ -422,7 +427,16 @@ } while (text != null && isStopWord(text)); pos = i; - return text != null ? new Token(text, start, i) : null; + if (text == null) + { + return null; + } + token.clear(); + token.setTermBuffer(text); + token.setStartOffset(start); + token.setEndOffset(i); + token.setType(Token.DEFAULT_TYPE); + return token; } private boolean isTokenChar(char c, boolean isLetter) { Index: lucene/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java =================================================================== --- lucene/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java (revision 682416) +++ lucene/contrib/memory/src/java/org/apache/lucene/index/memory/SynonymTokenFilter.java (working copy) @@ -68,23 +68,22 @@ } /** Returns the next token in the stream, or null at EOS. */ - public Token next() throws IOException { - Token token; + public Token next(Token token) throws IOException { while (todo > 0 && index < stack.length) { // pop from stack - token = createToken(stack[index++], current); + token = createToken(stack[index++], current, token); if (token != null) { todo--; return token; } } - token = input.next(); + token = input.next(token); if (token == null) return null; // EOS; iterator exhausted - stack = synonyms.getSynonyms(token.termText()); // push onto stack + stack = synonyms.getSynonyms(token.term()); // push onto stack if (stack.length > maxSynonyms) randomize(stack); index = 0; - current = token; + current = (Token) token.clone(); todo = maxSynonyms; return token; } @@ -100,11 +99,14 @@ * @return a new token, or null to indicate that the given synonym should be * ignored */ - protected Token createToken(String synonym, Token current) { - Token token = new Token( - synonym, current.startOffset(), current.endOffset(), SYNONYM_TOKEN_TYPE); - token.setPositionIncrement(0); - return token; + protected Token createToken(String synonym, Token current, Token result) { + result.clear(); + result.setTermBuffer(synonym); + result.setStartOffset(current.startOffset()); + result.setEndOffset(current.endOffset()); + result.setType(SYNONYM_TOKEN_TYPE); + result.setPositionIncrement(0); + return result; } /**