Index: lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java =================================================================== --- lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java (revision 682416) +++ lucene/src/test/org/apache/lucene/queryParser/TestMultiAnalyzer.java (working copy) @@ -141,34 +141,38 @@ private final class TestFilter extends TokenFilter { - private org.apache.lucene.analysis.Token prevToken; + private Token prevToken; public TestFilter(TokenStream in) { super(in); } - public final org.apache.lucene.analysis.Token next() throws java.io.IOException { + public final Token next(Token token) throws java.io.IOException { if (multiToken > 0) { - org.apache.lucene.analysis.Token token = - new org.apache.lucene.analysis.Token("multi"+(multiToken+1), prevToken.startOffset(), - prevToken.endOffset(), prevToken.type()); + token.clear(); + token.setTermBuffer("multi"+(multiToken+1)); + token.setStartOffset(prevToken.startOffset()); + token.setEndOffset(prevToken.endOffset()); + token.setType(prevToken.type()); token.setPositionIncrement(0); multiToken--; return token; } else { - org.apache.lucene.analysis.Token t = input.next(); - prevToken = t; - if (t == null) + token = input.next(token); + if (token == null) { + prevToken = null; return null; - String text = t.termText(); + } + prevToken = (Token) token.clone(); + String text = token.term(); if (text.equals("triplemulti")) { multiToken = 2; - return t; + return token; } else if (text.equals("multi")) { multiToken = 1; - return t; + return token; } else { - return t; + return token; } } } @@ -197,20 +201,14 @@ super(in); } - public final org.apache.lucene.analysis.Token next() throws java.io.IOException { - for (Token t = input.next(); t != null; t = input.next()) { - if (t.termText().equals("the")) { + public final Token next(Token token) throws java.io.IOException { + for (token = input.next(token); token != null; token = input.next(token)) { + if (token.term().equals("the")) { // stopword, do nothing - } else if (t.termText().equals("quick")) { - org.apache.lucene.analysis.Token token = - new org.apache.lucene.analysis.Token(t.termText(), t.startOffset(), - t.endOffset(), t.type()); + } else if (token.term().equals("quick")) { token.setPositionIncrement(2); return token; } else { - org.apache.lucene.analysis.Token token = - new org.apache.lucene.analysis.Token(t.termText(), t.startOffset(), - t.endOffset(), t.type()); token.setPositionIncrement(1); return token; } Index: lucene/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java =================================================================== --- lucene/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java (revision 682416) +++ lucene/src/test/org/apache/lucene/queryParser/TestMultiFieldQueryParser.java (working copy) @@ -319,7 +319,7 @@ } private static class EmptyTokenStream extends TokenStream { - public Token next() { + public Token next(Token token) { return null; } } Index: lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java =================================================================== --- lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java (revision 682416) +++ lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java (working copy) @@ -75,19 +75,29 @@ boolean inPhrase = false; int savedStart = 0, savedEnd = 0; - public Token next() throws IOException { + public Token next(Token token) throws IOException { if (inPhrase) { inPhrase = false; - return new Token("phrase2", savedStart, savedEnd); + token.clear(); + token.setTermBuffer("phrase2"); + token.setStartOffset(savedStart); + token.setEndOffset(savedEnd); + token.setType(Token.DEFAULT_TYPE); + return token; } else - for (Token token = input.next(); token != null; token = input.next()) { - if (token.termText().equals("phrase")) { + for (token = input.next(token); token != null; token = input.next(token)) { + if (token.term().equals("phrase")) { inPhrase = true; savedStart = token.startOffset(); savedEnd = token.endOffset(); - return new Token("phrase1", savedStart, savedEnd); - } else if (!token.termText().equals("stop")) + token.clear(); + token.setTermBuffer("phrase1"); + token.setStartOffset(savedStart); + token.setEndOffset(savedEnd); + token.setType(Token.DEFAULT_TYPE); return token; + } else if (!token.term().equals("stop")) + return token; } return null; } Index: lucene/src/test/org/apache/lucene/AnalysisTest.java =================================================================== --- lucene/src/test/org/apache/lucene/AnalysisTest.java (revision 682416) +++ lucene/src/test/org/apache/lucene/AnalysisTest.java (working copy) @@ -70,9 +70,9 @@ Date start = new Date(); int count = 0; - for (Token t = stream.next(); t!=null; t = stream.next()) { + for (Token t = stream.next(new Token()); t!=null; t = stream.next(t)) { if (verbose) { - System.out.println("Text=" + new String(t.termBuffer(), 0, t.termLength()) + System.out.println("Text=" + t.term() + " start=" + t.startOffset() + " end=" + t.endOffset()); } Index: lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java =================================================================== --- lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java (revision 682416) +++ lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java (working copy) @@ -49,13 +49,17 @@ private final int[] INCREMENTS = {1, 2, 1, 0, 1}; private int i = 0; - public Token next() { + public Token next(Token token) { if (i == TOKENS.length) return null; - Token t = new Token(TOKENS[i], i, i); - t.setPositionIncrement(INCREMENTS[i]); + token.clear(); + token.setTermBuffer(TOKENS[i]); + token.setStartOffset(i); + token.setEndOffset(i); + token.setType(Token.DEFAULT_TYPE); + token.setPositionIncrement(INCREMENTS[i]); i++; - return t; + return token; } }; } @@ -204,11 +208,8 @@ Analyzer analyzer = new WhitespaceAnalyzer(); TokenStream ts = analyzer.tokenStream("field", new StringReader("one two three four five")); - - while (true) { - Token token = ts.next(); - if (token == null) break; - assertEquals(token.termText(), 1, token.getPositionIncrement()); + for (Token token = ts.next(new Token()); token != null; token = ts.next(token)) { + assertEquals(token.term(), 1, token.getPositionIncrement()); } } } Index: lucene/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java =================================================================== --- lucene/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java (revision 682416) +++ lucene/src/test/org/apache/lucene/search/payloads/TestBoostingTermQuery.java (working copy) @@ -16,22 +16,32 @@ * limitations under the License. */ -import org.apache.lucene.util.LuceneTestCase; -import org.apache.lucene.analysis.*; +import java.io.IOException; +import java.io.Reader; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.LowerCaseTokenizer; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.Payload; import org.apache.lucene.index.Term; -import org.apache.lucene.search.*; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.CheckHits; +import org.apache.lucene.search.DefaultSimilarity; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreDoc; +import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.spans.Spans; import org.apache.lucene.search.spans.TermSpans; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.English; +import org.apache.lucene.util.LuceneTestCase; -import java.io.IOException; -import java.io.Reader; - public class TestBoostingTermQuery extends LuceneTestCase { private IndexSearcher searcher; private BoostingSimilarity similarity = new BoostingSimilarity(); @@ -62,8 +72,8 @@ this.fieldName = fieldName; } - public Token next() throws IOException { - Token result = input.next(); + public Token next(Token token) throws IOException { + Token result = input.next(token); if (result != null) { if (fieldName.equals("field")) { result.setPayload(new Payload(payloadField)); Index: lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (revision 682416) +++ lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java (working copy) @@ -118,20 +118,21 @@ private class MyTokenStream extends TokenStream { int tokenUpto; - public Token next() { + public Token next(Token token) { if (tokenUpto >= tokens.length) return null; else { - final Token t = new Token(); final TestToken testToken = tokens[tokenUpto++]; - t.setTermText(testToken.text); + token.clear(); + token.setTermBuffer(testToken.text); if (tokenUpto > 1) - t.setPositionIncrement(testToken.pos - tokens[tokenUpto-2].pos); + token.setPositionIncrement(testToken.pos - tokens[tokenUpto-2].pos); else - t.setPositionIncrement(testToken.pos+1); - t.setStartOffset(testToken.startOffset); - t.setEndOffset(testToken.endOffset); - return t; + token.setPositionIncrement(testToken.pos+1); + token.setStartOffset(testToken.startOffset); + token.setEndOffset(testToken.endOffset); + token.setType(Token.DEFAULT_TYPE); + return token; } } } Index: lucene/src/test/org/apache/lucene/index/TestIndexWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (revision 682416) +++ lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (working copy) @@ -1786,11 +1786,11 @@ return new TokenFilter(new StandardTokenizer(reader)) { private int count = 0; - public Token next() throws IOException { + public Token next(Token token) throws IOException { if (count++ == 5) { throw new IOException(); } - return input.next(); + return input.next(token); } }; } @@ -3574,13 +3574,13 @@ public void testNegativePositions() throws Throwable { SinkTokenizer tokens = new SinkTokenizer(); Token t = new Token(); - t.setTermText("a"); + t.setTermBuffer("a"); t.setPositionIncrement(0); tokens.add(t); - t.setTermText("b"); + t.setTermBuffer("b"); t.setPositionIncrement(1); tokens.add(t); - t.setTermText("c"); + t.setTermBuffer("c"); tokens.add(t); MockRAMDirectory dir = new MockRAMDirectory(); Index: lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (revision 682416) +++ lucene/src/test/org/apache/lucene/index/TestMultiLevelSkipList.java (working copy) @@ -103,12 +103,12 @@ super(input); } - public Token next() throws IOException { - Token t = input.next(); - if (t != null) { - t.setPayload(new Payload(new byte[] { (byte) count++ })); + public Token next(Token token) throws IOException { + token = input.next(token); + if (token != null) { + token.setPayload(new Payload(new byte[] { (byte) count++ })); } - return t; + return token; } } Index: lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java (revision 682416) +++ lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java (working copy) @@ -40,10 +40,11 @@ Token t; public RepeatingTokenStream(String val) { - t = new Token(val,0,val.length()); + t = new Token(0,val.length()); + t.setTermBuffer(val); } - public Token next() throws IOException { + public Token next(Token token) throws IOException { return --num<0 ? null : t; } } Index: lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (revision 682416) +++ lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java (working copy) @@ -17,21 +17,27 @@ * limitations under the License. */ -import org.apache.lucene.analysis.*; +import java.io.IOException; +import java.io.Reader; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.WhitespaceAnalyzer; +import org.apache.lucene.analysis.WhitespaceTokenizer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field.TermVector; -import org.apache.lucene.document.Fieldable; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; -import java.io.IOException; -import java.io.Reader; - public class TestDocumentWriter extends LuceneTestCase { private RAMDirectory dir; @@ -134,10 +140,6 @@ boolean first=true; Token buffered; - public Token next() throws IOException { - return input.next(); - } - public Token next(Token result) throws IOException { if (buffered != null) { Token t = buffered; @@ -199,11 +201,16 @@ private String[] tokens = new String[] {"term1", "term2", "term3", "term2"}; private int index = 0; - public Token next() throws IOException { + public Token next(Token token) throws IOException { if (index == tokens.length) { return null; } else { - return new Token(tokens[index++], 0, 0); + token.clear(); + token.setTermBuffer(tokens[index++]); + token.setStartOffset(0); + token.setEndOffset(0); + token.setType(Token.DEFAULT_TYPE); + return token; } } Index: lucene/src/test/org/apache/lucene/index/TestPayloads.java =================================================================== --- lucene/src/test/org/apache/lucene/index/TestPayloads.java (revision 682416) +++ lucene/src/test/org/apache/lucene/index/TestPayloads.java (working copy) @@ -536,11 +536,14 @@ first = true; } - public Token next() throws IOException { + public Token next(Token token) throws IOException { if (!first) return null; - Token t = new Token(term, 0, 0); - t.setPayload(new Payload(payload)); - return t; + token.setTermBuffer(term); + token.setStartOffset(0); + token.setEndOffset(0); + token.setType(Token.DEFAULT_TYPE); + token.setPayload(new Payload(payload)); + return token; } public void close() throws IOException { Index: lucene/src/java/org/apache/lucene/queryParser/QueryParser.java =================================================================== --- lucene/src/java/org/apache/lucene/queryParser/QueryParser.java (revision 682416) +++ lucene/src/java/org/apache/lucene/queryParser/QueryParser.java (working copy) @@ -1,14 +1,35 @@ /* Generated By:JavaCC: Do not edit this line. QueryParser.java */ package org.apache.lucene.queryParser; +import java.io.IOException; +import java.io.StringReader; +import java.text.DateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; import java.util.Vector; -import java.io.*; -import java.text.*; -import java.util.*; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.DateField; +import org.apache.lucene.document.DateTools; import org.apache.lucene.index.Term; -import org.apache.lucene.analysis.*; -import org.apache.lucene.document.*; -import org.apache.lucene.search.*; +import org.apache.lucene.search.BooleanClause; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreRangeQuery; +import org.apache.lucene.search.FuzzyQuery; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.MultiPhraseQuery; +import org.apache.lucene.search.PhraseQuery; +import org.apache.lucene.search.PrefixQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.RangeQuery; +import org.apache.lucene.search.TermQuery; +import org.apache.lucene.search.WildcardQuery; import org.apache.lucene.util.Parameter; /** @@ -451,20 +472,20 @@ TokenStream source = analyzer.tokenStream(field, new StringReader(queryText)); Vector v = new Vector(); - org.apache.lucene.analysis.Token t; + org.apache.lucene.analysis.Token t = new org.apache.lucene.analysis.Token(); int positionCount = 0; boolean severalTokensAtSamePosition = false; while (true) { try { - t = source.next(); + t = source.next(t); } catch (IOException e) { t = null; } if (t == null) break; - v.addElement(t); + v.addElement(t.clone()); if (t.getPositionIncrement() != 0) positionCount += t.getPositionIncrement(); else @@ -481,7 +502,7 @@ return null; else if (v.size() == 1) { t = (org.apache.lucene.analysis.Token) v.elementAt(0); - return new TermQuery(new Term(field, t.termText())); + return new TermQuery(new Term(field, t.term())); } else { if (severalTokensAtSamePosition) { if (positionCount == 1) { @@ -490,7 +511,7 @@ for (int i = 0; i < v.size(); i++) { t = (org.apache.lucene.analysis.Token) v.elementAt(i); TermQuery currentQuery = new TermQuery( - new Term(field, t.termText())); + new Term(field, t.term())); q.add(currentQuery, BooleanClause.Occur.SHOULD); } return q; @@ -512,7 +533,7 @@ multiTerms.clear(); } position += t.getPositionIncrement(); - multiTerms.add(new Term(field, t.termText())); + multiTerms.add(new Term(field, t.term())); } if (enablePositionIncrements) { mpq.add((Term[])multiTerms.toArray(new Term[0]),position); @@ -530,9 +551,9 @@ t = (org.apache.lucene.analysis.Token) v.elementAt(i); if (enablePositionIncrements) { position += t.getPositionIncrement(); - pq.add(new Term(field, t.termText()),position); + pq.add(new Term(field, t.term()),position); } else { - pq.add(new Term(field, t.termText())); + pq.add(new Term(field, t.term())); } } return pq; Index: lucene/src/java/org/apache/lucene/search/QueryTermVector.java =================================================================== --- lucene/src/java/org/apache/lucene/search/QueryTermVector.java (revision 682416) +++ lucene/src/java/org/apache/lucene/search/QueryTermVector.java (working copy) @@ -17,15 +17,20 @@ * limitations under the License. */ +import java.io.IOException; +import java.io.StringReader; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.TermFreqVector; -import java.io.IOException; -import java.io.StringReader; -import java.util.*; - /** * * @@ -51,12 +56,10 @@ TokenStream stream = analyzer.tokenStream("", new StringReader(queryString)); if (stream != null) { - Token next = null; List terms = new ArrayList(); try { - while ((next = stream.next()) != null) - { - terms.add(next.termText()); + for (Token next = stream.next(new Token()); next != null; next = stream.next(next)) { + terms.add(next.term()); } processTerms((String[])terms.toArray(new String[terms.size()])); } catch (IOException e) { Index: lucene/src/java/org/apache/lucene/index/DocInverterPerField.java =================================================================== --- lucene/src/java/org/apache/lucene/index/DocInverterPerField.java (revision 682416) +++ lucene/src/java/org/apache/lucene/index/DocInverterPerField.java (working copy) @@ -81,13 +81,9 @@ final int valueLength = stringValue.length(); Token token = perThread.localToken; token.clear(); - char[] termBuffer = token.termBuffer(); - if (termBuffer.length < valueLength) - termBuffer = token.resizeTermBuffer(valueLength); - stringValue.getChars(0, valueLength, termBuffer, 0); - token.setTermLength(valueLength); token.setStartOffset(fieldState.offset); - token.setEndOffset(fieldState.offset + stringValue.length()); + token.setEndOffset(fieldState.offset + valueLength); + token.setTermBuffer(stringValue, 0, valueLength); boolean success = false; try { consumer.add(token);