Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicNormalizationFilter.java (working copy) @@ -19,35 +19,33 @@ import java.io.IOException; -import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; /** * A TokenFilter that applies {@link ArabicNormalizer} to normalize the orthography. * */ -public class ArabicNormalizationFilter extends TokenFilter { +public final class ArabicNormalizationFilter extends TokenFilter { protected ArabicNormalizer normalizer = null; - + private TermAttribute termAtt; + public ArabicNormalizationFilter(TokenStream input) { super(input); normalizer = new ArabicNormalizer(); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } - - - public Token next(Token reusableToken) throws IOException { - if ((reusableToken = input.next(reusableToken)) == null) { - return null; + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + int newlen = normalizer.normalize(termAtt.termBuffer(), termAtt.termLength()); + termAtt.setTermLength(newlen); + return true; } else { - int oldlen = reusableToken.termLength(); - int newlen = normalizer.normalize(reusableToken.termBuffer(), oldlen); - if (oldlen != newlen) - reusableToken.setTermLength(newlen); - return reusableToken; + return false; } } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicStemFilter.java (working copy) @@ -19,43 +19,33 @@ import java.io.IOException; -import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; /** * A TokenFilter that applies {@link ArabicStemmer} to stem Arabic words.. * */ -public class ArabicStemFilter extends TokenFilter { +public final class ArabicStemFilter extends TokenFilter { protected ArabicStemmer stemmer = null; - + private TermAttribute termAtt; + public ArabicStemFilter(TokenStream input) { super(input); stemmer = new ArabicStemmer(); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } - - - /** - * @return Returns the next token in the stream, or null at EOS - */ - public Token next(Token reusableToken) throws IOException { - /** - * The actual token in the input stream. - */ - - - if ((reusableToken = input.next(reusableToken)) == null) { - return null; + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + int newlen = stemmer.stem(termAtt.termBuffer(), termAtt.termLength()); + termAtt.setTermLength(newlen); + return true; } else { - int oldlen = reusableToken.termLength(); - int newlen = stemmer.stem(reusableToken.termBuffer(), oldlen); - if (oldlen != newlen) - reusableToken.setTermLength(newlen); - return reusableToken; + return false; } } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/br/BrazilianStemFilter.java (working copy) @@ -17,14 +17,13 @@ * limitations under the License. */ -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; - import java.io.IOException; -import java.util.HashSet; import java.util.Set; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; + /** * Based on GermanStemFilter * @@ -36,10 +35,12 @@ */ private BrazilianStemmer stemmer = null; private Set exclusions = null; - + private TermAttribute termAtt; + public BrazilianStemFilter(TokenStream in) { super(in); stemmer = new BrazilianStemmer(); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } public BrazilianStemFilter(TokenStream in, Set exclusiontable) { @@ -47,26 +48,20 @@ this.exclusions = exclusiontable; } - /** - * @return Returns the next token in the stream, or null at EOS. - */ - public final Token next(final Token reusableToken) - throws IOException { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - if (nextToken == null) - return null; - - String term = nextToken.term(); - - // Check the exclusion table. - if (exclusions == null || !exclusions.contains(term)) { - String s = stemmer.stem(term); - // If not stemmed, don't waste the time adjusting the token. - if ((s != null) && !s.equals(term)) - nextToken.setTermBuffer(s); + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + String term = termAtt.term(); + // Check the exclusion table. + if (exclusions == null || !exclusions.contains(term)) { + String s = stemmer.stem(term); + // If not stemmed, don't waste the time adjusting the token. + if ((s != null) && !s.equals(term)) + termAtt.setTermBuffer(s); + } + return true; + } else { + return false; } - return nextToken; } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/cjk/CJKTokenizer.java (working copy) @@ -17,12 +17,15 @@ * limitations under the License. */ -import org.apache.lucene.analysis.Token; +import java.io.IOException; +import java.io.Reader; + import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.TypeAttribute; -import java.io.Reader; - /** * CJKTokenizer was modified from StopTokenizer which does a decent job for * most European languages. It performs other token methods for double-byte @@ -88,6 +91,10 @@ */ private boolean preIsTokened = false; + private TermAttribute termAtt; + private OffsetAttribute offsetAtt; + private TypeAttribute typeAtt; + //~ Constructors ----------------------------------------------------------- /** @@ -97,25 +104,26 @@ */ public CJKTokenizer(Reader in) { super(in); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); + offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class); + typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class); } //~ Methods ---------------------------------------------------------------- /** - * Returns the next token in the stream, or null at EOS. + * Returns true for the next token in the stream, or false at EOS. * See http://java.sun.com/j2se/1.3/docs/api/java/lang/Character.UnicodeBlock.html * for detail. * - * @param reusableToken a reusable token - * @return Token + * @return false for end of stream, true otherwise * * @throws java.io.IOException - throw IOException when read error
* happened in the InputStream * */ - public final Token next(final Token reusableToken) throws java.io.IOException { + public boolean incrementToken() throws IOException { /** how many character(s) has been stored in buffer */ - assert reusableToken != null; while(true) { // loop until we find a non-empty token @@ -147,7 +155,7 @@ break; } else { - return null; + return false; } } else { //get current character @@ -252,10 +260,12 @@ } if (length > 0) { - return reusableToken.reinit - (buffer, 0, length, input.correctOffset(start), input.correctOffset(start+length), TOKEN_TYPE_NAMES[tokenType]); + termAtt.setTermBuffer(buffer, 0, length); + offsetAtt.setOffset(input.correctOffset(start), input.correctOffset(start+length)); + typeAtt.setType(TOKEN_TYPE_NAMES[tokenType]); + return true; } else if (dataLen == -1) { - return null; + return false; } // Cycle back and try for the next token (don't Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java (working copy) @@ -17,13 +17,13 @@ * limitations under the License. */ -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; - import java.io.IOException; import java.util.Set; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; + /** * A filter that stems German words. It supports a table of words that should * not be stemmed at all. The stemmer used can be changed at runtime after the @@ -40,10 +40,13 @@ private GermanStemmer stemmer = null; private Set exclusionSet = null; + private TermAttribute termAtt; + public GermanStemFilter( TokenStream in ) { super(in); stemmer = new GermanStemmer(); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } /** @@ -56,26 +59,22 @@ } /** - * @return Returns the next token in the stream, or null at EOS + * @return Returns true for next token in the stream, or false at EOS */ - public final Token next(final Token reusableToken) - throws IOException - { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - - if (nextToken == null) - return null; - - String term = nextToken.term(); - // Check the exclusion table. - if (exclusionSet == null || !exclusionSet.contains(term)) { - String s = stemmer.stem(term); - // If not stemmed, don't waste the time adjusting the token. - if ((s != null) && !s.equals(term)) - nextToken.setTermBuffer(s); + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + String term = termAtt.term(); + // Check the exclusion table. + if (exclusionSet == null || !exclusionSet.contains(term)) { + String s = stemmer.stem(term); + // If not stemmed, don't waste the time adjusting the token. + if ((s != null) && !s.equals(term)) + termAtt.setTermBuffer(s); + } + return true; + } else { + return false; } - return nextToken; } /** Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/el/GreekLowerCaseFilter.java (working copy) @@ -16,9 +16,11 @@ * limitations under the License. */ +import java.io.IOException; + import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; /** * Normalizes token text to lower case, analyzing given ("greek") charset. @@ -28,26 +30,26 @@ { char[] charset; + private TermAttribute termAtt; + public GreekLowerCaseFilter(TokenStream in, char[] charset) { super(in); this.charset = charset; + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } - public final Token next(final Token reusableToken) throws java.io.IOException - { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - - if (nextToken == null) - return null; - - char[] chArray = nextToken.termBuffer(); - int chLen = nextToken.termLength(); + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + char[] chArray = termAtt.termBuffer(); + int chLen = termAtt.termLength(); for (int i = 0; i < chLen; i++) { - chArray[i] = GreekCharsets.toLowerCase(chArray[i], charset); + chArray[i] = GreekCharsets.toLowerCase(chArray[i], charset); } - return nextToken; + return true; + } else { + return false; + } } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/ElisionFilter.java (working copy) @@ -25,6 +25,7 @@ import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; /** * Removes elisions from a token stream. For example, "l'avion" (the plane) will be @@ -38,7 +39,9 @@ private Set articles = null; private static char[] apostrophes = {'\'', '’'}; - + + private TermAttribute termAtt; + public void setArticles(Set articles) { this.articles = new HashSet(); Iterator iter = articles.iterator(); @@ -54,6 +57,7 @@ super(input); this.articles = new HashSet(Arrays.asList(new String[] { "l", "m", "t", "qu", "n", "s", "j" })); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } /** @@ -62,6 +66,7 @@ public ElisionFilter(TokenStream input, Set articles) { super(input); setArticles(articles); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } /** @@ -70,39 +75,50 @@ public ElisionFilter(TokenStream input, String[] articles) { super(input); setArticles(new HashSet(Arrays.asList(articles))); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } /** * Returns the next input Token with term() without elisioned start */ - public Token next(final Token reusableToken) throws IOException { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - if (nextToken == null) - return null; + public final boolean incrementToken() throws IOException { + if (input.incrementToken()) { + char[] termBuffer = termAtt.termBuffer(); + int termLength = termAtt.termLength(); - char[] termBuffer = nextToken.termBuffer(); - int termLength = nextToken.termLength(); - - int minPoz = Integer.MAX_VALUE; - for (int i = 0; i < apostrophes.length; i++) { - char apos = apostrophes[i]; - // The equivalent of String.indexOf(ch) - for (int poz = 0; poz < termLength ; poz++) { - if (termBuffer[poz] == apos) { + int minPoz = Integer.MAX_VALUE; + for (int i = 0; i < apostrophes.length; i++) { + char apos = apostrophes[i]; + // The equivalent of String.indexOf(ch) + for (int poz = 0; poz < termLength ; poz++) { + if (termBuffer[poz] == apos) { minPoz = Math.min(poz, minPoz); break; + } } } - } - // An apostrophe has been found. If the prefix is an article strip it off. - if (minPoz != Integer.MAX_VALUE - && articles.contains(new String(nextToken.termBuffer(), 0, minPoz).toLowerCase())) { - nextToken.setTermBuffer(nextToken.termBuffer(), minPoz + 1, nextToken.termLength() - (minPoz + 1)); + // An apostrophe has been found. If the prefix is an article strip it off. + if (minPoz != Integer.MAX_VALUE + && articles.contains(new String(termAtt.termBuffer(), 0, minPoz).toLowerCase())) { + termAtt.setTermBuffer(termAtt.termBuffer(), minPoz + 1, termAtt.termLength() - (minPoz + 1)); + } + + return true; + } else { + return false; } + } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next(final Token reusableToken) throws java.io.IOException { + return super.next(reusableToken); + } - return nextToken; + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next() throws java.io.IOException { + return super.next(); } - } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/fr/FrenchStemFilter.java (working copy) @@ -20,6 +20,7 @@ import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; import java.io.IOException; import java.util.HashSet; @@ -39,10 +40,13 @@ */ private FrenchStemmer stemmer = null; private Set exclusions = null; + + private TermAttribute termAtt; public FrenchStemFilter( TokenStream in ) { super(in); stemmer = new FrenchStemmer(); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } @@ -52,25 +56,23 @@ } /** - * @return Returns the next token in the stream, or null at EOS + * @return Returns true for the next token in the stream, or false at EOS */ - public final Token next(final Token reusableToken) - throws IOException { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - if (nextToken == null) - return null; + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + String term = termAtt.term(); - String term = nextToken.term(); - - // Check the exclusion table - if ( exclusions == null || !exclusions.contains( term ) ) { - String s = stemmer.stem( term ); - // If not stemmed, don't waste the time adjusting the token. - if ((s != null) && !s.equals( term ) ) - nextToken.setTermBuffer(s); - } - return nextToken; + // Check the exclusion table + if ( exclusions == null || !exclusions.contains( term ) ) { + String s = stemmer.stem( term ); + // If not stemmed, don't waste the time adjusting the token. + if ((s != null) && !s.equals( term ) ) + termAtt.setTermBuffer(s); + } + return true; + } else { + return false; + } } /** * Set a alternative/custom FrenchStemmer for this filter. Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/EmptyTokenStream.java (working copy) @@ -27,8 +27,19 @@ */ public class EmptyTokenStream extends TokenStream { - public Token next(final Token reusableToken) throws IOException { - assert reusableToken != null; - return null; + public final boolean incrementToken() throws IOException { + return false; } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next(final Token reusableToken) throws java.io.IOException { + return super.next(reusableToken); + } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next() throws java.io.IOException { + return super.next(); + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/nl/DutchStemFilter.java (working copy) @@ -17,16 +17,16 @@ * limitations under the License. */ -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; - import java.io.IOException; import java.util.HashMap; import java.util.HashSet; +import java.util.Map; import java.util.Set; -import java.util.Map; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; + /** * A filter that stems Dutch words. It supports a table of words that should * not be stemmed at all. The stemmer used can be changed at runtime after the @@ -39,10 +39,13 @@ */ private DutchStemmer stemmer = null; private Set exclusions = null; + + private TermAttribute termAtt; public DutchStemFilter(TokenStream _in) { super(_in); stemmer = new DutchStemmer(); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } /** @@ -62,24 +65,23 @@ } /** - * @return Returns the next token in the stream, or null at EOS + * Returns the next token in the stream, or null at EOS */ - public Token next(Token reusableToken) throws IOException { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - if (nextToken == null) - return null; + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + String term = termAtt.term(); - String term = nextToken.term(); - - // Check the exclusion table. - if (exclusions == null || !exclusions.contains(term)) { - String s = stemmer.stem(term); - // If not stemmed, don't waste the time adjusting the token. - if ((s != null) && !s.equals(term)) - nextToken.setTermBuffer(s); + // Check the exclusion table. + if (exclusions == null || !exclusions.contains(term)) { + String s = stemmer.stem(term); + // If not stemmed, don't waste the time adjusting the token. + if ((s != null) && !s.equals(term)) + termAtt.setTermBuffer(s); + } + return true; + } else { + return false; } - return nextToken; } /** Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/DelimitedPayloadTokenFilter.java (working copy) @@ -16,15 +16,14 @@ * limitations under the License. */ +import java.io.IOException; + import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.analysis.tokenattributes.TermAttribute; -import java.io.IOException; - /** * Characters before the delimiter are the "token", those after are the payload. *

@@ -37,7 +36,7 @@ * * @see PayloadEncoder */ -public class DelimitedPayloadTokenFilter extends TokenFilter { +public final class DelimitedPayloadTokenFilter extends TokenFilter { public static final char DEFAULT_DELIMITER = '|'; protected char delimiter = DEFAULT_DELIMITER; protected TermAttribute termAtt; @@ -83,27 +82,4 @@ } return result; } - - - public Token next(Token reusableToken) throws IOException { - Token result = input.next(reusableToken); - if (result != null) { - final char[] buffer = result.termBuffer(); - final int length = result.termLength(); - boolean seen = false; - for (int i = 0; i < length; i++) { - if (buffer[i] == delimiter) { - result.setTermBuffer(buffer, 0, i); - result.setPayload(encoder.encode(buffer, i + 1, (length - (i + 1)))); - seen = true; - break;//at this point, we know the whole piece, so we can exit. If we don't see the delimiter, then the termAtt is the same - } - } - if (seen == false) { - //no delimiter - payAtt.setPayload(null); - } - } - return result; - } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java (working copy) @@ -20,6 +20,8 @@ import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.index.Payload; import java.io.IOException; @@ -34,19 +36,37 @@ private String typeMatch; private Payload thePayload; + private PayloadAttribute payloadAtt; + private TypeAttribute typeAtt; + public NumericPayloadTokenFilter(TokenStream input, float payload, String typeMatch) { super(input); //Need to encode the payload thePayload = new Payload(PayloadHelper.encodeFloat(payload)); this.typeMatch = typeMatch; + payloadAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class); + typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class); } - public Token next(final Token reusableToken) throws IOException { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - if (nextToken != null && nextToken.type().equals(typeMatch)){ - nextToken.setPayload(thePayload); + public final boolean incrementToken() throws IOException { + if (input.incrementToken()) { + if (typeAtt.type().equals(typeMatch)) + payloadAtt.setPayload(thePayload); + return true; + } else { + return false; } - return nextToken; } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next(final Token reusableToken) throws java.io.IOException { + return super.next(reusableToken); + } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next() throws java.io.IOException { + return super.next(); + } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TokenOffsetPayloadTokenFilter.java (working copy) @@ -17,14 +17,16 @@ */ +import java.io.IOException; + import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; import org.apache.lucene.index.Payload; -import java.io.IOException; - /** * Adds the {@link org.apache.lucene.analysis.Token#setStartOffset(int)} * and {@link org.apache.lucene.analysis.Token#setEndOffset(int)} @@ -32,22 +34,37 @@ * **/ public class TokenOffsetPayloadTokenFilter extends TokenFilter { + protected OffsetAttribute offsetAtt; + protected PayloadAttribute payAtt; - public TokenOffsetPayloadTokenFilter(TokenStream input) { super(input); + offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class); + payAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class); } - public Token next(final Token reusableToken) throws IOException { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - if (nextToken != null){ + public final boolean incrementToken() throws IOException { + if (input.incrementToken()) { byte[] data = new byte[8]; - PayloadHelper.encodeInt(nextToken.startOffset(), data, 0); - PayloadHelper.encodeInt(nextToken.endOffset(), data, 4); + PayloadHelper.encodeInt(offsetAtt.startOffset(), data, 0); + PayloadHelper.encodeInt(offsetAtt.endOffset(), data, 4); Payload payload = new Payload(data); - nextToken.setPayload(payload); + payAtt.setPayload(payload); + return true; + } else { + return false; } - return nextToken; } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next(final Token reusableToken) throws java.io.IOException { + return super.next(reusableToken); + } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next() throws java.io.IOException { + return super.next(); + } } \ No newline at end of file Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/payloads/TypeAsPayloadTokenFilter.java (working copy) @@ -20,6 +20,8 @@ import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.index.Payload; import java.io.IOException; @@ -32,19 +34,37 @@ * **/ public class TypeAsPayloadTokenFilter extends TokenFilter { + private PayloadAttribute payloadAtt; + private TypeAttribute typeAtt; public TypeAsPayloadTokenFilter(TokenStream input) { super(input); - + payloadAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class); + typeAtt = (TypeAttribute) addAttribute(TypeAttribute.class); } - public Token next(final Token reusableToken) throws IOException { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - if (nextToken != null && nextToken.type() != null && nextToken.type().equals("") == false){ - nextToken.setPayload(new Payload(nextToken.type().getBytes("UTF-8"))); + public final boolean incrementToken() throws IOException { + if (input.incrementToken()) { + String type = typeAtt.type(); + if (type != null && type.equals("") == false) { + payloadAtt.setPayload(new Payload(type.getBytes("UTF-8"))); + } + return true; + } else { + return false; } - return nextToken; } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next(final Token reusableToken) throws java.io.IOException { + return super.next(reusableToken); + } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next() throws java.io.IOException { + return super.next(); + } } \ No newline at end of file Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/position/PositionFilter.java (working copy) @@ -22,6 +22,7 @@ import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; /** Set the positionIncrement of all tokens to the "positionIncrement", * except the first return token which retains its original positionIncrement value. @@ -34,6 +35,8 @@ /** The first token must have non-zero positionIncrement **/ private boolean firstTokenPositioned = false; + + private PositionIncrementAttribute posIncrAtt; /** * Constructs a PositionFilter that assigns a position increment of zero to @@ -43,6 +46,7 @@ */ public PositionFilter(final TokenStream input) { super(input); + posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class); } /** @@ -58,20 +62,31 @@ this.positionIncrement = positionIncrement; } - public Token next(Token reusableToken) throws IOException { - - assert reusableToken != null; - reusableToken = input.next(reusableToken); - if (null != reusableToken) { + public final boolean incrementToken() throws IOException { + if (input.incrementToken()) { if (firstTokenPositioned) { - reusableToken.setPositionIncrement(positionIncrement); + posIncrAtt.setPositionIncrement(positionIncrement); } else { firstTokenPositioned = true; } + return true; + } else { + return false; } - return reusableToken; } + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next(final Token reusableToken) throws java.io.IOException { + return super.next(reusableToken); + } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next() throws java.io.IOException { + return super.next(); + } + public void reset() throws IOException { super.reset(); firstTokenPositioned = false; Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/reverse/ReverseStringFilter.java (working copy) @@ -19,7 +19,7 @@ import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; import java.io.IOException; @@ -30,16 +30,20 @@ */ public final class ReverseStringFilter extends TokenFilter { + private TermAttribute termAtt; + public ReverseStringFilter(TokenStream in) { super(in); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } - public final Token next(Token in) throws IOException { - assert in != null; - Token token=input.next(in); - if( token == null ) return null; - reverse( token.termBuffer(), token.termLength() ); - return token; + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + reverse( termAtt.termBuffer(), termAtt.termLength() ); + return true; + } else { + return false; + } } public static String reverse( final String input ){ Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java (working copy) @@ -17,9 +17,12 @@ * limitations under the License. */ +import java.io.IOException; + import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; /** * Normalizes token text to lower case, analyzing given ("russian") charset. @@ -31,26 +34,27 @@ { char[] charset; + private TermAttribute termAtt; + public RussianLowerCaseFilter(TokenStream in, char[] charset) { super(in); this.charset = charset; + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } - public final Token next(final Token reusableToken) throws java.io.IOException + public final boolean incrementToken() throws IOException { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - - if (nextToken == null) - return null; - - char[] chArray = nextToken.termBuffer(); - int chLen = nextToken.termLength(); + if (input.incrementToken()) { + char[] chArray = termAtt.termBuffer(); + int chLen = termAtt.termLength(); for (int i = 0; i < chLen; i++) { - chArray[i] = RussianCharsets.toLowerCase(chArray[i], charset); + chArray[i] = RussianCharsets.toLowerCase(chArray[i], charset); } - return nextToken; + return true; + } else { + return false; + } } } Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java (working copy) @@ -20,6 +20,8 @@ import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; + import java.io.IOException; /** @@ -37,29 +39,32 @@ */ private RussianStemmer stemmer = null; + private TermAttribute termAtt; + public RussianStemFilter(TokenStream in, char[] charset) { super(in); stemmer = new RussianStemmer(charset); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } /** - * @return Returns the next token in the stream, or null at EOS + * Returns the next token in the stream, or null at EOS */ - public final Token next(final Token reusableToken) throws IOException + public final boolean incrementToken() throws IOException { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - if (nextToken == null) - return null; - - String term = nextToken.term(); + if (input.incrementToken()) { + String term = termAtt.term(); String s = stemmer.stem(term); if (s != null && !s.equals(term)) - nextToken.setTermBuffer(s); - return nextToken; + termAtt.setTermBuffer(s); + return true; + } else { + return false; + } } + /** * Set a alternative/custom RussianStemmer for this filter. */ Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (revision 797224) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (working copy) @@ -22,6 +22,9 @@ import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; + import java.text.BreakIterator; /** @@ -32,46 +35,62 @@ public class ThaiWordFilter extends TokenFilter { private BreakIterator breaker = null; - private Token thaiToken = null; + private TermAttribute termAtt; + private OffsetAttribute offsetAtt; + + private State thaiState = null; + public ThaiWordFilter(TokenStream input) { super(input); breaker = BreakIterator.getWordInstance(new Locale("th")); + termAtt = (TermAttribute) addAttribute(TermAttribute.class); + offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class); } - public Token next(final Token reusableToken) throws IOException { - assert reusableToken != null; - if (thaiToken != null) { + public final boolean incrementToken() throws IOException { + if (thaiState != null) { int start = breaker.current(); int end = breaker.next(); if (end != BreakIterator.DONE) { - reusableToken.reinit(thaiToken, thaiToken.termBuffer(), start, end - start); - reusableToken.setStartOffset(thaiToken.startOffset()+start); - reusableToken.setEndOffset(thaiToken.startOffset()+end); - return reusableToken; + restoreState(thaiState); + termAtt.setTermBuffer(termAtt.termBuffer(), start, end - start); + offsetAtt.setOffset(offsetAtt.startOffset() + start, offsetAtt.startOffset() + end); + return true; } - thaiToken = null; + thaiState = null; } - Token nextToken = input.next(reusableToken); - if (nextToken == null || nextToken.termLength() == 0) { - return null; - } + if (input.incrementToken() == false || termAtt.termLength() == 0) + return false; - String text = nextToken.term(); + String text = termAtt.term(); if (UnicodeBlock.of(text.charAt(0)) != UnicodeBlock.THAI) { - nextToken.setTermBuffer(text.toLowerCase()); - return nextToken; + termAtt.setTermBuffer(text.toLowerCase()); + return true; } + + thaiState = captureState(); - thaiToken = (Token) nextToken.clone(); breaker.setText(text); int end = breaker.next(); if (end != BreakIterator.DONE) { - nextToken.setTermBuffer(text, 0, end); - nextToken.setEndOffset(nextToken.startOffset() + end); - return nextToken; + termAtt.setTermBuffer(text, 0, end); + offsetAtt.setOffset(offsetAtt.startOffset(), offsetAtt.startOffset() + end); + return true; } - return null; + return false; } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next(final Token reusableToken) throws java.io.IOException { + return super.next(reusableToken); + } + + /** @deprecated Will be removed in Lucene 3.0. This method is final, as it should + * not be overridden. Delegates to the backwards compatibility layer. */ + public final Token next() throws java.io.IOException { + return super.next(); + } } Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java (revision 797224) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicNormalizationFilter.java (working copy) @@ -17,18 +17,12 @@ * limitations under the License. */ -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.io.StringReader; import junit.framework.TestCase; -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.WhitespaceTokenizer; -import org.apache.lucene.analysis.standard.StandardTokenizer; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; /** * Test the Arabic Normalization Filter @@ -95,11 +89,10 @@ private void check(final String input, final String expected) throws IOException { ArabicLetterTokenizer tokenStream = new ArabicLetterTokenizer(new StringReader(input)); ArabicNormalizationFilter filter = new ArabicNormalizationFilter(tokenStream); - final Token reusableToken = new Token(); - Token nextToken = filter.next(reusableToken); - if (nextToken == null) - fail(); - assertEquals(expected, nextToken.term()); + TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class); + + assertTrue(filter.incrementToken()); + assertEquals(expected, termAtt.term()); filter.close(); } Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java (revision 797224) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java (working copy) @@ -17,17 +17,12 @@ * limitations under the License. */ -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStreamReader; import java.io.StringReader; import junit.framework.TestCase; -import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.standard.StandardTokenizer; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; /** * Test the Arabic Normalization Filter @@ -118,11 +113,10 @@ private void check(final String input, final String expected) throws IOException { ArabicLetterTokenizer tokenStream = new ArabicLetterTokenizer(new StringReader(input)); ArabicStemFilter filter = new ArabicStemFilter(tokenStream); - final Token reusableToken = new Token(); - Token nextToken = filter.next(reusableToken); - if (nextToken == null) - fail(); - assertEquals(expected, nextToken.term()); + TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class); + + assertTrue(filter.incrementToken()); + assertEquals(expected, termAtt.term()); filter.close(); } Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java (revision 797224) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/br/TestBrazilianStemmer.java (working copy) @@ -23,8 +23,8 @@ import junit.framework.TestCase; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; /** * Test the Brazilian Stem Filter, which only modifies the term text. @@ -122,12 +122,10 @@ private void check(final String input, final String expected) throws IOException { Analyzer analyzer = new BrazilianAnalyzer(); TokenStream stream = analyzer.tokenStream("dummy", new StringReader(input)); - final Token reusableToken = new Token(); - Token nextToken = stream.next(reusableToken); - if (nextToken == null) - fail(); - assertEquals(expected, nextToken.term()); - assertTrue(stream.next(nextToken) == null); + TermAttribute text = (TermAttribute) stream.getAttribute(TermAttribute.class); + assertTrue(stream.incrementToken()); + assertEquals(expected, text.term()); + assertFalse(stream.incrementToken()); stream.close(); } Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java (revision 797224) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/cz/TestCzechAnalyzer.java (working copy) @@ -22,8 +22,8 @@ import junit.framework.TestCase; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; /** * Test the CzechAnalyzer @@ -39,13 +39,12 @@ private void assertAnalyzesTo(Analyzer a, String input, String[] output) throws Exception { TokenStream ts = a.tokenStream("dummy", new StringReader(input)); - final Token reusableToken = new Token(); + TermAttribute text = (TermAttribute) ts.getAttribute(TermAttribute.class); for (int i=0; i */ -public class CollationKeyFilter extends TokenFilter { +public final class CollationKeyFilter extends TokenFilter { private Collator collator = null; + private TermAttribute termAtt; /** * @param input Source token stream @@ -83,25 +85,26 @@ public CollationKeyFilter(TokenStream input, Collator collator) { super(input); this.collator = collator; + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } - public final Token next(final Token reusableToken) throws IOException { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - if (nextToken != null) { - char[] termBuffer = nextToken.termBuffer(); - String termText = new String(termBuffer, 0, nextToken.termLength()); + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + char[] termBuffer = termAtt.termBuffer(); + String termText = new String(termBuffer, 0, termAtt.termLength()); byte[] collationKey = collator.getCollationKey(termText).toByteArray(); ByteBuffer collationKeyBuf = ByteBuffer.wrap(collationKey); int encodedLength = IndexableBinaryStringTools.getEncodedLength(collationKeyBuf); if (encodedLength > termBuffer.length) { - nextToken.resizeTermBuffer(encodedLength); + termAtt.resizeTermBuffer(encodedLength); } - nextToken.setTermLength(encodedLength); - CharBuffer wrappedTermBuffer = CharBuffer.wrap(nextToken.termBuffer()); + termAtt.setTermLength(encodedLength); + CharBuffer wrappedTermBuffer = CharBuffer.wrap(termAtt.termBuffer()); IndexableBinaryStringTools.encode(collationKeyBuf, wrappedTermBuffer); + return true; + } else { + return false; } - return nextToken; } } Index: contrib/collation/src/java/org/apache/lucene/collation/ICUCollationKeyFilter.java =================================================================== --- contrib/collation/src/java/org/apache/lucene/collation/ICUCollationKeyFilter.java (revision 797224) +++ contrib/collation/src/java/org/apache/lucene/collation/ICUCollationKeyFilter.java (working copy) @@ -24,6 +24,7 @@ import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.tokenattributes.TermAttribute; import org.apache.lucene.util.IndexableBinaryStringTools; import java.io.IOException; @@ -69,9 +70,10 @@ * java.text.Collator over several languages. *

*/ -public class ICUCollationKeyFilter extends TokenFilter { +public final class ICUCollationKeyFilter extends TokenFilter { private Collator collator = null; private RawCollationKey reusableKey = new RawCollationKey(); + private TermAttribute termAtt; /** * @@ -81,25 +83,26 @@ public ICUCollationKeyFilter(TokenStream input, Collator collator) { super(input); this.collator = collator; + termAtt = (TermAttribute) addAttribute(TermAttribute.class); } - public final Token next(final Token reusableToken) throws IOException { - assert reusableToken != null; - Token nextToken = input.next(reusableToken); - if (nextToken != null) { - char[] termBuffer = nextToken.termBuffer(); - String termText = new String(termBuffer, 0, nextToken.termLength()); + public boolean incrementToken() throws IOException { + if (input.incrementToken()) { + char[] termBuffer = termAtt.termBuffer(); + String termText = new String(termBuffer, 0, termAtt.termLength()); collator.getRawCollationKey(termText, reusableKey); ByteBuffer collationKeyBuf = ByteBuffer.wrap(reusableKey.bytes, 0, reusableKey.size); int encodedLength = IndexableBinaryStringTools.getEncodedLength(collationKeyBuf); if (encodedLength > termBuffer.length) { - nextToken.resizeTermBuffer(encodedLength); + termAtt.resizeTermBuffer(encodedLength); } - nextToken.setTermLength(encodedLength); - CharBuffer wrappedTermBuffer = CharBuffer.wrap(nextToken.termBuffer()); + termAtt.setTermLength(encodedLength); + CharBuffer wrappedTermBuffer = CharBuffer.wrap(termAtt.termBuffer()); IndexableBinaryStringTools.encode(collationKeyBuf, wrappedTermBuffer); + return true; + } else { + return false; } - return nextToken; } }