? analyzer.patch ? build ? dist ? src/java/org/apache/lucene/analysis/AbstractTokenFilter.java ? src/java/org/apache/lucene/analysis/AbstractTokenizer.java ? src/java/org/apache/lucene/analysis/BaseAnalyzer.java ? src/test/org/apache/lucene/analysis/TestBaseAnalyzer.java Index: src/java/org/apache/lucene/analysis/CharTokenizer.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/CharTokenizer.java,v retrieving revision 1.5 diff -u -r1.5 CharTokenizer.java --- src/java/org/apache/lucene/analysis/CharTokenizer.java 29 Mar 2004 22:48:00 -0000 1.5 +++ src/java/org/apache/lucene/analysis/CharTokenizer.java 3 Apr 2004 21:36:10 -0000 @@ -19,10 +19,7 @@ import java.io.Reader; /** An abstract base class for simple, character-oriented tokenizers.*/ -public abstract class CharTokenizer extends Tokenizer { - public CharTokenizer(Reader input) { - super(input); - } +public abstract class CharTokenizer extends AbstractTokenizer implements Tokenizer { private int offset = 0, bufferIndex = 0, dataLen = 0; private static final int MAX_WORD_LEN = 255; @@ -36,6 +33,14 @@ * define token boundaries and are not included in tokens. */ protected abstract boolean isTokenChar(char c); + public CharTokenizer() { + + } + + public CharTokenizer(Reader input) { + setReader(input); + } + /** Called on each token character to normalize it before it is added to the * token. The default implementation does nothing. Subclasses may use this * to, e.g., lowercase tokens. */ @@ -52,7 +57,7 @@ offset++; if (bufferIndex >= dataLen) { - dataLen = input.read(ioBuffer); + dataLen = reader.read(ioBuffer); bufferIndex = 0; } ; Index: src/java/org/apache/lucene/analysis/LetterTokenizer.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/LetterTokenizer.java,v retrieving revision 1.4 diff -u -r1.4 LetterTokenizer.java --- src/java/org/apache/lucene/analysis/LetterTokenizer.java 29 Mar 2004 22:48:00 -0000 1.4 +++ src/java/org/apache/lucene/analysis/LetterTokenizer.java 3 Apr 2004 21:36:10 -0000 @@ -26,6 +26,11 @@ job for some Asian languages, where words are not separated by spaces. */ public class LetterTokenizer extends CharTokenizer { + + public LetterTokenizer() { + super(); + } + /** Construct a new LetterTokenizer. */ public LetterTokenizer(Reader in) { super(in); Index: src/java/org/apache/lucene/analysis/LowerCaseFilter.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/LowerCaseFilter.java,v retrieving revision 1.4 diff -u -r1.4 LowerCaseFilter.java --- src/java/org/apache/lucene/analysis/LowerCaseFilter.java 29 Mar 2004 22:48:00 -0000 1.4 +++ src/java/org/apache/lucene/analysis/LowerCaseFilter.java 3 Apr 2004 21:36:10 -0000 @@ -23,9 +23,13 @@ * * @version $Id: LowerCaseFilter.java,v 1.4 2004/03/29 22:48:00 cutting Exp $ */ -public final class LowerCaseFilter extends TokenFilter { +public final class LowerCaseFilter extends AbstractTokenFilter implements TokenFilter { + public LowerCaseFilter(){ + + } + public LowerCaseFilter(TokenStream in) { - super(in); + setTokenStream(in); } public final Token next() throws IOException { Index: src/java/org/apache/lucene/analysis/LowerCaseTokenizer.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/LowerCaseTokenizer.java,v retrieving revision 1.5 diff -u -r1.5 LowerCaseTokenizer.java --- src/java/org/apache/lucene/analysis/LowerCaseTokenizer.java 29 Mar 2004 22:48:00 -0000 1.5 +++ src/java/org/apache/lucene/analysis/LowerCaseTokenizer.java 3 Apr 2004 21:36:10 -0000 @@ -29,6 +29,11 @@ * job for some Asian languages, where words are not separated by spaces. */ public final class LowerCaseTokenizer extends LetterTokenizer { + + public LowerCaseTokenizer() { + + } + /** Construct a new LowerCaseTokenizer. */ public LowerCaseTokenizer(Reader in) { super(in); Index: src/java/org/apache/lucene/analysis/PorterStemFilter.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/PorterStemFilter.java,v retrieving revision 1.5 diff -u -r1.5 PorterStemFilter.java --- src/java/org/apache/lucene/analysis/PorterStemFilter.java 29 Mar 2004 22:48:00 -0000 1.5 +++ src/java/org/apache/lucene/analysis/PorterStemFilter.java 3 Apr 2004 21:36:10 -0000 @@ -36,13 +36,16 @@ } */ -public final class PorterStemFilter extends TokenFilter { +public final class PorterStemFilter extends AbstractTokenFilter implements TokenFilter { private PorterStemmer stemmer; - public PorterStemFilter(TokenStream in) { - super(in); + public PorterStemFilter() { stemmer = new PorterStemmer(); } + + public PorterStemFilter(TokenStream in){ + setTokenStream(in); + } /** Returns the next input Token, after being stemmed */ public final Token next() throws IOException { Index: src/java/org/apache/lucene/analysis/StopFilter.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/StopFilter.java,v retrieving revision 1.12 diff -u -r1.12 StopFilter.java --- src/java/org/apache/lucene/analysis/StopFilter.java 29 Mar 2004 22:48:00 -0000 1.12 +++ src/java/org/apache/lucene/analysis/StopFilter.java 3 Apr 2004 21:36:10 -0000 @@ -25,42 +25,51 @@ * Removes stop words from a token stream. */ -public final class StopFilter extends TokenFilter { +public final class StopFilter extends AbstractTokenFilter implements TokenFilter { private Set stopWords; + public StopFilter(String[] stopWords) { + this(null, stopWords); + } + + public StopFilter(Set stopWords) { + this(null, stopWords); + } + + /** * Constructs a filter which removes words from the input * TokenStream that are named in the array of words. */ public StopFilter(TokenStream in, String[] stopWords) { - super(in); - this.stopWords = makeStopSet(stopWords); + setTokenStream(in); + this.stopWords = makeStopSet(stopWords); } - /** - * Constructs a filter which removes words from the input - * TokenStream that are named in the Hashtable. - * - * @deprecated Use {@link #StopFilter(TokenStream, Set)} instead - */ - public StopFilter(TokenStream in, Hashtable stopTable) { - super(in); - stopWords = new HashSet(stopTable.keySet()); - } + /** + * Constructs a filter which removes words from the input + * TokenStream that are named in the Hashtable. + * + * @deprecated Use {@link #StopFilter(TokenStream, Set)} instead + */ + public StopFilter(TokenStream in, Hashtable stopTable) { + setTokenStream(in); + stopWords = new HashSet(stopTable.keySet()); + } - /** - * Constructs a filter which removes words from the input - * TokenStream that are named in the Set. - * It is crucial that an efficient Set implementation is used - * for maximum performance. - * - * @see #makeStopSet(java.lang.String[]) - */ - public StopFilter(TokenStream in, Set stopWords) { - super(in); - this.stopWords = stopWords; - } + /** + * Constructs a filter which removes words from the input + * TokenStream that are named in the Set. + * It is crucial that an efficient Set implementation is used + * for maximum performance. + * + * @see #makeStopSet(java.lang.String[]) + */ + public StopFilter(TokenStream in, Set stopWords) { + setTokenStream(in); + this.stopWords = stopWords; + } /** * Builds a Hashtable from an array of stop words, Index: src/java/org/apache/lucene/analysis/TokenFilter.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/TokenFilter.java,v retrieving revision 1.4 diff -u -r1.4 TokenFilter.java --- src/java/org/apache/lucene/analysis/TokenFilter.java 29 Mar 2004 22:48:00 -0000 1.4 +++ src/java/org/apache/lucene/analysis/TokenFilter.java 3 Apr 2004 21:36:11 -0000 @@ -23,23 +23,11 @@ This is an abstract class. */ -public abstract class TokenFilter extends TokenStream { - /** The source of tokens for this filter. */ - protected TokenStream input; - - /** Call TokenFilter(TokenStream) instead. - * @deprecated */ - protected TokenFilter() {} - - /** Construct a token stream filtering the given input. */ - protected TokenFilter(TokenStream input) { - this.input = input; - } - - /** Close the input TokenStream. */ - public void close() throws IOException { - input.close(); - } +public interface TokenFilter extends TokenStream { + /** + * Set the previous Token Stream, if there was one + */ + public void setTokenStream(TokenStream previousTokenStream); } Index: src/java/org/apache/lucene/analysis/TokenStream.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/TokenStream.java,v retrieving revision 1.3 diff -u -r1.3 TokenStream.java --- src/java/org/apache/lucene/analysis/TokenStream.java 29 Mar 2004 22:48:00 -0000 1.3 +++ src/java/org/apache/lucene/analysis/TokenStream.java 3 Apr 2004 21:36:11 -0000 @@ -18,22 +18,34 @@ import java.io.IOException; -/** A TokenStream enumerates the sequence of tokens, either from - fields of a document or from query text. -

- This is an abstract class. Concrete subclasses are: -

- */ +/** + * A TokenStream enumerates the sequence of tokens, either from + * fields of a document or from query text. + *

+ * This is an abstract class. Concrete subclasses are: + *

+ */ -public abstract class TokenStream { - /** Returns the next token in the stream, or null at EOS. */ - public abstract Token next() throws IOException; +public interface TokenStream { + /** + * Initializes the TokenStream + * + */ + public void init(); + /** + * Returns the next token in the stream, or null at EOS. + */ + public abstract Token next() throws IOException; - /** Releases resources associated with this stream. */ - public void close() throws IOException {} + /** + * Releases resources associated with this stream. + */ + public void close() throws IOException; + + } Index: src/java/org/apache/lucene/analysis/Tokenizer.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/Tokenizer.java,v retrieving revision 1.4 diff -u -r1.4 Tokenizer.java --- src/java/org/apache/lucene/analysis/Tokenizer.java 29 Mar 2004 22:48:00 -0000 1.4 +++ src/java/org/apache/lucene/analysis/Tokenizer.java 3 Apr 2004 21:36:11 -0000 @@ -24,21 +24,12 @@ This is an abstract class. */ -public abstract class Tokenizer extends TokenStream { - /** The text source for this Tokenizer. */ - protected Reader input; +public interface Tokenizer extends TokenStream { - /** Construct a tokenizer with null input. */ - protected Tokenizer() {} - - /** Construct a token stream processing the given input. */ - protected Tokenizer(Reader input) { - this.input = input; - } - - /** By default, closes the input Reader. */ - public void close() throws IOException { - input.close(); - } + /** + * Set the Reader for the Tokenizer + * @param reader The reader for the Tokenizer + */ + public void setReader(Reader reader); } Index: src/java/org/apache/lucene/analysis/WhitespaceTokenizer.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/WhitespaceTokenizer.java,v retrieving revision 1.4 diff -u -r1.4 WhitespaceTokenizer.java --- src/java/org/apache/lucene/analysis/WhitespaceTokenizer.java 29 Mar 2004 22:48:00 -0000 1.4 +++ src/java/org/apache/lucene/analysis/WhitespaceTokenizer.java 3 Apr 2004 21:36:11 -0000 @@ -27,6 +27,9 @@ super(in); } + public WhitespaceTokenizer() { + } + /** Collects only characters which do not satisfy * {@link Character#isWhitespace(char)}.*/ protected boolean isTokenChar(char c) { Index: src/java/org/apache/lucene/analysis/de/GermanStemFilter.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/de/GermanStemFilter.java,v retrieving revision 1.8 diff -u -r1.8 GermanStemFilter.java --- src/java/org/apache/lucene/analysis/de/GermanStemFilter.java 30 Mar 2004 15:54:48 -0000 1.8 +++ src/java/org/apache/lucene/analysis/de/GermanStemFilter.java 3 Apr 2004 21:36:11 -0000 @@ -19,6 +19,8 @@ import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.AbstractTokenFilter; + import java.io.IOException; import java.util.Hashtable; import java.util.Set; @@ -32,7 +34,9 @@ * @author Gerhard Schwarz * @version $Id: GermanStemFilter.java,v 1.8 2004/03/30 15:54:48 otis Exp $ */ -public final class GermanStemFilter extends TokenFilter +public final class GermanStemFilter extends AbstractTokenFilter implements TokenFilter + + { /** * The actual token in the input stream. @@ -40,15 +44,13 @@ private Token token = null; private GermanStemmer stemmer = null; private Set exclusionSet = null; - - public GermanStemFilter( TokenStream in ) - { - super(in); - stemmer = new GermanStemmer(); + public GermanStemFilter(TokenStream in) { + setTokenStream(in); + stemmer = new GermanStemmer(); } - /** * Builds a GermanStemFilter that uses an exclusiontable. + * * @deprecated Use {@link #GermanStemFilter(org.apache.lucene.analysis.TokenStream, java.util.Set)} instead. */ public GermanStemFilter( TokenStream in, Hashtable exclusiontable ) @@ -65,7 +67,6 @@ this( in ); this.exclusionSet = exclusionSet; } - /** * @return Returns the next token in the stream, or null at EOS */ Index: src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java,v retrieving revision 1.3 diff -u -r1.3 RussianLetterTokenizer.java --- src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java 29 Mar 2004 22:48:01 -0000 1.3 +++ src/java/org/apache/lucene/analysis/ru/RussianLetterTokenizer.java 3 Apr 2004 21:36:11 -0000 @@ -40,6 +40,11 @@ this.charset = charset; } + public RussianLetterTokenizer(char[] charset) + { + this.charset = charset; + } + /** * Collects only characters which satisfy * {@link Character#isLetter(char)}. Index: src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java,v retrieving revision 1.4 diff -u -r1.4 RussianLowerCaseFilter.java --- src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java 29 Mar 2004 22:48:01 -0000 1.4 +++ src/java/org/apache/lucene/analysis/ru/RussianLowerCaseFilter.java 3 Apr 2004 21:36:11 -0000 @@ -19,6 +19,7 @@ import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.AbstractTokenFilter; /** * Normalizes token text to lower case, analyzing given ("russian") charset. @@ -26,13 +27,17 @@ * @author Boris Okner, b.okner@rogers.com * @version $Id: RussianLowerCaseFilter.java,v 1.4 2004/03/29 22:48:01 cutting Exp $ */ -public final class RussianLowerCaseFilter extends TokenFilter +public final class RussianLowerCaseFilter extends AbstractTokenFilter implements TokenFilter { char[] charset; + public RussianLowerCaseFilter(char[] charset) { + this(null, charset); + } + public RussianLowerCaseFilter(TokenStream in, char[] charset) { - super(in); + setTokenStream(in); this.charset = charset; } Index: src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java,v retrieving revision 1.5 diff -u -r1.5 RussianStemFilter.java --- src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java 29 Mar 2004 22:48:01 -0000 1.5 +++ src/java/org/apache/lucene/analysis/ru/RussianStemFilter.java 3 Apr 2004 21:36:12 -0000 @@ -19,6 +19,8 @@ import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.AbstractTokenFilter; + import java.io.IOException; /** @@ -29,7 +31,7 @@ * @author Boris Okner, b.okner@rogers.com * @version $Id: RussianStemFilter.java,v 1.5 2004/03/29 22:48:01 cutting Exp $ */ -public final class RussianStemFilter extends TokenFilter +public final class RussianStemFilter extends AbstractTokenFilter implements TokenFilter { /** * The actual token in the input stream. @@ -37,9 +39,14 @@ private Token token = null; private RussianStemmer stemmer = null; + public RussianStemFilter(char [] charset) + { + this(null, charset); + } + public RussianStemFilter(TokenStream in, char[] charset) { - super(in); + setTokenStream(in); stemmer = new RussianStemmer(charset); } Index: src/java/org/apache/lucene/analysis/standard/StandardFilter.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/standard/StandardFilter.java,v retrieving revision 1.3 diff -u -r1.3 StandardFilter.java --- src/java/org/apache/lucene/analysis/standard/StandardFilter.java 29 Mar 2004 22:48:01 -0000 1.3 +++ src/java/org/apache/lucene/analysis/standard/StandardFilter.java 3 Apr 2004 21:36:12 -0000 @@ -20,13 +20,14 @@ /** Normalizes tokens extracted with {@link StandardTokenizer}. */ -public final class StandardFilter extends TokenFilter - implements StandardTokenizerConstants { +public final class StandardFilter extends AbstractTokenFilter + implements StandardTokenizerConstants, TokenFilter { /** Construct filtering in. */ public StandardFilter(TokenStream in) { - super(in); + super(); + setTokenStream(in); } private static final String APOSTROPHE_TYPE = tokenImage[APOSTROPHE]; Index: src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java,v retrieving revision 1.3 diff -u -r1.3 StandardTokenizer.java --- src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java 22 Dec 2003 22:12:24 -0000 1.3 +++ src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java 3 Apr 2004 21:36:12 -0000 @@ -1,6 +1,8 @@ /* Generated By:JavaCC: Do not edit this line. StandardTokenizer.java */ package org.apache.lucene.analysis.standard; +import org.apache.lucene.analysis.Tokenizer; + import java.io.*; /** A grammar-based tokenizer constructed with JavaCC. @@ -11,13 +13,36 @@ * not suit your application, please consider copying this source code * directory to your project and maintaining your own grammar-based tokenizer. */ -public class StandardTokenizer extends org.apache.lucene.analysis.Tokenizer implements StandardTokenizerConstants { +public class StandardTokenizer extends org.apache.lucene.analysis.AbstractTokenizer implements Tokenizer, StandardTokenizerConstants { - /** Constructs a tokenizer for this Reader. */ - public StandardTokenizer(Reader reader) { - this(new FastCharStream(reader)); - this.input = reader; - } + public StandardTokenizer(Reader reader) { + this((StandardTokenizerTokenManager)null); + setReader(reader); + } + + public StandardTokenizer(StandardTokenizerTokenManager tm) { + token_source = tm; + token = new Token(); + jj_ntk = -1; + jj_gen = 0; + for (int i = 0; i < 1; i++) jj_la1[i] = -1; + } + + + public StandardTokenizer(CharStream stream) { + this(new StandardTokenizerTokenManager(stream)); + } + + public void setReader(Reader reader) { + token_source = new StandardTokenizerTokenManager(new FastCharStream(reader)); + this.reader = reader; + } + + public void init() { + super.init(); + ReInit(new FastCharStream(reader)); + + } /** Returns the next token in the stream, or null at EOS. *

The returned token's type is set to an element of {@link @@ -82,13 +107,6 @@ jj_la1_0 = new int[] {0x10ff,}; } - public StandardTokenizer(CharStream stream) { - token_source = new StandardTokenizerTokenManager(stream); - token = new Token(); - jj_ntk = -1; - jj_gen = 0; - for (int i = 0; i < 1; i++) jj_la1[i] = -1; - } public void ReInit(CharStream stream) { token_source.ReInit(stream); @@ -98,13 +116,7 @@ for (int i = 0; i < 1; i++) jj_la1[i] = -1; } - public StandardTokenizer(StandardTokenizerTokenManager tm) { - token_source = tm; - token = new Token(); - jj_ntk = -1; - jj_gen = 0; - for (int i = 0; i < 1; i++) jj_la1[i] = -1; - } + public void ReInit(StandardTokenizerTokenManager tm) { token_source = tm; Index: src/test/org/apache/lucene/queryParser/TestQueryParser.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/test/org/apache/lucene/queryParser/TestQueryParser.java,v retrieving revision 1.25 diff -u -r1.25 TestQueryParser.java --- src/test/org/apache/lucene/queryParser/TestQueryParser.java 3 Mar 2004 12:07:13 -0000 1.25 +++ src/test/org/apache/lucene/queryParser/TestQueryParser.java 3 Apr 2004 21:36:25 -0000 @@ -17,13 +17,8 @@ */ import junit.framework.TestCase; -import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.LowerCaseTokenizer; -import org.apache.lucene.analysis.SimpleAnalyzer; +import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenFilter; -import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.DateField; import org.apache.lucene.search.BooleanQuery; @@ -46,13 +41,13 @@ public static Analyzer qpAnalyzer = new QPTestAnalyzer(); - public static class QPTestFilter extends TokenFilter { + public static class QPTestFilter extends AbstractTokenFilter { /** * Filter which discards the token 'stop' and which expands the * token 'phrase' into 'phrase1 phrase2' */ public QPTestFilter(TokenStream in) { - super(in); + setTokenStream(in); } boolean inPhrase = false; Index: src/test/org/apache/lucene/search/TestPositionIncrement.java =================================================================== RCS file: /home/cvspublic/jakarta-lucene/src/test/org/apache/lucene/search/TestPositionIncrement.java,v retrieving revision 1.4 diff -u -r1.4 TestPositionIncrement.java --- src/test/org/apache/lucene/search/TestPositionIncrement.java 29 Mar 2004 22:48:06 -0000 1.4 +++ src/test/org/apache/lucene/search/TestPositionIncrement.java 3 Apr 2004 21:36:26 -0000 @@ -28,7 +28,7 @@ import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; - +import org.apache.lucene.analysis.AbstractTokenFilter; import java.io.Reader; import java.io.IOException; import java.io.StringReader; @@ -50,6 +50,10 @@ private final String[] TOKENS = {"1", "2", "3", "4", "5"}; private final int[] INCREMENTS = {1, 2, 1, 0, 1}; private int i = 0; + + public void close() throws IOException { + + } public Token next() throws IOException { if (i == TOKENS.length)