Index: common-build.xml =================================================================== --- common-build.xml (revision 704800) +++ common-build.xml (working copy) @@ -53,8 +53,8 @@ - - + + Index: src/java/org/apache/lucene/analysis/CachingTokenFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/CachingTokenFilter.java (revision 704800) +++ src/java/org/apache/lucene/analysis/CachingTokenFilter.java (working copy) @@ -40,6 +40,26 @@ super(input); } + public boolean incrementToken() throws IOException { + assert reusableToken != null; + if (cache == null) { + // fill cache lazily + cache = new LinkedList(); + fillCache(); + iterator = cache.iterator(); + } + + if (!iterator.hasNext()) { + // the cache is exhausted, return null + return false; + } + // Since the TokenFilter can be reset, the tokens need to be preserved as immutable. + Token t = (Token) iterator.next(); + reusableToken.copyFrom(t); + return true; + } + + /** @deprecated */ public Token next(final Token reusableToken) throws IOException { assert reusableToken != null; if (cache == null) { @@ -60,10 +80,17 @@ public void reset() throws IOException { if(cache != null) { - iterator = cache.iterator(); + iterator = cache.iterator(); } } + private void fillCache() throws IOException { + while(input.incrementToken()) { + cache.add((Token)reusableToken.clone()); + } + } + + /** @deprecated */ private void fillCache(final Token reusableToken) throws IOException { for (Token nextToken = input.next(reusableToken); nextToken != null; nextToken = input.next(reusableToken)) { cache.add(nextToken.clone()); Index: src/java/org/apache/lucene/analysis/CharTokenizer.java =================================================================== --- src/java/org/apache/lucene/analysis/CharTokenizer.java (revision 704800) +++ src/java/org/apache/lucene/analysis/CharTokenizer.java (working copy) @@ -20,6 +20,8 @@ import java.io.IOException; import java.io.Reader; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; + /** An abstract base class for simple, character-oriented tokenizers.*/ public abstract class CharTokenizer extends Tokenizer { public CharTokenizer(Reader input) { @@ -30,6 +32,8 @@ private static final int MAX_WORD_LEN = 255; private static final int IO_BUFFER_SIZE = 4096; private final char[] ioBuffer = new char[IO_BUFFER_SIZE]; + + private OffsetAttribute offsetAtt; /** Returns true iff a character should be included in a token. This * tokenizer generates as tokens adjacent sequences of characters which @@ -43,7 +47,56 @@ protected char normalize(char c) { return c; } + + protected void addTokenAttributes() { + offsetAtt = reusableToken.addAttribute(OffsetAttribute.class); + } + public final boolean incrementToken() throws IOException { + assert reusableToken != null; + reusableToken.clear(); + int length = 0; + int start = bufferIndex; + char[] buffer = reusableToken.termBuffer(); + while (true) { + + if (bufferIndex >= dataLen) { + offset += dataLen; + dataLen = input.read(ioBuffer); + if (dataLen == -1) { + if (length > 0) + break; + else + return false; + } + bufferIndex = 0; + } + + final char c = ioBuffer[bufferIndex++]; + + if (isTokenChar(c)) { // if it's a token char + + if (length == 0) // start of token + start = offset + bufferIndex - 1; + else if (length == buffer.length) + buffer = reusableToken.resizeTermBuffer(1+length); + + buffer[length++] = normalize(c); // buffer it, normalized + + if (length == MAX_WORD_LEN) // buffer overflow! + break; + + } else if (length > 0) // at non-Letter w/ chars + break; // return 'em + } + + reusableToken.setTermLength(length); + offsetAtt.setStartOffset(start); + offsetAtt.setEndOffset(start+length); + return true; + } + + /** @deprecated */ public final Token next(final Token reusableToken) throws IOException { assert reusableToken != null; reusableToken.clear(); Index: src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java (revision 704800) +++ src/java/org/apache/lucene/analysis/ISOLatin1AccentFilter.java (working copy) @@ -32,6 +32,28 @@ private char[] output = new char[256]; private int outputPos; + public final boolean incrementToken() throws java.io.IOException { + assert reusableToken != null; + + if (input.incrementToken()) { + final char[] buffer = reusableToken.termBuffer(); + final int length = reusableToken.termLength(); + // If no characters actually require rewriting then we + // just return token as-is: + for(int i=0;i= '\u00c0' && c <= '\uFB06') { + removeAccents(buffer, length); + reusableToken.setTermBuffer(output, 0, outputPos); + break; + } + } + return true; + } else + return false; + } + + /** @deprecated */ public final Token next(final Token reusableToken) throws java.io.IOException { assert reusableToken != null; Token nextToken = input.next(reusableToken); Index: src/java/org/apache/lucene/analysis/KeywordTokenizer.java =================================================================== --- src/java/org/apache/lucene/analysis/KeywordTokenizer.java (revision 704800) +++ src/java/org/apache/lucene/analysis/KeywordTokenizer.java (working copy) @@ -37,7 +37,28 @@ super(input); this.done = false; } + + public boolean incrementToken() throws IOException { + assert reusableToken != null; + if (!done) { + done = true; + int upto = 0; + reusableToken.clear(); + char[] buffer = reusableToken.termBuffer(); + while (true) { + final int length = input.read(buffer, upto, buffer.length-upto); + if (length == -1) break; + upto += length; + if (upto == buffer.length) + buffer = reusableToken.resizeTermBuffer(1+buffer.length); + } + reusableToken.setTermLength(upto); + return true; + } + return false; + } + /** @deprecated */ public Token next(final Token reusableToken) throws IOException { assert reusableToken != null; if (!done) { Index: src/java/org/apache/lucene/analysis/LengthFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/LengthFilter.java (revision 704800) +++ src/java/org/apache/lucene/analysis/LengthFilter.java (working copy) @@ -40,9 +40,28 @@ this.min = min; this.max = max; } + + /** + * Returns the next input Token whose term() is the right len + */ + public final boolean incrementToken() throws IOException + { + assert reusableToken != null; + // return the first non-stop word found + while (input.incrementToken()) { + int len = reusableToken.termLength(); + if (len >= min && len <= max) { + return true; + } + // note: else we ignore it but should we index each part of it? + } + // reached EOS -- return null + return false; + } /** * Returns the next input Token whose term() is the right len + * @deprecated */ public final Token next(final Token reusableToken) throws IOException { Index: src/java/org/apache/lucene/analysis/LowerCaseFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/LowerCaseFilter.java (revision 704800) +++ src/java/org/apache/lucene/analysis/LowerCaseFilter.java (working copy) @@ -29,6 +29,22 @@ super(in); } + public final boolean incrementToken() throws IOException { + assert reusableToken != null; + + if (input.incrementToken()) { + + final char[] buffer = reusableToken.termBuffer(); + final int length = reusableToken.termLength(); + for(int i=0;iRemoves 's from the end of words. *

Removes dots from acronyms. */ + public final boolean incrementToken() throws java.io.IOException { + assert reusableToken != null; + if (!input.incrementToken()) { + return false; + } + + char[] buffer = reusableToken.termBuffer(); + final int bufferLength = reusableToken.termLength(); + final String type = typeAtt.type(); + + if (type == APOSTROPHE_TYPE && // remove 's + bufferLength >= 2 && + buffer[bufferLength-2] == '\'' && + (buffer[bufferLength-1] == 's' || buffer[bufferLength-1] == 'S')) { + // Strip last 2 characters off + reusableToken.setTermLength(bufferLength - 2); + } else if (type == ACRONYM_TYPE) { // remove dots + int upto = 0; + for(int i=0;iRemoves 's from the end of words. + *

Removes dots from acronyms. + * @deprecated + */ public final Token next(final Token reusableToken) throws java.io.IOException { assert reusableToken != null; Token nextToken = input.next(reusableToken); Index: src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java =================================================================== --- src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (revision 704800) +++ src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (working copy) @@ -22,6 +22,9 @@ import org.apache.lucene.analysis.Token; import org.apache.lucene.analysis.Tokenizer; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.analysis.tokenattributes.TypeAttribute; /** A grammar-based tokenizer constructed with JFlex * @@ -127,11 +130,68 @@ this.scanner = new StandardTokenizerImpl(input); } + // this tokenizer generates three attributes: + // offset, positionIncrement and type + private OffsetAttribute offsetAtt; + private PositionIncrementAttribute posIncrAtt; + private TypeAttribute typeAtt; + + protected final void addTokenAttributes() { + offsetAtt = reusableToken.addAttribute(OffsetAttribute.class); + posIncrAtt = reusableToken.addAttribute(PositionIncrementAttribute.class); + typeAtt = reusableToken.addAttribute(TypeAttribute.class); + } + /* * (non-Javadoc) * * @see org.apache.lucene.analysis.TokenStream#next() */ + public boolean incrementToken() throws IOException { + assert reusableToken != null; + int posIncr = 1; + + while(true) { + int tokenType = scanner.getNextToken(); + + if (tokenType == StandardTokenizerImpl.YYEOF) { + return false; + } + + if (scanner.yylength() <= maxTokenLength) { + reusableToken.clear(); + posIncrAtt.setPositionIncrement(posIncr); + scanner.getText(reusableToken); + final int start = scanner.yychar(); + offsetAtt.setStartOffset(start); + offsetAtt.setEndOffset(start+reusableToken.termLength()); + // This 'if' should be removed in the next release. For now, it converts + // invalid acronyms to HOST. When removed, only the 'else' part should + // remain. + if (tokenType == StandardTokenizerImpl.ACRONYM_DEP) { + if (replaceInvalidAcronym) { + typeAtt.setType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.HOST]); + reusableToken.setTermLength(reusableToken.termLength() - 1); // remove extra '.' + } else { + typeAtt.setType(StandardTokenizerImpl.TOKEN_TYPES[StandardTokenizerImpl.ACRONYM]); + } + } else { + typeAtt.setType(StandardTokenizerImpl.TOKEN_TYPES[tokenType]); + } + return true; + } else + // When we skip a too-long term, we still increment the + // position increment + posIncr++; + } + } + + /* + * (non-Javadoc) + * + * @see org.apache.lucene.analysis.TokenStream#next() + */ + /** @deprecated */ public Token next(final Token reusableToken) throws IOException { assert reusableToken != null; int posIncr = 1; Index: src/java/org/apache/lucene/analysis/StopFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/StopFilter.java (revision 704800) +++ src/java/org/apache/lucene/analysis/StopFilter.java (working copy) @@ -21,6 +21,8 @@ import java.util.Arrays; import java.util.Set; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; + /** * Removes stop words from a token stream. */ @@ -109,10 +111,37 @@ stopSet.addAll(Arrays.asList(stopWords)); return stopSet; } + + private PositionIncrementAttribute posIncrAtt = null; + + protected final void addTokenAttributes() { + posIncrAtt = reusableToken.addAttribute(PositionIncrementAttribute.class); + } /** * Returns the next input Token whose term() is not a stop word. */ + public final boolean incrementToken() throws IOException { + assert reusableToken != null; + // return the first non-stop word found + int skippedPositions = 0; + while (input.incrementToken()) { + if (!stopWords.contains(reusableToken.termBuffer(), 0, reusableToken.termLength())) { + if (enablePositionIncrements) { + posIncrAtt.setPositionIncrement(posIncrAtt.getPositionIncrement() + skippedPositions); + } + return true; + } + skippedPositions += posIncrAtt.getPositionIncrement(); + } + // reached EOS -- return null + return false; + } + + /** + * Returns the next input Token whose term() is not a stop word. + * @deprecated + */ public final Token next(final Token reusableToken) throws IOException { assert reusableToken != null; // return the first non-stop word found Index: src/java/org/apache/lucene/analysis/TeeTokenFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/TeeTokenFilter.java (revision 704800) +++ src/java/org/apache/lucene/analysis/TeeTokenFilter.java (working copy) @@ -62,6 +62,16 @@ this.sink = sink; } + public boolean incrementToken() throws IOException { + assert reusableToken != null; + if (input.incrementToken()) { + sink.add(reusableToken); + return true; + } + return false; + } + + /** @deprecated */ public Token next(final Token reusableToken) throws IOException { assert reusableToken != null; Token nextToken = input.next(reusableToken); Index: src/java/org/apache/lucene/analysis/Token.java =================================================================== --- src/java/org/apache/lucene/analysis/Token.java (revision 704800) +++ src/java/org/apache/lucene/analysis/Token.java (working copy) @@ -17,6 +17,12 @@ * limitations under the License. */ +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; + +import org.apache.lucene.analysis.tokenattributes.Attribute; import org.apache.lucene.index.Payload; import org.apache.lucene.index.TermPositions; // for javadoc import org.apache.lucene.util.ArrayUtil; @@ -120,7 +126,53 @@ public static final String DEFAULT_TYPE = "word"; private static int MIN_BUFFER_SIZE = 10; + + private transient Map, Attribute> attributes; + public T addAttribute(Class attClass) { + if (this.attributes == null) { + this.attributes = new HashMap, Attribute>(); + } + + T att = (T) attributes.get(attClass); + if (att == null) { + try { + att = attClass.newInstance(); + } catch (InstantiationException e) { + throw new IllegalArgumentException("Could not instantiate class " + attClass); + } catch (IllegalAccessException e) { + throw new IllegalArgumentException("Could not instantiate class " + attClass); + } + + attributes.put(attClass, att); + } + return att; + } + + public boolean hasAttribute(Class attClass) { + return this.attributes.containsKey(attClass); + } + + public T getAttribute(Class attClass) { + Attribute att = this.attributes.get(attClass); + if (att == null) { + throw new IllegalArgumentException("This token does not have the attribute '" + attClass + "'."); + } + + return (T) att; + } + + public void copyFrom(Token prototype) { + prototype.initTermBuffer(); + setTermBuffer(prototype.termBuffer, 0, prototype.termLength); + Iterator, Attribute>> it = prototype.attributes.entrySet().iterator(); + while (it.hasNext()) { + Entry, Attribute> entry = it.next(); + getAttribute(entry.getKey()).copyFrom(entry.getValue()); + } + } + + /** @deprecated We will remove this when we remove the * deprecated APIs */ private String termText; @@ -615,6 +667,14 @@ sb.append(",type=").append(type); if (positionIncrement != 1) sb.append(",posIncr=").append(positionIncrement); + + if (attributes != null) { + Iterator it = attributes.values().iterator(); + while (it.hasNext()) { + sb.append(','); + sb.append(it.next().toString()); + } + } sb.append(')'); return sb.toString(); } @@ -643,6 +703,15 @@ if (payload != null) { t.setPayload((Payload) payload.clone()); } + if (attributes != null) { + t.attributes = new HashMap, Attribute>(); + Iterator, Attribute>> it = attributes.entrySet().iterator(); + while (it.hasNext()) { + Entry, Attribute> entry = it.next(); + Attribute clone = (Attribute) entry.getValue().clone(); + t.attributes.put(entry.getKey(), clone); + } + } return t; } catch (CloneNotSupportedException e) { throw new RuntimeException(e); // shouldn't happen Index: src/java/org/apache/lucene/analysis/tokenattributes/Attribute.java =================================================================== --- src/java/org/apache/lucene/analysis/tokenattributes/Attribute.java (revision 0) +++ src/java/org/apache/lucene/analysis/tokenattributes/Attribute.java (revision 0) @@ -0,0 +1,17 @@ +package org.apache.lucene.analysis.tokenattributes; + +import java.io.Serializable; + +public abstract class Attribute implements Cloneable, Serializable { + public abstract void clear(); + public abstract String toString(); + public abstract void copyFrom(Attribute prototype); + + public Attribute() { + // empty ctor used to instantiate by reflection + } + + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } +} Index: src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java =================================================================== --- src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java (revision 0) +++ src/java/org/apache/lucene/analysis/tokenattributes/FlagsAttribute.java (revision 0) @@ -0,0 +1,45 @@ +package org.apache.lucene.analysis.tokenattributes; + +import java.io.Serializable; + +public class FlagsAttribute extends Attribute implements Cloneable, Serializable { + private int flags = 0; + + /** + * EXPERIMENTAL: While we think this is here to stay, we may want to change it to be a long. + *

+ * + * Get the bitset for any bits that have been set. This is completely distinct from {@link #type()}, although they do share similar purposes. + * The flags can be used to encode information about the token for use by other {@link org.apache.lucene.analysis.TokenFilter}s. + * + * + * @return The bits + */ + public int getFlags() { + return flags; + } + + /** + * @see #getFlags() + */ + public void setFlags(int flags) { + this.flags = flags; + } + + @Override + public void clear() { + flags = 0; + } + + @Override + public void copyFrom(Attribute prototype) { + FlagsAttribute flagsAtt = (FlagsAttribute) prototype; + flags = flagsAtt.flags; + } + + @Override + public String toString() { + return "flags=" + flags; + } + +} Index: src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java =================================================================== --- src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java (revision 0) +++ src/java/org/apache/lucene/analysis/tokenattributes/OffsetAttribute.java (revision 0) @@ -0,0 +1,57 @@ +package org.apache.lucene.analysis.tokenattributes; + +import java.io.Serializable; + +public class OffsetAttribute extends Attribute implements Cloneable, Serializable { + private int startOffset; + private int endOffset; + + /** Returns this Token's starting offset, the position of the first character + corresponding to this token in the source text. + + Note that the difference between endOffset() and startOffset() may not be + equal to termText.length(), as the term text may have been altered by a + stemmer or some other filter. */ + public final int startOffset() { + return startOffset; + } + + /** Set the starting offset. + @see #startOffset() */ + public void setStartOffset(int offset) { + this.startOffset = offset; + } + + /** Returns this Token's ending offset, one greater than the position of the + last character corresponding to this token in the source text. The length + of the token in the source text is (endOffset - startOffset). */ + public final int endOffset() { + return endOffset; + } + + /** Set the ending offset. + @see #endOffset() */ + public void setEndOffset(int offset) { + this.endOffset = offset; + } + + public void clear() { + startOffset = 0; + endOffset = 0; + } + + public String toString() { + return "start=" + startOffset + ",end=" + endOffset; + } + + public void copyFrom(Attribute prototype) { + OffsetAttribute p = (OffsetAttribute) prototype; + startOffset = p.startOffset; + endOffset = p.endOffset; + } + + public Object clone() throws CloneNotSupportedException { + return super.clone(); + } + +} Index: src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java =================================================================== --- src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java (revision 0) +++ src/java/org/apache/lucene/analysis/tokenattributes/PayloadAttribute.java (revision 0) @@ -0,0 +1,56 @@ +package org.apache.lucene.analysis.tokenattributes; + +import java.io.Serializable; + +import org.apache.lucene.index.Payload; + +public class PayloadAttribute extends Attribute implements Cloneable, Serializable { + private Payload payload; + + public PayloadAttribute() {} + + public PayloadAttribute(Payload payload) { + this.payload = payload; + } + + /** + * Returns this Token's payload. + */ + public Payload getPayload() { + return this.payload; + } + + /** + * Sets this Token's payload. + */ + public void setPayload(Payload payload) { + this.payload = payload; + } + + @Override + public void clear() { + payload = null; + } + + @Override + public void copyFrom(Attribute prototype) { + PayloadAttribute payloadAtt = (PayloadAttribute) prototype; + if (payloadAtt.payload != null) { + this.payload = (Payload) payloadAtt.payload.clone(); + } + } + + @Override + public String toString() { + return payload.toString(); + } + + public Object clone() throws CloneNotSupportedException { + PayloadAttribute clone = (PayloadAttribute) super.clone(); + if (payload != null) { + clone.payload = (Payload) payload.clone(); + } + return clone; + } + +} Index: src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java =================================================================== --- src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java (revision 0) +++ src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java (revision 0) @@ -0,0 +1,62 @@ +package org.apache.lucene.analysis.tokenattributes; + +import java.io.Serializable; + +import org.apache.lucene.analysis.TokenStream; + +public class PositionIncrementAttribute extends Attribute implements Cloneable, Serializable { + private int positionIncrement = 1; + + /** Set the position increment. This determines the position of this token + * relative to the previous Token in a {@link TokenStream}, used in phrase + * searching. + * + *

The default value is one. + * + *

Some common uses for this are:

    + * + *
  • Set it to zero to put multiple terms in the same position. This is + * useful if, e.g., a word has multiple stems. Searches for phrases + * including either stem will match. In this case, all but the first stem's + * increment should be set to zero: the increment of the first instance + * should be one. Repeating a token with an increment of zero can also be + * used to boost the scores of matches on that token. + * + *
  • Set it to values greater than one to inhibit exact phrase matches. + * If, for example, one does not want phrases to match across removed stop + * words, then one could build a stop word filter that removes stop words and + * also sets the increment to the number of stop words removed before each + * non-stop word. Then exact phrase queries will only match when the terms + * occur with no intervening stop words. + * + *
+ * @param positionIncrement the distance from the prior term + * @see org.apache.lucene.index.TermPositions + */ + public void setPositionIncrement(int positionIncrement) { + if (positionIncrement < 0) + throw new IllegalArgumentException + ("Increment must be zero or greater: " + positionIncrement); + this.positionIncrement = positionIncrement; + } + + /** Returns the position increment of this Token. + * @see #setPositionIncrement + */ + public int getPositionIncrement() { + return positionIncrement; + } + + public void clear() { + this.positionIncrement = 1; + } + + public String toString() { + return "positionIncrement=" + positionIncrement; + } + + public void copyFrom(Attribute prototype) { + PositionIncrementAttribute p = (PositionIncrementAttribute) prototype; + positionIncrement = p.positionIncrement; + } +} Index: src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java =================================================================== --- src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java (revision 0) +++ src/java/org/apache/lucene/analysis/tokenattributes/TypeAttribute.java (revision 0) @@ -0,0 +1,44 @@ +package org.apache.lucene.analysis.tokenattributes; + +import java.io.Serializable; + +public class TypeAttribute extends Attribute implements Cloneable, Serializable { + private String type; + public static final String DEFAULT_TYPE = "word"; + + public TypeAttribute() { + this(DEFAULT_TYPE); + } + + public TypeAttribute(String type) { + this.type = type; + } + + /** Returns this Token's lexical type. Defaults to "word". */ + public final String type() { + return type; + } + + /** Set the lexical type. + @see #type() */ + public final void setType(String type) { + this.type = type; + } + + @Override + public void clear() { + type = DEFAULT_TYPE; + } + + @Override + public void copyFrom(Attribute prototype) { + // String is immutable, no clone necessary + TypeAttribute typeAtt = (TypeAttribute) prototype; + type = typeAtt.type; + } + + @Override + public String toString() { + return "type=" + type; + } +} Index: src/java/org/apache/lucene/analysis/TokenFilter.java =================================================================== --- src/java/org/apache/lucene/analysis/TokenFilter.java (revision 704800) +++ src/java/org/apache/lucene/analysis/TokenFilter.java (working copy) @@ -34,7 +34,16 @@ protected TokenFilter(TokenStream input) { this.input = input; } - + + public final Token getToken() throws IOException { + if (reusableToken == null) { + reusableToken = input.getToken(); + addTokenAttributes(); + } + + return reusableToken; + } + /** Close the input TokenStream. */ public void close() throws IOException { input.close(); Index: src/java/org/apache/lucene/analysis/TokenStream.java =================================================================== --- src/java/org/apache/lucene/analysis/TokenStream.java (revision 704800) +++ src/java/org/apache/lucene/analysis/TokenStream.java (working copy) @@ -37,7 +37,21 @@ */ public abstract class TokenStream { - + protected Token reusableToken = null; + + /** @deprecated */ + private static boolean useNewAPI = false; + + /** @deprecated */ + public static boolean isUseNewAPI() { + return useNewAPI; + } + + /** @deprecated */ + public static void setUseNewAPI(boolean use) { + useNewAPI = use; + } + /** Returns the next token in the stream, or null at EOS. * @deprecated The returned Token is a "full private copy" (not * re-used across calls to next()) but will be slower @@ -83,6 +97,9 @@ * return; this parameter should never be null (the callee * is not required to check for null before using it, but it is a * good idea to assert that it is not null.) + * @deprecated This method will be removed in Lucene 3.0. Use the + * new {@link #getToken()} and {@link #incrementToken()} APIs + * instead. * @return next token in the stream or null if end-of-stream was hit */ public Token next(final Token reusableToken) throws IOException { @@ -90,6 +107,23 @@ assert reusableToken != null; return next(); } + + public Token getToken() throws IOException { + if (reusableToken == null) { + reusableToken = new Token(); + addTokenAttributes(); + } + + return reusableToken; + } + + protected void addTokenAttributes() { + // don't add any attributes by default + } + + public boolean incrementToken() throws IOException { + return false; + } /** Resets this stream to the beginning. This is an * optional operation, so subclasses may or may not Index: src/java/org/apache/lucene/index/DocInverterPerField.java =================================================================== --- src/java/org/apache/lucene/index/DocInverterPerField.java (revision 704800) +++ src/java/org/apache/lucene/index/DocInverterPerField.java (working copy) @@ -122,7 +122,15 @@ try { int offsetEnd = fieldState.offset-1; - final Token localToken = perThread.localToken; + + boolean useNewTokenStreamAPI = TokenStream.isUseNewAPI(); + Token localToken; + if (useNewTokenStreamAPI) { + localToken = stream.getToken(); + } else { + localToken = perThread.localToken; + } + for(;;) { // If we hit an exception in stream.next below @@ -131,7 +139,16 @@ // non-aborting and (above) this one document // will be marked as deleted, but still // consume a docID - Token token = stream.next(localToken); + Token token; + if (useNewTokenStreamAPI) { + if (stream.incrementToken()) { + token = localToken; + } else { + token = null; + } + } else { + token = stream.next(localToken); + } if (token == null) break; fieldState.position += (token.getPositionIncrement() - 1); Index: src/test/org/apache/lucene/analysis/TestNewTokenStreamAPI.java =================================================================== --- src/test/org/apache/lucene/analysis/TestNewTokenStreamAPI.java (revision 0) +++ src/test/org/apache/lucene/analysis/TestNewTokenStreamAPI.java (revision 0) @@ -0,0 +1,170 @@ +package org.apache.lucene.analysis; + +import java.io.IOException; +import java.io.StringReader; + +import org.apache.lucene.analysis.tokenattributes.FlagsAttribute; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PayloadAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.apache.lucene.util.LuceneTestCase; + +public class TestNewTokenStreamAPI extends LuceneTestCase { + public void test() {} + + static StringReader getStringReader() { + String s = "Hello, this is the new Token API!!"; + return new StringReader(s); + } + + static void _testPerformanceNew(int numTokens, int numReads) throws Exception { + TokenStream ts = new CachingTokenFilter(new RepeatingTokenFilter(new WhitespaceTokenizer(getStringReader()), numTokens)); + Token reusableToken = ts.getToken(); + reusableToken.addAttribute(PayloadAttribute.class); + reusableToken.addAttribute(PositionIncrementAttribute.class); + reusableToken.addAttribute(FlagsAttribute.class); + reusableToken.addAttribute(TypeAttribute.class); + + for (int i = 0; i < numReads; i++) { + while (ts.incrementToken()) { + } + ts.reset(); + } + + + } + + static void _testPerformanceOld(int numTokens, int numReads) throws Exception { + TokenStream ts = new CachingTokenFilter(new RepeatingTokenFilter(new WhitespaceTokenizer(getStringReader()), numTokens)); + + Token reusableToken = new Token(); + + for (int i = 0; i < numReads; i++) { + while (reusableToken != null) { + reusableToken = ts.next(reusableToken); + } + ts.reset(); + reusableToken = new Token(); + } + + + } + + + static class RepeatingTokenFilter extends TokenFilter { + protected RepeatingTokenFilter(TokenStream input, int n) { + super(input); + this.n = n; + } + + int n; + public boolean incrementToken() throws IOException { + boolean next = input.incrementToken(); + if (!next && n > 0) { + n--; + ((WhitespaceTokenizer)input).reset(getStringReader()); + return input.incrementToken(); + } + + return next; + } + + public Token next(Token reusableToken) throws IOException { + Token token = input.next(reusableToken); + if (token == null && n > 0) { + n--; + ((WhitespaceTokenizer)input).reset(getStringReader()); + token = input.next(reusableToken); + } + + return token; + } + } + + static void _testPerformance() throws Exception { + final int numReads = 200; + final int numTokens = 10000; + + long start = System.currentTimeMillis(); + _testPerformanceOld(numTokens, numReads); + long end = System.currentTimeMillis(); + System.out.println("Time old: " + (end - start) + "ms."); + + start = System.currentTimeMillis(); + _testPerformanceNew(numTokens, numReads); + end = System.currentTimeMillis(); + System.out.println("Time new: " + (end - start) + "ms."); + } + + public static void main(String[] args) throws Exception { + TokenStream.setUseNewAPI(true); + //testPerformance(); + demo(); + } + + static void demo() throws Exception { + MyTokenFilter tf = new MyTokenFilter(new CachingTokenFilter(new WhitespaceTokenizer(getStringReader()))); + + Token token = tf.getToken(); + PositionIncrementAttribute posAtt = (PositionIncrementAttribute) token.getAttribute(PositionIncrementAttribute.class); + OffsetAttribute offAtt = (OffsetAttribute) token.getAttribute(OffsetAttribute.class); + + tf.setIncrement(5); + + int totalLength = 0; + int position = 0; + + while (tf.incrementToken()) { + System.out.println(token); + totalLength += (offAtt.endOffset() - offAtt.startOffset()); + position += posAtt.getPositionIncrement(); + } + + System.out.println("Position: " + position); + System.out.println("Total legnth: " + totalLength); + System.out.println("\n"); + + tf.reset(); + tf.setIncrement(10); + + totalLength = 0; + position = 0; + + while (tf.incrementToken()) { + System.out.println(token); + totalLength += (offAtt.endOffset() - offAtt.startOffset()); + position += posAtt.getPositionIncrement(); + } + + System.out.println("Position: " + position); + System.out.println("Total legnth: " + totalLength); + + } + + + private static class MyTokenFilter extends TokenFilter { + private PositionIncrementAttribute positionIncrement; + private int increment; + + protected MyTokenFilter(TokenStream input) { + super(input); + this.increment = 1; + } + + void setIncrement(int increment) { + this.increment = increment; + } + + protected final void addTokenAttributes() { + positionIncrement = reusableToken.addAttribute(PositionIncrementAttribute.class); + } + + public boolean incrementToken() throws IOException { + boolean next = input.incrementToken(); + positionIncrement.setPositionIncrement(increment); + return next; + } + } + +} Index: src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java =================================================================== --- src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java (revision 704800) +++ src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java (working copy) @@ -1,6 +1,9 @@ package org.apache.lucene.analysis; import org.apache.lucene.analysis.standard.StandardAnalyzer; +import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; +import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; +import org.apache.lucene.analysis.tokenattributes.TypeAttribute; import org.apache.lucene.util.LuceneTestCase; import java.io.StringReader; @@ -35,19 +38,21 @@ public void assertAnalyzesTo(Analyzer a, String input, String[] expectedImages, String[] expectedTypes, int[] expectedPosIncrs) throws Exception { TokenStream ts = a.tokenStream("dummy", new StringReader(input)); - final Token reusableToken = new Token(); + final Token reusableToken = ts.getToken(); + final TypeAttribute typeAtt = reusableToken.getAttribute(TypeAttribute.class); + final PositionIncrementAttribute posIncrAtt = reusableToken.getAttribute(PositionIncrementAttribute.class); + for (int i = 0; i < expectedImages.length; i++) { - Token nextToken = ts.next(reusableToken); - assertNotNull(nextToken); - assertEquals(expectedImages[i], nextToken.term()); + assertTrue(ts.incrementToken()); + assertEquals(expectedImages[i], new String(reusableToken.termBuffer(), 0, reusableToken.termLength())); if (expectedTypes != null) { - assertEquals(expectedTypes[i], nextToken.type()); + assertEquals(expectedTypes[i], typeAtt.type()); } if (expectedPosIncrs != null) { - assertEquals(expectedPosIncrs[i], nextToken.getPositionIncrement()); + assertEquals(expectedPosIncrs[i], posIncrAtt.getPositionIncrement()); } } - assertNull(ts.next(reusableToken)); + assertFalse(ts.incrementToken()); ts.close(); } Index: src/test/org/apache/lucene/util/LuceneTestCase.java =================================================================== --- src/test/org/apache/lucene/util/LuceneTestCase.java (revision 704800) +++ src/test/org/apache/lucene/util/LuceneTestCase.java (working copy) @@ -17,6 +17,7 @@ * limitations under the License. */ +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.index.ConcurrentMergeScheduler; import junit.framework.TestCase; @@ -42,6 +43,7 @@ protected void setUp() throws Exception { ConcurrentMergeScheduler.setTestMode(); + TokenStream.setUseNewAPI(false); } protected void tearDown() throws Exception {