Index: lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java =================================================================== --- lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java (revision 937906) +++ lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/ShingleFilterTest.java (working copy) @@ -34,7 +34,7 @@ protected int index = 0; protected Token[] testToken; - private TermAttribute termAtt; + private CharTermAttribute termAtt; private OffsetAttribute offsetAtt; private PositionIncrementAttribute posIncrAtt; private TypeAttribute typeAtt; @@ -42,7 +42,7 @@ public TestTokenStream(Token[] testToken) { super(); this.testToken = testToken; - this.termAtt = addAttribute(TermAttribute.class); + this.termAtt = addAttribute(CharTermAttribute.class); this.offsetAtt = addAttribute(OffsetAttribute.class); this.posIncrAtt = addAttribute(PositionIncrementAttribute.class); this.typeAtt = addAttribute(TypeAttribute.class); @@ -53,7 +53,7 @@ clearAttributes(); if (index < testToken.length) { Token t = testToken[index++]; - termAtt.setTermBuffer(t.termBuffer(), 0, t.termLength()); + termAtt.copyBuffer(t.buffer(), 0, t.length()); offsetAtt.setOffset(t.startOffset(), t.endOffset()); posIncrAtt.setPositionIncrement(t.getPositionIncrement()); typeAtt.setType(TypeAttributeImpl.DEFAULT_TYPE); @@ -103,19 +103,22 @@ createToken("please divide", 0, 13), createToken("divide", 7, 13), createToken("divide _", 7, 19), - createToken("_", 19, 19), createToken("_ sentence", 19, 27), createToken("sentence", 19, 27), createToken("sentence _", 19, 33), - createToken("_", 33, 33), createToken("_ shingles", 33, 39), createToken("shingles", 33, 39), }; public static final int[] BI_GRAM_POSITION_INCREMENTS_WITH_HOLES = new int[] { - 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1 + 1, 0, 1, 0, 1, 1, 0, 1, 1 }; + private static final String[] BI_GRAM_TYPES_WITH_HOLES = { + "word", "shingle", + "word", "shingle", "shingle", "word", "shingle", "shingle", "word" + }; + public static final Token[] BI_GRAM_TOKENS_WITHOUT_UNIGRAMS = new Token[] { createToken("please divide", 0, 13), createToken("divide this", 7, 18), @@ -642,18 +645,157 @@ "word" }; + public static final Token[] TEST_TOKEN_POS_INCR_EQUAL_TO_N = new Token[] { + createToken("please", 0, 6), + createToken("divide", 7, 13), + createToken("this", 14, 18), + createToken("sentence", 29, 37, 3), + createToken("into", 38, 42), + createToken("shingles", 43, 49), + }; + + public static final Token[] TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N = new Token[] { + createToken("please", 0, 6), + createToken("please divide", 0, 13), + createToken("please divide this", 0, 18), + createToken("divide", 7, 13), + createToken("divide this", 7, 18), + createToken("divide this _", 7, 29), + createToken("this", 14, 18), + createToken("this _", 14, 29), + createToken("this _ _", 14, 29), + createToken("_ _ sentence", 29, 37), + createToken("_ sentence", 29, 37), + createToken("_ sentence into", 29, 42), + createToken("sentence", 29, 37), + createToken("sentence into", 29, 42), + createToken("sentence into shingles", 29, 49), + createToken("into", 38, 42), + createToken("into shingles", 38, 49), + createToken("shingles", 43, 49) + }; + + public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N = new int[] { + 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1 + }; + + public static final String[] TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N = new String[] { + "word", "shingle", "shingle", + "word", "shingle", "shingle", + "word", "shingle", "shingle", + "shingle", "shingle", "shingle", "word", "shingle", "shingle", + "word", "shingle", + "word" + }; + + public static final Token[] TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new Token[] { + createToken("please divide", 0, 13), + createToken("please divide this", 0, 18), + createToken("divide this", 7, 18), + createToken("divide this _", 7, 29), + createToken("this _", 14, 29), + createToken("this _ _", 14, 29), + createToken("_ _ sentence", 29, 37), + createToken("_ sentence", 29, 37), + createToken("_ sentence into", 29, 42), + createToken("sentence into", 29, 42), + createToken("sentence into shingles", 29, 49), + createToken("into shingles", 38, 49), + }; + + public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new int[] { + 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1 + }; + + public static final String[] TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS = new String[] { + "shingle", "shingle", + "shingle", "shingle", + "shingle", "shingle", + "shingle", "shingle", "shingle", + "shingle", "shingle", + "shingle", + }; + + public static final Token[] TEST_TOKEN_POS_INCR_GREATER_THAN_N = new Token[] { + createToken("please", 0, 6), + createToken("divide", 57, 63, 8), + createToken("this", 64, 68), + createToken("sentence", 69, 77), + createToken("into", 78, 82), + createToken("shingles", 83, 89), + }; + + public static final Token[] TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N = new Token[] { + createToken("please", 0, 6), + createToken("please _", 0, 57), + createToken("please _ _", 0, 57), + createToken("_ _ divide", 57, 63), + createToken("_ divide", 57, 63), + createToken("_ divide this", 57, 68), + createToken("divide", 57, 63), + createToken("divide this", 57, 68), + createToken("divide this sentence", 57, 77), + createToken("this", 64, 68), + createToken("this sentence", 64, 77), + createToken("this sentence into", 64, 82), + createToken("sentence", 69, 77), + createToken("sentence into", 69, 82), + createToken("sentence into shingles", 69, 89), + createToken("into", 78, 82), + createToken("into shingles", 78, 89), + createToken("shingles", 83, 89) + }; + + public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N = new int[] { + 1, 0, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1 + }; + public static final String[] TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N = new String[] { + "word", "shingle", "shingle", + "shingle", + "shingle", "shingle", + "word", "shingle", "shingle", + "word", "shingle", "shingle", + "word", "shingle", "shingle", + "word", "shingle", + "word" + }; + + public static final Token[] TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new Token[] { + createToken("please _", 0, 57), + createToken("please _ _", 0, 57), + createToken("_ _ divide", 57, 63), + createToken("_ divide", 57, 63), + createToken("_ divide this", 57, 68), + createToken("divide this", 57, 68), + createToken("divide this sentence", 57, 77), + createToken("this sentence", 64, 77), + createToken("this sentence into", 64, 82), + createToken("sentence into", 69, 82), + createToken("sentence into shingles", 69, 89), + createToken("into shingles", 78, 89), + }; + + public static final int[] TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new int[] { + 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1 + }; + + public static final String[] TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS = new String[] { + "shingle", "shingle", + "shingle", "shingle", + "shingle", "shingle", + "shingle", "shingle", "shingle", "shingle", "shingle", + "shingle", + }; + @Override protected void setUp() throws Exception { super.setUp(); testTokenWithHoles = new Token[] { createToken("please", 0, 6), createToken("divide", 7, 13), - createToken("sentence", 19, 27), - createToken("shingles", 33, 39), + createToken("sentence", 19, 27, 2), + createToken("shingles", 33, 39, 2), }; - - testTokenWithHoles[2].setPositionIncrement(2); - testTokenWithHoles[3].setPositionIncrement(2); } /* @@ -667,7 +809,8 @@ public void testBiGramFilterWithHoles() throws IOException { this.shingleFilterTest(2, testTokenWithHoles, BI_GRAM_TOKENS_WITH_HOLES, - BI_GRAM_POSITION_INCREMENTS, BI_GRAM_TYPES, + BI_GRAM_POSITION_INCREMENTS_WITH_HOLES, + BI_GRAM_TYPES_WITH_HOLES, true); } @@ -832,8 +975,32 @@ TRI_GRAM_POSITION_INCREMENTS_NULL_SEPARATOR, TRI_GRAM_TYPES_NULL_SEPARATOR, true); } + + public void testPositionIncrementEqualToN() throws IOException { + this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_EQUAL_TO_N, TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N, + TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N, + TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N, true); + } + public void testPositionIncrementEqualToNWithoutUnigrams() throws IOException { + this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_EQUAL_TO_N, TRI_GRAM_TOKENS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, + TRI_GRAM_POSITION_INCREMENTS_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, + TRI_GRAM_TYPES_POS_INCR_EQUAL_TO_N_WITHOUT_UNIGRAMS, false); + } + + public void testPositionIncrementGreaterThanN() throws IOException { + this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_GREATER_THAN_N, TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N, + TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N, + TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N, true); + } + + public void testPositionIncrementGreaterThanNWithoutUnigrams() throws IOException { + this.shingleFilterTest(2, 3, TEST_TOKEN_POS_INCR_GREATER_THAN_N, TRI_GRAM_TOKENS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, + TRI_GRAM_POSITION_INCREMENTS_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, + TRI_GRAM_TYPES_POS_INCR_GREATER_THAN_N_WITHOUT_UNIGRAMS, false); + } + public void testReset() throws Exception { Tokenizer wsTokenizer = new WhitespaceTokenizer(TEST_VERSION_CURRENT, new StringReader("please divide this sentence")); TokenStream filter = new ShingleFilter(wsTokenizer, 2); @@ -896,18 +1063,24 @@ int endOffsets[] = new int[tokensToCompare.length]; for (int i = 0; i < tokensToCompare.length; i++) { - text[i] = tokensToCompare[i].term(); + text[i] = new String(tokensToCompare[i].buffer(),0, tokensToCompare[i].length()); startOffsets[i] = tokensToCompare[i].startOffset(); endOffsets[i] = tokensToCompare[i].endOffset(); } assertTokenStreamContents(filter, text, startOffsets, endOffsets, types, positionIncrements); } + + private static Token createToken(String term, int start, int offset) { + return createToken(term, start, offset, 1); + } - private static Token createToken(String term, int start, int offset) + private static Token createToken + (String term, int start, int offset, int positionIncrement) { Token token = new Token(start, offset); - token.setTermBuffer(term); + token.copyBuffer(term.toCharArray(), 0, term.length()); + token.setPositionIncrement(positionIncrement); return token; } } Index: lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java =================================================================== --- lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java (revision 937906) +++ lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleFilter.java (working copy) @@ -18,14 +18,16 @@ */ import java.io.IOException; +import java.util.Iterator; import java.util.LinkedList; import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import org.apache.lucene.analysis.tokenattributes.TypeAttribute; +import org.apache.lucene.util.AttributeSource; /** @@ -66,12 +68,12 @@ */ public static final String TOKEN_SEPARATOR = " "; - /** * The sequence of input stream tokens (or filler tokens, if necessary) * that will be composed to form output shingles. */ - private LinkedList inputWindow = new LinkedList(); + private LinkedList inputWindow + = new LinkedList(); /** * The number of input tokens in the next output token. This is the "n" in @@ -80,9 +82,9 @@ private CircularSequence gramSize; /** - * Shingle text is composed here. + * Shingle and unigram text is composed here. */ - private StringBuilder shingleBuilder = new StringBuilder(); + private StringBuilder gramBuilder = new StringBuilder(); /** * The token type attribute value to use - default is "shingle" @@ -111,18 +113,31 @@ private int minShingleSize; /** - * The remaining number of filler tokens inserted into the input stream + * The remaining number of filler tokens to be inserted into the input stream * from which shingles are composed, to handle position increments greater * than one. */ private int numFillerTokensToInsert; /** - * The next input stream token. + * When the next input stream token has a position increment greater than + * one, it is stored in this field until sufficient filler tokens have been + * inserted to account for the position increment. */ - private State nextInputStreamToken; + private AttributeSource nextInputStreamToken; + + /** + * Whether or not there is a next input stream token. + */ + private boolean isNextInputStreamToken = false; + + /** + * Whether at least one unigram or shingle has been output at the current + * position. + */ + private boolean isOutputHere = false; - private final TermAttribute termAtt; + private final CharTermAttribute termAtt; private final OffsetAttribute offsetAtt; private final PositionIncrementAttribute posIncrAtt; private final TypeAttribute typeAtt; @@ -140,7 +155,7 @@ super(input); setMaxShingleSize(maxShingleSize); setMinShingleSize(minShingleSize); - this.termAtt = addAttribute(TermAttribute.class); + this.termAtt = addAttribute(CharTermAttribute.class); this.offsetAtt = addAttribute(OffsetAttribute.class); this.posIncrAtt = addAttribute(PositionIncrementAttribute.class); this.typeAtt = addAttribute(TypeAttribute.class); @@ -241,109 +256,121 @@ this.tokenSeparator = null == tokenSeparator ? "" : tokenSeparator; } - /* (non-Javadoc) - * @see org.apache.lucene.analysis.TokenStream#next() - */ @Override public final boolean incrementToken() throws IOException { - boolean tokenAvailable = false; + boolean tokenAvailable = false; + int builtGramSize = 0; if (gramSize.atMinValue() || inputWindow.size() < gramSize.getValue()) { shiftInputWindow(); + gramBuilder.setLength(0); + } else { + builtGramSize = gramSize.getPreviousValue(); } - if ( ! inputWindow.isEmpty()) { - restoreState(inputWindow.getFirst()); - if (1 == gramSize.getValue()) { - posIncrAtt.setPositionIncrement(1); + if (inputWindow.size() >= gramSize.getValue()) { + boolean isAllFiller = true; + InputWindowToken nextToken = null; + Iterator iter = inputWindow.iterator(); + for (int gramNum = 1 ; + iter.hasNext() && builtGramSize < gramSize.getValue() ; + ++gramNum) { + nextToken = iter.next(); + if (builtGramSize < gramNum) { + if (builtGramSize > 0) { + gramBuilder.append(tokenSeparator); + } + gramBuilder.append(nextToken.termAtt.buffer(), 0, + nextToken.termAtt.length()); + ++builtGramSize; + } + if (isAllFiller && nextToken.isFiller) { + if (gramNum == gramSize.getValue()) { + gramSize.advance(); + } + } else { + isAllFiller = false; + } + } + if ( ! isAllFiller && builtGramSize == gramSize.getValue()) { + inputWindow.getFirst().attSource.copyTo(this); + posIncrAtt.setPositionIncrement(isOutputHere ? 0 : 1); + termAtt.setEmpty().append(gramBuilder); + if (gramSize.getValue() > 1) { + typeAtt.setType(tokenType); + } + offsetAtt.setOffset(offsetAtt.startOffset(), nextToken.offsetAtt.endOffset()); + isOutputHere = true; gramSize.advance(); tokenAvailable = true; - } else if (inputWindow.size() >= gramSize.getValue()) { - getNextShingle(); - gramSize.advance(); - tokenAvailable = true; } } return tokenAvailable; } /** - *

Makes the next token a shingle of length {@link #gramSize}, - * composed of tokens taken from {@link #inputWindow}. - *

Callers of this method must first insure that there are at least - * gramSize tokens available in inputWindow. - */ - private void getNextShingle() { - int startOffset = offsetAtt.startOffset(); - - int minTokNum = gramSize.getValue() - 1; // zero-based inputWindow position - if (gramSize.getValue() == minShingleSize) { - // Clear the shingle text buffer if this is the first shingle - // at the current position in the input stream. - shingleBuilder.setLength(0); - minTokNum = 0; - } - for (int tokNum = minTokNum ; tokNum < gramSize.getValue() ; ++tokNum) { - if (tokNum > 0) { - shingleBuilder.append(tokenSeparator); - } - restoreState(inputWindow.get(tokNum)); - shingleBuilder.append(termAtt.termBuffer(), 0, termAtt.termLength()); - } - char[] termBuffer = termAtt.termBuffer(); - int termLength = shingleBuilder.length(); - if (termBuffer.length < termLength) { - termBuffer = termAtt.resizeTermBuffer(termLength); - } - shingleBuilder.getChars(0, termLength, termBuffer, 0); - termAtt.setTermLength(termLength); - posIncrAtt.setPositionIncrement(gramSize.atMinValue() ? 1 : 0); - typeAtt.setType(tokenType); - offsetAtt.setOffset(startOffset, offsetAtt.endOffset()); - } - - /** *

Get the next token from the input stream. *

If the next token has positionIncrement > 1, * positionIncrement - 1 {@link #FILLER_TOKEN}s are * inserted first. - * @return false for end of stream; true otherwise + * @param target Where to put the new token; if null, a new instance is created. + * @return On success, the populated token; null otherwise * @throws IOException if the input stream has a problem */ - private boolean getNextToken() throws IOException { - boolean success = false; + private InputWindowToken getNextToken(InputWindowToken target) + throws IOException { + InputWindowToken newTarget = target; if (numFillerTokensToInsert > 0) { - insertFillerToken(); - success = true; - } else if (null != nextInputStreamToken) { - restoreState(nextInputStreamToken); - nextInputStreamToken = null; - success = true; + if (null == target) { + newTarget = new InputWindowToken(nextInputStreamToken.cloneAttributes()); + } else { + nextInputStreamToken.copyTo(target.attSource); + } + // A filler token occupies no space + newTarget.offsetAtt.setOffset(newTarget.offsetAtt.startOffset(), + newTarget.offsetAtt.startOffset()); + newTarget.termAtt.copyBuffer(FILLER_TOKEN, 0, FILLER_TOKEN.length); + newTarget.isFiller = true; + --numFillerTokensToInsert; + } else if (isNextInputStreamToken) { + if (null == target) { + newTarget = new InputWindowToken(nextInputStreamToken.cloneAttributes()); + } else { + nextInputStreamToken.copyTo(target.attSource); + } + isNextInputStreamToken = false; + newTarget.isFiller = false; } else if (input.incrementToken()) { + if (null == target) { + newTarget = new InputWindowToken(cloneAttributes()); + } else { + this.copyTo(target.attSource); + } if (posIncrAtt.getPositionIncrement() > 1) { - numFillerTokensToInsert = posIncrAtt.getPositionIncrement() - 1; - insertFillerToken(); + // Each output shingle must contain at least one input token, + // so no more than (maxShingleSize - 1) filler tokens will be inserted. + numFillerTokensToInsert + = Math.min(posIncrAtt.getPositionIncrement() - 1, maxShingleSize - 1); + // Save the current token as the next input stream token + if (null == nextInputStreamToken) { + nextInputStreamToken = cloneAttributes(); + } else { + this.copyTo(nextInputStreamToken); + } + isNextInputStreamToken = true; + // A filler token occupies no space + newTarget.offsetAtt.setOffset(offsetAtt.startOffset(), offsetAtt.startOffset()); + newTarget.termAtt.copyBuffer(FILLER_TOKEN, 0, FILLER_TOKEN.length); + newTarget.isFiller = true; + --numFillerTokensToInsert; + } else { + newTarget.isFiller = false; } - success = true; + } else { + newTarget = null; } - return success; + return newTarget; } /** - * Inserts a {@link #FILLER_TOKEN} and decrements - * {@link #numFillerTokensToInsert}. - */ - private void insertFillerToken() { - if (null == nextInputStreamToken) { - nextInputStreamToken = captureState(); - } else { - restoreState(nextInputStreamToken); - } - --numFillerTokensToInsert; - // A filler token occupies no space - offsetAtt.setOffset(offsetAtt.startOffset(), offsetAtt.startOffset()); - termAtt.setTermBuffer(FILLER_TOKEN, 0, FILLER_TOKEN.length); - } - - /** *

Fills {@link #inputWindow} with input stream tokens, if available, * shifting to the right if the window was previously full. *

Resets {@link #gramSize} to its minimum value. @@ -351,16 +378,29 @@ * @throws IOException if there's a problem getting the next token */ private void shiftInputWindow() throws IOException { + InputWindowToken firstToken = null; if (inputWindow.size() > 0) { - inputWindow.removeFirst(); + firstToken = inputWindow.removeFirst(); } - while (getNextToken()) { - inputWindow.add(captureState()); - if (inputWindow.size() == maxShingleSize) { - break; + while (inputWindow.size() < maxShingleSize) { + if (null != firstToken) { // recycle the firstToken, if available + if (null != getNextToken(firstToken)) { + inputWindow.add(firstToken); // the firstToken becomes the last + firstToken = null; + } else { + break; // end of input stream + } + } else { + InputWindowToken nextToken = getNextToken(null); + if (null != nextToken) { + inputWindow.add(nextToken); + } else { + break; // end of input stream + } } } gramSize.reset(); + isOutputHere = false; } @Override @@ -369,6 +409,7 @@ gramSize.reset(); inputWindow.clear(); numFillerTokensToInsert = 0; + isOutputHere = false; } @@ -383,6 +424,7 @@ */ private class CircularSequence { private int value; + private int previousValue; private int minValue; public CircularSequence() { @@ -405,10 +447,9 @@ * { [ 1, ] {@link #minShingleSize} [ , ... , {@link #maxShingleSize} ] }. *

1 is included in the circular sequence only if * {@link #outputUnigrams} = true. - * - * @return the next member in the circular sequence */ - public int advance() { + public void advance() { + previousValue = value; if (value == 1) { value = minShingleSize; } else if (value == maxShingleSize) { @@ -416,7 +457,6 @@ } else { ++value; } - return value; } /** @@ -428,7 +468,7 @@ * {@link #outputUnigrams} = true. */ public void reset() { - value = minValue; + previousValue = value = minValue; } /** @@ -443,5 +483,25 @@ public boolean atMinValue() { return value == minValue; } + + /** + * @return the value this instance had before the last advance() call + */ + public int getPreviousValue() { + return previousValue; + } } + + private class InputWindowToken { + final AttributeSource attSource; + final CharTermAttribute termAtt; + final OffsetAttribute offsetAtt; + boolean isFiller = false; + + public InputWindowToken(AttributeSource attSource) { + this.attSource = attSource; + this.termAtt = attSource.getAttribute(CharTermAttribute.class); + this.offsetAtt = attSource.getAttribute(OffsetAttribute.class); + } + } }