Index: src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java =================================================================== --- src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java (revision 899467) +++ src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java (working copy) @@ -79,25 +79,19 @@ final TeeSinkTokenFilter source = new TeeSinkTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString()))); final TokenStream sink1 = source.newSinkTokenStream(); final TokenStream sink2 = source.newSinkTokenStream(theFilter); - int i = 0; - TermAttribute termAtt = source.getAttribute(TermAttribute.class); - while (source.incrementToken()) { - assertEquals(tokens1[i], termAtt.term()); - i++; - } - assertEquals(tokens1.length, i); - i = 0; - termAtt = sink1.getAttribute(TermAttribute.class); - while (sink1.incrementToken()) { - assertEquals(tokens1[i], termAtt.term()); - i++; - } - assertEquals(tokens1.length, i); + source.addAttribute(CheckClearAttributesAttribute.class); + sink1.addAttribute(CheckClearAttributesAttribute.class); + CheckClearAttributesAttribute checkClearAtt = sink2.addAttribute(CheckClearAttributesAttribute.class); - i = 0; - termAtt = sink2.getAttribute(TermAttribute.class); + assertTokenStreamContents(source, tokens1); + assertTokenStreamContents(sink1, tokens1); + + int i = 0; + TermAttribute termAtt = sink2.getAttribute(TermAttribute.class); + checkClearAtt.getAndResetClearCalled(); // reset it while (sink2.incrementToken()) { + assertTrue("clearAttributes() was not correctly called in TokenStreamChain", checkClearAtt.getAndResetClearCalled()); assertTrue(termAtt.term().equalsIgnoreCase("The")); i++; } @@ -109,36 +103,35 @@ final TeeSinkTokenFilter.SinkTokenStream dogDetector = tee1.newSinkTokenStream(dogFilter); final TeeSinkTokenFilter.SinkTokenStream theDetector = tee1.newSinkTokenStream(theFilter); final TokenStream source1 = new CachingTokenFilter(tee1); + + tee1.addAttribute(CheckClearAttributesAttribute.class); + dogDetector.addAttribute(CheckClearAttributesAttribute.class); + theDetector.addAttribute(CheckClearAttributesAttribute.class); final TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(new WhitespaceTokenizer(new StringReader(buffer2.toString()))); tee2.addSinkTokenStream(dogDetector); tee2.addSinkTokenStream(theDetector); final TokenStream source2 = tee2; + assertTokenStreamContents(source1, tokens1); + assertTokenStreamContents(source2, tokens2); + int i = 0; - TermAttribute termAtt = source1.getAttribute(TermAttribute.class); - while (source1.incrementToken()) { - assertEquals(tokens1[i], termAtt.term()); - i++; - } - assertEquals(tokens1.length, i); - i = 0; - termAtt = source2.getAttribute(TermAttribute.class); - while (source2.incrementToken()) { - assertEquals(tokens2[i], termAtt.term()); - i++; - } - assertEquals(tokens2.length, i); - i = 0; - termAtt = theDetector.getAttribute(TermAttribute.class); + CheckClearAttributesAttribute checkClearAtt = theDetector.getAttribute(CheckClearAttributesAttribute.class); + TermAttribute termAtt = theDetector.getAttribute(TermAttribute.class); + checkClearAtt.getAndResetClearCalled(); // reset it while (theDetector.incrementToken()) { + assertTrue("clearAttributes() was not correctly called in TokenStreamChain", checkClearAtt.getAndResetClearCalled()); assertTrue("'" + termAtt.term() + "' is not equal to 'The'", termAtt.term().equalsIgnoreCase("The")); i++; } assertEquals("there must be 4 times 'The' in the stream", 4, i); i = 0; + checkClearAtt = dogDetector.getAttribute(CheckClearAttributesAttribute.class); termAtt = dogDetector.getAttribute(TermAttribute.class); + checkClearAtt.getAndResetClearCalled(); // reset it while (dogDetector.incrementToken()) { + assertTrue("clearAttributes() was not correctly called in TokenStreamChain", checkClearAtt.getAndResetClearCalled()); assertTrue("'" + termAtt.term() + "' is not equal to 'Dogs'", termAtt.term().equalsIgnoreCase("Dogs")); i++; } @@ -147,8 +140,11 @@ source1.reset(); TokenStream lowerCasing = new LowerCaseFilter(Version.LUCENE_CURRENT, source1); i = 0; + checkClearAtt = lowerCasing.getAttribute(CheckClearAttributesAttribute.class); termAtt = lowerCasing.getAttribute(TermAttribute.class); + checkClearAtt.getAndResetClearCalled(); // reset it while (lowerCasing.incrementToken()) { + assertTrue("clearAttributes() was not correctly called in TokenStreamChain", checkClearAtt.getAndResetClearCalled()); assertEquals(tokens1[i].toLowerCase(), termAtt.term()); i++; } Index: src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java =================================================================== --- src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 899467) +++ src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (working copy) @@ -21,6 +21,8 @@ import java.io.IOException; import org.apache.lucene.analysis.tokenattributes.*; +import org.apache.lucene.util.Attribute; +import org.apache.lucene.util.AttributeImpl; import org.apache.lucene.util.LuceneTestCase; /** @@ -38,8 +40,49 @@ // some helpers to test Analyzers and TokenStreams: + public static interface CheckClearAttributesAttribute extends Attribute { + boolean getAndResetClearCalled(); + } + + public static final class CheckClearAttributesAttributeImpl extends AttributeImpl implements CheckClearAttributesAttribute { + private boolean clearCalled = false; + + public boolean getAndResetClearCalled() { + try { + return clearCalled; + } finally { + clearCalled = false; + } + } + + @Override + public void clear() { + clearCalled = true; + } + + @Override + public boolean equals(Object other) { + return ( + other instanceof CheckClearAttributesAttributeImpl && + ((CheckClearAttributesAttributeImpl) other).clearCalled == this.clearCalled + ); + } + + @Override + public int hashCode() { + return 76137213 ^ Boolean.valueOf(clearCalled).hashCode(); + } + + @Override + public void copyTo(AttributeImpl target) { + ((CheckClearAttributesAttributeImpl) target).clear(); + } + } + public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException { assertNotNull(output); + CheckClearAttributesAttribute checkClearAtt = ts.addAttribute(CheckClearAttributesAttribute.class); + assertTrue("has TermAttribute", ts.hasAttribute(TermAttribute.class)); TermAttribute termAtt = ts.getAttribute(TermAttribute.class); @@ -62,15 +105,18 @@ } ts.reset(); + checkClearAtt.getAndResetClearCalled(); // reset it for (int i = 0; i < output.length; i++) { // extra safety to enforce, that the state is not preserved and also assign bogus values - ts.clearAttributes(); termAtt.setTermBuffer("bogusTerm"); if (offsetAtt != null) offsetAtt.setOffset(14584724,24683243); if (typeAtt != null) typeAtt.setType("bogusType"); if (posIncrAtt != null) posIncrAtt.setPositionIncrement(45987657); assertTrue("token "+i+" exists", ts.incrementToken()); + + assertTrue("clearAttributes() was not called correctly in TokenStream chain", checkClearAtt.getAndResetClearCalled()); + assertEquals("term "+i, output[i], termAtt.term()); if (startOffsets != null) assertEquals("startOffset "+i, startOffsets[i], offsetAtt.startOffset()); Index: contrib/wikipedia/src/test/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerTest.java =================================================================== --- contrib/wikipedia/src/test/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerTest.java (revision 899467) +++ contrib/wikipedia/src/test/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerTest.java (working copy) @@ -18,8 +18,6 @@ package org.apache.lucene.wikipedia.analysis; -import junit.framework.TestCase; - import java.io.StringReader; import java.io.IOException; import java.util.HashMap; @@ -46,6 +44,12 @@ super(s); } + public void testSimple() throws Exception { + WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader("This is a [[Category:foo]]")); + assertTokenStreamContents(tf, + new String[] { "This", "is", "a", "foo" }); + } + public void testHandwritten() throws Exception { //make sure all tokens are in only one type String test = "[[link]] This is a [[Category:foo]] Category This is a linked [[:Category:bar none withstanding]] " + Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java (revision 899467) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java (working copy) @@ -85,22 +85,12 @@ ts = new ShingleMatrixFilter(tls, 1, 2, new Character(' '), false, new ShingleMatrixFilter.OneDimensionalNonWeightedTokenSettingsCodec()); - - assertNext(ts, "please", 0, 6); - assertNext(ts, "please divide", 0, 13); - assertNext(ts, "divide", 7, 13); - assertNext(ts, "divide this", 7, 18); - assertNext(ts, "this", 14, 18); - assertNext(ts, "this sentence", 14, 27); - assertNext(ts, "sentence", 19, 27); - assertNext(ts, "sentence into", 19, 32); - assertNext(ts, "into", 28, 32); - assertNext(ts, "into shingles", 28, 39); - assertNext(ts, "shingles", 33, 39); - - - assertFalse(ts.incrementToken()); - + assertTokenStreamContents(ts, + new String[] { "please", "please divide", "divide", "divide this", + "this", "this sentence", "sentence", "sentence into", "into", + "into shingles", "shingles" }, + new int[] { 0, 0, 7, 7, 14, 14, 19, 19, 28, 28, 33 }, + new int[] { 6, 13, 13, 18, 18, 27, 27, 32, 32, 39, 39 }); } /** Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java (revision 899467) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java (working copy) @@ -19,10 +19,7 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.WhitespaceTokenizer; -import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import java.io.IOException; import java.io.StringReader; @@ -36,33 +33,22 @@ ts = new PrefixAwareTokenFilter( new SingleTokenTokenStream(createToken("a", 0, 1)), new SingleTokenTokenStream(createToken("b", 0, 1))); - assertNext(ts, "a", 0, 1); - assertNext(ts, "b", 1, 2); - assertFalse(ts.incrementToken()); + assertTokenStreamContents(ts, + new String[] { "a", "b" }, + new int[] { 0, 1 }, + new int[] { 1, 2 }); // prefix and suffix using 2x prefix ts = new PrefixAwareTokenFilter(new SingleTokenTokenStream(createToken("^", 0, 0)), new WhitespaceTokenizer(new StringReader("hello world"))); ts = new PrefixAwareTokenFilter(ts, new SingleTokenTokenStream(createToken("$", 0, 0))); - assertNext(ts, "^", 0, 0); - assertNext(ts, "hello", 0, 5); - assertNext(ts, "world", 6, 11); - assertNext(ts, "$", 11, 11); - assertFalse(ts.incrementToken()); + assertTokenStreamContents(ts, + new String[] { "^", "hello", "world", "$" }, + new int[] { 0, 0, 6, 11 }, + new int[] { 0, 5, 11, 11 }); } - - private void assertNext(TokenStream ts, String text, int startOffset, int endOffset) throws IOException { - TermAttribute termAtt = ts.addAttribute(TermAttribute.class); - OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class); - - assertTrue(ts.incrementToken()); - assertEquals(text, termAtt.term()); - assertEquals(startOffset, offsetAtt.startOffset()); - assertEquals(endOffset, offsetAtt.endOffset()); - } - private static Token createToken(String term, int start, int offset) { Token token = new Token(start, offset); Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java =================================================================== --- contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java (revision 899467) +++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java (working copy) @@ -19,10 +19,7 @@ import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.Token; -import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.WhitespaceTokenizer; -import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; -import org.apache.lucene.analysis.tokenattributes.TermAttribute; import java.io.IOException; import java.io.StringReader; @@ -36,24 +33,12 @@ new WhitespaceTokenizer(new StringReader("hello world")), new SingleTokenTokenStream(createToken("$", 0, 0))); - assertNext(ts, "^", 0, 0); - assertNext(ts, "hello", 0, 5); - assertNext(ts, "world", 6, 11); - assertNext(ts, "$", 11, 11); - assertFalse(ts.incrementToken()); + assertTokenStreamContents(ts, + new String[] { "^", "hello", "world", "$" }, + new int[] { 0, 0, 6, 11 }, + new int[] { 0, 5, 11, 11 }); } - - private void assertNext(TokenStream ts, String text, int startOffset, int endOffset) throws IOException { - TermAttribute termAtt = ts.addAttribute(TermAttribute.class); - OffsetAttribute offsetAtt = ts.addAttribute(OffsetAttribute.class); - - assertTrue(ts.incrementToken()); - assertEquals(text, termAtt.term()); - assertEquals(startOffset, offsetAtt.startOffset()); - assertEquals(endOffset, offsetAtt.endOffset()); - } - private static Token createToken(String term, int start, int offset) { Token token = new Token(start, offset); Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java (revision 899467) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java (working copy) @@ -188,6 +188,7 @@ } private final void setToken(final Token token) throws IOException { + clearAttributes(); termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength()); flagsAtt.setFlags(token.getFlags()); typeAtt.setType(token.type()); Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java (revision 899467) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java (working copy) @@ -377,6 +377,7 @@ } while (token == request_next_token); if (token == null) return false; + clearAttributes(); termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength()); posIncrAtt.setPositionIncrement(token.getPositionIncrement()); flagsAtt.setFlags(token.getFlags()); Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java (revision 899467) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java (working copy) @@ -114,6 +114,7 @@ private void setCurrentToken(Token token) { if (token == null) return; + clearAttributes(); termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength()); posIncrAtt.setPositionIncrement(token.getPositionIncrement()); flagsAtt.setFlags(token.getFlags()); Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java (revision 899467) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java (working copy) @@ -134,6 +134,7 @@ // grab gramSize chars from front or back int start = side == Side.FRONT ? 0 : curTermLength - curGramSize; int end = start + curGramSize; + clearAttributes(); offsetAtt.setOffset(start, end); termAtt.setTermBuffer(curTermBuffer, start, curGramSize); curGramSize++; Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java =================================================================== --- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (revision 899467) +++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java (working copy) @@ -87,6 +87,7 @@ } while (curGramSize <= maxGram) { while (curPos+curGramSize <= curTermLength) { // while there is input + clearAttributes(); termAtt.setTermBuffer(curTermBuffer, curPos, curGramSize); offsetAtt.setOffset(curPos, curPos+curGramSize); curPos++;