Property changes on: .
___________________________________________________________________
Modified: svn:mergeinfo
   Merged /lucene/java/branches/lucene_3_0:r899639
   Merged /lucene/java/trunk:r821888,899627


Property changes on: build.xml
___________________________________________________________________
Added: svn:mergeinfo
   Merged /lucene/java/trunk/build.xml:r821888,899627

Index: CHANGES.txt
===================================================================
--- CHANGES.txt	(revision 899639)
+++ CHANGES.txt	(working copy)
@@ -37,6 +37,11 @@
    is necessary with per-segment searching to notify the subclass
    which reader the int doc, passed to customScore, refers to.  (Paul
    chez Jamespot via Mike McCandless)
+  
+ * LUCENE-2211: Fix missing clearAttributes() calls in contrib:
+   ShingleMatrix, PrefixAware, compounds, NGramTokenFilter,
+   EdgeNGramTokenFilter, Highlighter, and MemoryIndex.
+   (Uwe Schindler, Robert Muir)
 
 Optimizations
 
@@ -54,7 +59,12 @@
    provided reader is per-segment (Simon Willnauer via Mike
    McCandless)
 
+Test Cases
 
+ * LUCENE-2211: Improves BaseTokenStreamTestCase to use a fake attribute
+   that checks if clearAttributes() was called correctly.
+   (Uwe Schindler, Robert Muir)
+
 ======================= Release 2.9.1 2009-11-06 =======================
 
 Changes in backwards compatibility policy
@@ -3909,4 +3919,4 @@
 
 The code has been re-organized into a new package and directory
 structure for this release.  It builds OK, but has not been tested
-beyond that since the re-organization.
+beyond that since the re-organization.
\ No newline at end of file

Property changes on: CHANGES.txt
___________________________________________________________________
Modified: svn:mergeinfo
   Merged /lucene/java/branches/lucene_3_0/CHANGES.txt:r899639
   Merged /lucene/java/trunk/CHANGES.txt:r821888,899627


Property changes on: contrib
___________________________________________________________________
Modified: svn:mergeinfo
   Merged /lucene/java/branches/lucene_3_0/contrib:r899639
   Merged /lucene/java/trunk/contrib:r821888,899627

Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java
===================================================================
--- contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java	(revision 899639)
+++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/compound/CompoundWordTokenFilterBase.java	(working copy)
@@ -129,6 +129,7 @@
   }
   
   private final void setToken(final Token token) throws IOException {
+    clearAttributes();
     termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
     flagsAtt.setFlags(token.getFlags());
     typeAtt.setType(token.type());
Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java
===================================================================
--- contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java	(revision 899639)
+++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PrefixAwareTokenFilter.java	(working copy)
@@ -125,6 +125,7 @@
   
   private void setCurrentToken(Token token) {
     if (token == null) return;
+    clearAttributes();
     termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
     posIncrAtt.setPositionIncrement(token.getPositionIncrement());
     flagsAtt.setFlags(token.getFlags());
Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java
===================================================================
--- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java	(revision 899639)
+++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java	(working copy)
@@ -140,6 +140,7 @@
           // grab gramSize chars from front or back
           int start = side == Side.FRONT ? 0 : curTermLength - curGramSize;
           int end = start + curGramSize;
+          clearAttributes();
           offsetAtt.setOffset(start, end);
           termAtt.setTermBuffer(curTermBuffer, start, curGramSize);
           curGramSize++;
Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java
===================================================================
--- contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java	(revision 899639)
+++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java	(working copy)
@@ -86,6 +86,7 @@
       }
       while (curGramSize <= maxGram) {
         while (curPos+curGramSize <= curTermLength) {     // while there is input
+          clearAttributes();
           termAtt.setTermBuffer(curTermBuffer, curPos, curGramSize);
           offsetAtt.setOffset(curPos, curPos+curGramSize);
           curPos++;
Index: contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java
===================================================================
--- contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java	(revision 899639)
+++ contrib/analyzers/common/src/java/org/apache/lucene/analysis/shingle/ShingleMatrixFilter.java	(working copy)
@@ -373,6 +373,7 @@
     } while (token == request_next_token);
     if (token == null) return false;
     
+    clearAttributes();
     termAtt.setTermBuffer(token.termBuffer(), 0, token.termLength());
     posIncrAtt.setPositionIncrement(token.getPositionIncrement());
     flagsAtt.setFlags(token.getFlags());
@@ -429,7 +430,7 @@
         if (ignoringSinglePrefixOrSuffixShingle
             && currentShingleLength == 1
             && (((Matrix.Column.Row) currentPermutationRows.get(currentPermutationTokensStartOffset)).getColumn().isFirst() || ((Matrix.Column.Row) currentPermutationRows.get(currentPermutationTokensStartOffset)).getColumn().isLast())) {
-          return next(reusableToken);
+          return next();
         }
 
         int termLength = 0;
@@ -838,7 +839,8 @@
 
         public boolean hasNext() {
           int s = columnRowCounters.length;
-          return s != 0 && columnRowCounters[s - 1] < ((Column) columns.get(s - 1)).getRows().size();
+          int n = columns.size();
+          return s != 0 && n >= s && columnRowCounters[s - 1] < ((Column) columns.get(s - 1)).getRows().size();
         }
 
         public Object next() {
Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java
===================================================================
--- contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java	(revision 899639)
+++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAndSuffixAwareTokenFilter.java	(working copy)
@@ -19,10 +19,7 @@
 
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -36,24 +33,12 @@
         new WhitespaceTokenizer(new StringReader("hello world")),
         new SingleTokenTokenStream(createToken("$", 0, 0)));
 
-    assertNext(ts, "^", 0, 0);
-    assertNext(ts, "hello", 0, 5);
-    assertNext(ts, "world", 6, 11);
-    assertNext(ts, "$", 11, 11);
-    assertFalse(ts.incrementToken());
+    assertTokenStreamContents(ts,
+        new String[] { "^", "hello", "world", "$" },
+        new int[] { 0, 0, 6, 11 },
+        new int[] { 0, 5, 11, 11 });
   }
 
-
-  private void assertNext(TokenStream ts, String text, int startOffset, int endOffset) throws IOException {
-    TermAttribute termAtt = (TermAttribute) ts.addAttribute(TermAttribute.class);
-    OffsetAttribute offsetAtt = (OffsetAttribute) ts.addAttribute(OffsetAttribute.class);
-
-    assertTrue(ts.incrementToken());
-    assertEquals(text, termAtt.term());
-    assertEquals(startOffset, offsetAtt.startOffset());
-    assertEquals(endOffset, offsetAtt.endOffset());
-  }
-
   private static Token createToken(String term, int start, int offset)
   {
     Token token = new Token(start, offset);
Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java
===================================================================
--- contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java	(revision 899639)
+++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPrefixAwareTokenFilter.java	(working copy)
@@ -19,10 +19,7 @@
 
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WhitespaceTokenizer;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
 
 import java.io.IOException;
 import java.io.StringReader;
@@ -36,33 +33,22 @@
     ts = new PrefixAwareTokenFilter(
         new SingleTokenTokenStream(createToken("a", 0, 1)),
         new SingleTokenTokenStream(createToken("b", 0, 1)));
-    assertNext(ts, "a", 0, 1);
-    assertNext(ts, "b", 1, 2);
-    assertFalse(ts.incrementToken());
+    assertTokenStreamContents(ts, 
+        new String[] { "a", "b" },
+        new int[] { 0, 1 },
+        new int[] { 1, 2 });
 
     // prefix and suffix using 2x prefix
 
     ts = new PrefixAwareTokenFilter(new SingleTokenTokenStream(createToken("^", 0, 0)), new WhitespaceTokenizer(new StringReader("hello world")));
     ts = new PrefixAwareTokenFilter(ts, new SingleTokenTokenStream(createToken("$", 0, 0)));
 
-    assertNext(ts, "^", 0, 0);
-    assertNext(ts, "hello", 0, 5);
-    assertNext(ts, "world", 6, 11);
-    assertNext(ts, "$", 11, 11);
-    assertFalse(ts.incrementToken());
+    assertTokenStreamContents(ts,
+        new String[] { "^", "hello", "world", "$" },
+        new int[] { 0, 0, 6, 11 },
+        new int[] { 0, 5, 11, 11 });
   }
 
-
-  private void assertNext(TokenStream ts, String text, int startOffset, int endOffset) throws IOException {
-    TermAttribute termAtt = (TermAttribute) ts.addAttribute(TermAttribute.class);
-    OffsetAttribute offsetAtt = (OffsetAttribute) ts.addAttribute(OffsetAttribute.class);
-
-    assertTrue(ts.incrementToken());
-    assertEquals(text, termAtt.term());
-    assertEquals(startOffset, offsetAtt.startOffset());
-    assertEquals(endOffset, offsetAtt.endOffset());
-  }
-
   private static Token createToken(String term, int start, int offset)
   {
     Token token = new Token(start, offset);
Index: contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java
===================================================================
--- contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java	(revision 899639)
+++ contrib/analyzers/common/src/test/org/apache/lucene/analysis/shingle/TestShingleMatrixFilter.java	(working copy)
@@ -18,16 +18,14 @@
  */
 
 import java.io.IOException;
+import java.io.StringReader;
 import java.util.Collection;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.HashSet;
 import java.util.Arrays;
 
-import org.apache.lucene.analysis.BaseTokenStreamTestCase;
-import org.apache.lucene.analysis.CachingTokenFilter;
-import org.apache.lucene.analysis.Token;
-import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.*;
 import org.apache.lucene.analysis.miscellaneous.EmptyTokenStream;
 import org.apache.lucene.analysis.miscellaneous.PrefixAndSuffixAwareTokenFilter;
 import org.apache.lucene.analysis.miscellaneous.SingleTokenTokenStream;
@@ -41,10 +39,26 @@
   public TestShingleMatrixFilter(String name) {
     // use this ctor, because SingleTokenTokenStream only uses next(Token), so exclude it
     super(name, new HashSet(Arrays.asList(new String[]{
-      "testBehavingAsShingleFilter", "testMatrix"
+      "testBehavingAsShingleFilter", "testMatrix", "testIterator"
     })));
   }
 
+  public void testIterator() throws IOException {
+
+    WhitespaceTokenizer wst = new WhitespaceTokenizer(new StringReader("one two three four five"));
+    ShingleMatrixFilter smf = new ShingleMatrixFilter(wst, 2, 2, new Character('_'), false, new ShingleMatrixFilter.OneDimensionalNonWeightedTokenSettingsCodec());
+
+    int i;
+    for(i=0; smf.incrementToken(); i++);
+    assertEquals(4, i);
+
+    // call next once more. this should return false again rather than throwing an exception (LUCENE-1939)
+    assertFalse(smf.incrementToken());
+
+    System.currentTimeMillis();
+
+  }
+
   public void testBehavingAsShingleFilter() throws IOException {
 
     ShingleMatrixFilter.defaultSettingsCodec = null;
@@ -73,22 +87,12 @@
 
     ts = new ShingleMatrixFilter(tls, 1, 2, new Character(' '), false, new ShingleMatrixFilter.OneDimensionalNonWeightedTokenSettingsCodec());
 
-
-    assertNext(ts, "please", 0, 6);
-    assertNext(ts, "please divide", 0, 13);
-    assertNext(ts, "divide", 7, 13);
-    assertNext(ts, "divide this", 7, 18);
-    assertNext(ts, "this", 14, 18);
-    assertNext(ts, "this sentence", 14, 27);
-    assertNext(ts, "sentence", 19, 27);
-    assertNext(ts, "sentence into", 19, 32);
-    assertNext(ts, "into", 28, 32);
-    assertNext(ts, "into shingles", 28, 39);
-    assertNext(ts, "shingles", 33, 39);
-
-
-    assertFalse(ts.incrementToken());
-
+    assertTokenStreamContents(ts,
+      new String[] { "please", "please divide", "divide", "divide this",
+        "this", "this sentence", "sentence", "sentence into", "into",
+        "into shingles", "shingles" },
+      new int[] { 0, 0, 7, 7, 14, 14, 19, 19, 28, 28, 33 },
+      new int[] { 6, 13, 13, 18, 18, 27, 27, 32, 32, 39, 39 });
   }
 
   /**
@@ -96,7 +100,6 @@
    * @throws IOException
    */
   public void testTokenStream() throws IOException {
-
     ShingleMatrixFilter.defaultSettingsCodec = null;//new ShingleMatrixFilter.SimpleThreeDimensionalTokenSettingsCodec();
 
     TokenStream ts;
@@ -464,8 +467,8 @@
     TermAttribute termAtt = (TermAttribute) ts.addAttribute(TermAttribute.class);
     PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) ts.addAttribute(PositionIncrementAttribute.class);
     PayloadAttribute payloadAtt = (PayloadAttribute) ts.addAttribute(PayloadAttribute.class);
-    
-    assertTrue(ts.incrementToken());    
+
+    assertTrue(ts.incrementToken());
     assertEquals(text, termAtt.term());
     assertEquals(positionIncrement, posIncrAtt.getPositionIncrement());
     assertEquals(boost, payloadAtt.getPayload() == null ? 1f : PayloadHelper.decodeFloat(payloadAtt.getPayload().getData()), 0);
@@ -533,6 +536,7 @@
         return false;
       }
       Token prototype = (Token) iterator.next();
+      clearAttributes();
       termAtt.setTermBuffer(prototype.termBuffer(), 0, prototype.termLength());
       posIncrAtt.setPositionIncrement(prototype.getPositionIncrement());
       flagsAtt.setFlags(prototype.getFlags());
Index: contrib/CHANGES.txt
===================================================================
--- contrib/CHANGES.txt	(revision 899639)
+++ contrib/CHANGES.txt	(working copy)
@@ -20,6 +20,10 @@
  * LUCENE-2199: ShingleFilter skipped over tri-gram shingles if outputUnigram
    was set to false. (Simon Willnauer)
 
+ * LUCENE-1939: IndexOutOfBoundsException at ShingleMatrixFilter's
+   Iterator#hasNext method on exhausted streams.
+   (Patrick Jungermann via Karl Wettin)
+
 ======================= Release 2.9.1 2009-11-06 =======================
 
 Changes in backwards compatibility policy

Property changes on: contrib\CHANGES.txt
___________________________________________________________________
Added: svn:mergeinfo
   Merged /lucene/java/trunk/contrib/CHANGES.txt:r821888,899627

Index: contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
===================================================================
--- contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java	(revision 899639)
+++ contrib/fast-vector-highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java	(working copy)
@@ -200,7 +200,7 @@
     public boolean incrementToken() throws IOException {
       if( !getNextPartialSnippet() )
         return false;
-      
+      clearAttributes();
       termAtt.setTermBuffer(snippet, startTerm, lenTerm);
       offsetAtt.setOffset(correctOffset(startOffset), correctOffset(startOffset + lenTerm));
       return true;
Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
===================================================================
--- contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java	(revision 899639)
+++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java	(working copy)
@@ -155,6 +155,7 @@
             if (currentToken >= tokens.length) {
               return false;
             }
+            clearAttributes();
             Token token = tokens[currentToken++];
             termAtt.setTermBuffer(token.term());
             offsetAtt.setOffset(token.startOffset(), token.endOffset());

Property changes on: contrib\highlighter\src\test
___________________________________________________________________
Added: svn:mergeinfo
   Merged /lucene/java/trunk/contrib/highlighter/src/test:r821888,899627

Index: contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
===================================================================
--- contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java	(revision 899639)
+++ contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java	(working copy)
@@ -1433,6 +1433,7 @@
       public boolean incrementToken() throws IOException {
         if(iter.hasNext()) {
           Token token = (Token) iter.next();
+          clearAttributes();
           termAtt.setTermBuffer(token.term());
           posIncrAtt.setPositionIncrement(token.getPositionIncrement());
           offsetAtt.setOffset(token.startOffset(), token.endOffset());
@@ -1480,6 +1481,7 @@
       public boolean incrementToken() throws IOException {
         if(iter.hasNext()) {
           Token token = (Token) iter.next();
+          clearAttributes();
           termAtt.setTermBuffer(token.term());
           posIncrAtt.setPositionIncrement(token.getPositionIncrement());
           offsetAtt.setOffset(token.startOffset(), token.endOffset());
@@ -1813,6 +1815,7 @@
         return false;
       }
       //Token nextRealToken = new Token(, offsetAtt.startOffset(), offsetAtt.endOffset());
+      clearAttributes();
       termAtt.setTermBuffer(realTermAtt.term());
       offsetAtt.setOffset(realOffsetAtt.startOffset(), realOffsetAtt.endOffset());
       posIncrAtt.setPositionIncrement(realPosIncrAtt.getPositionIncrement());
@@ -1830,6 +1833,7 @@
       return true;
     } else {
       String tok = st.nextToken();
+      clearAttributes();
       termAtt.setTermBuffer(tok);
       offsetAtt.setOffset(currentRealToken.startOffset(), currentRealToken.endOffset());
       posIncrAtt.setPositionIncrement(0);

Property changes on: contrib\instantiated\src\test\org\apache\lucene\store\instantiated\TestIndicesEquals.java
___________________________________________________________________
Added: svn:mergeinfo
   Merged /lucene/java/trunk/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java:r821888,899627

Index: contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
===================================================================
--- contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java	(revision 899639)
+++ contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java	(working copy)
@@ -288,6 +288,7 @@
           throw new IllegalArgumentException("keyword must not be null");
         
         String term = obj.toString();
+        clearAttributes();
         termAtt.setTermBuffer(term);
         offsetAtt.setOffset(start, start+termAtt.termLength());
         start += term.length() + 1; // separate words by 1 (blank) character
Index: contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java
===================================================================
--- contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java	(revision 899639)
+++ contrib/misc/src/test/org/apache/lucene/queryParser/precedence/TestPrecedenceQueryParser.java	(working copy)
@@ -71,6 +71,7 @@
     OffsetAttribute offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
     
     public boolean incrementToken() throws IOException {
+      clearAttributes();
       if (inPhrase) {
         inPhrase = false;
         termAtt.setTermBuffer("phrase2");
Index: contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java
===================================================================
--- contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java	(revision 899639)
+++ contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQPHelper.java	(working copy)
@@ -143,6 +143,7 @@
     public boolean incrementToken() throws IOException {
       if (inPhrase) {
         inPhrase = false;
+        clearAttributes();
         termAtt.setTermBuffer("phrase2");
         offsetAtt.setOffset(savedStart, savedEnd);
         return true;
@@ -1186,6 +1187,7 @@
   private class CannedTokenStream extends TokenStream {
     private int upto = 0;
     public boolean incrementToken() {
+      clearAttributes();
       if (upto == 4) {
         return false;
       }
Index: contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java
===================================================================
--- contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java	(revision 899639)
+++ contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/TestQueryParserWrapper.java	(working copy)
@@ -138,6 +138,7 @@
     public boolean incrementToken() throws IOException {
       if (inPhrase) {
         inPhrase = false;
+        clearAttributes();
         termAtt.setTermBuffer("phrase2");
         offsetAtt.setOffset(savedStart, savedEnd);
         return true;
Index: contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java
===================================================================
--- contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java	(revision 899639)
+++ contrib/snowball/src/test/org/apache/lucene/analysis/snowball/TestSnowball.java	(working copy)
@@ -106,6 +106,7 @@
     }
     
     public boolean incrementToken() {
+      clearAttributes();
       termAtt.setTermBuffer("accents");
       offsetAtt.setOffset(2, 7);
       typeAtt.setType("wrd");
Index: contrib/wikipedia/src/test/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerTest.java
===================================================================
--- contrib/wikipedia/src/test/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerTest.java	(revision 899639)
+++ contrib/wikipedia/src/test/org/apache/lucene/wikipedia/analysis/WikipediaTokenizerTest.java	(working copy)
@@ -18,8 +18,6 @@
 
 package org.apache.lucene.wikipedia.analysis;
 
-import junit.framework.TestCase;
-
 import java.io.StringReader;
 import java.io.IOException;
 import java.util.HashMap;
@@ -46,6 +44,12 @@
     super(s);
   }
 
+  public void testSimple() throws Exception {
+    WikipediaTokenizer tf = new WikipediaTokenizer(new StringReader("This is a [[Category:foo]]"));
+    assertTokenStreamContents(tf,
+        new String[] { "This", "is", "a", "foo" });
+  }
+  
   public void testHandwritten() throws Exception {
     //make sure all tokens are in only one type
     String test = "[[link]] This is a [[Category:foo]] Category  This is a linked [[:Category:bar none withstanding]] " +

Property changes on: src\java\org\apache\lucene\analysis\Tokenizer.java
___________________________________________________________________
Added: svn:mergeinfo
   Merged /lucene/java/trunk/src/java/org/apache/lucene/analysis/Tokenizer.java:r821888,899627


Property changes on: src\java\org\apache\lucene\search\MultiTermQueryWrapperFilter.java
___________________________________________________________________
Modified: svn:mergeinfo
   Merged /lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:r899639
   Merged /lucene/java/trunk/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java:r821888,899627


Property changes on: src\java\org\apache\lucene\util\AttributeSource.java
___________________________________________________________________
Modified: svn:mergeinfo
   Merged /lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/util/AttributeSource.java:r899639
   Merged /lucene/java/trunk/src/java/org/apache/lucene/util/AttributeSource.java:r821888

Index: src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
===================================================================
--- src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java	(revision 899639)
+++ src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java	(working copy)
@@ -22,6 +22,8 @@
 import java.io.IOException;
  
 import org.apache.lucene.analysis.tokenattributes.*;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeImpl;
 import org.apache.lucene.util.LuceneTestCase;
 
 /** 
@@ -86,26 +88,67 @@
   
   // some helpers to test Analyzers and TokenStreams:
   
+  public static interface CheckClearAttributesAttribute extends Attribute {
+    boolean getAndResetClearCalled();
+  }
+
+  public static final class CheckClearAttributesAttributeImpl extends AttributeImpl implements CheckClearAttributesAttribute {
+    private boolean clearCalled = false;
+    
+    public boolean getAndResetClearCalled() {
+      try {
+        return clearCalled;
+      } finally {
+        clearCalled = false;
+      }
+    }
+
+    //@Override
+    public void clear() {
+      clearCalled = true;
+    }
+
+    //@Override
+    public boolean equals(Object other) {
+      return (
+        other instanceof CheckClearAttributesAttributeImpl &&
+        ((CheckClearAttributesAttributeImpl) other).clearCalled == this.clearCalled
+      );
+    }
+
+    //@Override
+    public int hashCode() {
+      return 76137213 ^ Boolean.valueOf(clearCalled).hashCode();
+    }
+    
+    //@Override
+    public void copyTo(AttributeImpl target) {
+      ((CheckClearAttributesAttributeImpl) target).clear();
+    }
+  }
+
   public static void assertTokenStreamContents(TokenStream ts, String[] output, int startOffsets[], int endOffsets[], String types[], int posIncrements[]) throws IOException {
     assertNotNull(output);
-    assertTrue("has TermAttribute", ts.hasAttribute(TermAttribute.class));
+    CheckClearAttributesAttribute checkClearAtt = (CheckClearAttributesAttribute) ts.addAttribute(CheckClearAttributesAttribute.class);
+    
+    assertTrue("has no TermAttribute", ts.hasAttribute(TermAttribute.class));
     TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class);
     
     OffsetAttribute offsetAtt = null;
     if (startOffsets != null || endOffsets != null) {
-      assertTrue("has OffsetAttribute", ts.hasAttribute(OffsetAttribute.class));
+      assertTrue("has no OffsetAttribute", ts.hasAttribute(OffsetAttribute.class));
       offsetAtt = (OffsetAttribute) ts.getAttribute(OffsetAttribute.class);
     }
     
     TypeAttribute typeAtt = null;
     if (types != null) {
-      assertTrue("has TypeAttribute", ts.hasAttribute(TypeAttribute.class));
+      assertTrue("has no TypeAttribute", ts.hasAttribute(TypeAttribute.class));
       typeAtt = (TypeAttribute) ts.getAttribute(TypeAttribute.class);
     }
     
     PositionIncrementAttribute posIncrAtt = null;
     if (posIncrements != null) {
-      assertTrue("has PositionIncrementAttribute", ts.hasAttribute(PositionIncrementAttribute.class));
+      assertTrue("has no PositionIncrementAttribute", ts.hasAttribute(PositionIncrementAttribute.class));
       posIncrAtt = (PositionIncrementAttribute) ts.getAttribute(PositionIncrementAttribute.class);
     }
     
@@ -118,7 +161,10 @@
       if (typeAtt != null) typeAtt.setType("bogusType");
       if (posIncrAtt != null) posIncrAtt.setPositionIncrement(45987657);
       
-      assertTrue("token "+i+" exists", ts.incrementToken());
+      checkClearAtt.getAndResetClearCalled(); // reset it, because we called clearAttribute() before
+      assertTrue("token "+i+" does not exist", ts.incrementToken());
+      assertTrue("clearAttributes() was not called correctly in TokenStream chain", checkClearAtt.getAndResetClearCalled());
+      
       assertEquals("term "+i, output[i], termAtt.term());
       if (startOffsets != null)
         assertEquals("startOffset "+i, startOffsets[i], offsetAtt.startOffset());

Property changes on: src\test\org\apache\lucene\analysis\BaseTokenStreamTestCase.java
___________________________________________________________________
Modified: svn:mergeinfo
   Merged /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java:r899639
   Merged /lucene/java/trunk/src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java:r821888

Index: src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
===================================================================
--- src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java	(revision 899639)
+++ src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java	(working copy)
@@ -48,6 +48,7 @@
         if (index == tokens.length) {
           return false;
         } else {
+          clearAttributes();
           termAtt.setTermBuffer(tokens[index++]);
           offsetAtt.setOffset(0,0);
           return true;

Property changes on: src\test\org\apache\lucene\analysis\TestISOLatin1AccentFilter.java
___________________________________________________________________
Modified: svn:mergeinfo
   Merged /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:r899639
   Merged /lucene/java/trunk/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java:r821888,899627

Index: src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java
===================================================================
--- src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java	(revision 899639)
+++ src/test/org/apache/lucene/analysis/TestTeeSinkTokenFilter.java	(working copy)
@@ -76,29 +76,14 @@
     final TeeSinkTokenFilter source = new TeeSinkTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString())));
     final TokenStream sink1 = source.newSinkTokenStream();
     final TokenStream sink2 = source.newSinkTokenStream(theFilter);
-    int i = 0;
-    TermAttribute termAtt = (TermAttribute) source.getAttribute(TermAttribute.class);
-    while (source.incrementToken()) {
-      assertEquals(tokens1[i], termAtt.term());
-      i++;
-    }
-    assertEquals(tokens1.length, i);
     
-    i = 0;
-    termAtt = (TermAttribute) sink1.getAttribute(TermAttribute.class);
-    while (sink1.incrementToken()) {
-      assertEquals(tokens1[i], termAtt.term());
-      i++;
-    }
-    assertEquals(tokens1.length, i);
+    source.addAttribute(CheckClearAttributesAttribute.class);
+    sink1.addAttribute(CheckClearAttributesAttribute.class);
+    sink2.addAttribute(CheckClearAttributesAttribute.class);
     
-    i = 0;
-    termAtt = (TermAttribute) sink2.getAttribute(TermAttribute.class);
-    while (sink2.incrementToken()) {
-      assertTrue(termAtt.term().equalsIgnoreCase("The"));
-      i++;
-    }
-    assertEquals("there should be two times 'the' in the stream", 2, i);
+    assertTokenStreamContents(source, tokens1);
+    assertTokenStreamContents(sink1, tokens1);
+    assertTokenStreamContents(sink2, new String[]{"The", "the"});
   }
 
   public void testMultipleSources() throws Exception {
@@ -106,50 +91,28 @@
     final TeeSinkTokenFilter.SinkTokenStream dogDetector = tee1.newSinkTokenStream(dogFilter);
     final TeeSinkTokenFilter.SinkTokenStream theDetector = tee1.newSinkTokenStream(theFilter);
     final TokenStream source1 = new CachingTokenFilter(tee1);
+    
+    tee1.addAttribute(CheckClearAttributesAttribute.class);
+    dogDetector.addAttribute(CheckClearAttributesAttribute.class);
+    theDetector.addAttribute(CheckClearAttributesAttribute.class);
 
     final TeeSinkTokenFilter tee2 = new TeeSinkTokenFilter(new WhitespaceTokenizer(new StringReader(buffer2.toString())));
     tee2.addSinkTokenStream(dogDetector);
     tee2.addSinkTokenStream(theDetector);
     final TokenStream source2 = tee2;
 
-    int i = 0;
-    TermAttribute termAtt = (TermAttribute) source1.getAttribute(TermAttribute.class);
-    while (source1.incrementToken()) {
-      assertEquals(tokens1[i], termAtt.term());
-      i++;
-    }
-    assertEquals(tokens1.length, i);
-    i = 0;
-    termAtt = (TermAttribute) source2.getAttribute(TermAttribute.class);
-    while (source2.incrementToken()) {
-      assertEquals(tokens2[i], termAtt.term());
-      i++;
-    }
-    assertEquals(tokens2.length, i);
-    i = 0;
-    termAtt = (TermAttribute) theDetector.getAttribute(TermAttribute.class);
-    while (theDetector.incrementToken()) {
-      assertTrue("'" + termAtt.term() + "' is not equal to 'The'", termAtt.term().equalsIgnoreCase("The"));
-      i++;
-    }
-    assertEquals("there must be 4 times 'The' in the stream", 4, i);
-    i = 0;
-    termAtt = (TermAttribute) dogDetector.getAttribute(TermAttribute.class);
-    while (dogDetector.incrementToken()) {
-      assertTrue("'" + termAtt.term() + "' is not equal to 'Dogs'", termAtt.term().equalsIgnoreCase("Dogs"));
-      i++;
-    }
-    assertEquals("there must be 2 times 'Dog' in the stream", 2, i);
+    assertTokenStreamContents(source1, tokens1);
+    assertTokenStreamContents(source2, tokens2);
+
+    assertTokenStreamContents(theDetector, new String[]{"The", "the", "The", "the"});
+    assertTokenStreamContents(dogDetector, new String[]{"Dogs", "Dogs"});
     
     source1.reset();
     TokenStream lowerCasing = new LowerCaseFilter(source1);
-    i = 0;
-    termAtt = (TermAttribute) lowerCasing.getAttribute(TermAttribute.class);
-    while (lowerCasing.incrementToken()) {
-      assertEquals(tokens1[i].toLowerCase(), termAtt.term());
-      i++;
-    }
-    assertEquals(i, tokens1.length);
+    String[] lowerCaseTokens = new String[tokens1.length];
+    for (int i = 0; i < tokens1.length; i++)
+      lowerCaseTokens[i] = tokens1[i].toLowerCase();
+    assertTokenStreamContents(lowerCasing, lowerCaseTokens);
   }
 
   /**

Property changes on: src\test\org\apache\lucene\document\TestDateTools.java
___________________________________________________________________
Modified: svn:mergeinfo
   Merged /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestDateTools.java:r899639
   Merged /lucene/java/trunk/src/test/org/apache/lucene/document/TestDateTools.java:r821888,899627


Property changes on: src\test\org\apache\lucene\document\TestNumberTools.java
___________________________________________________________________
Modified: svn:mergeinfo
   Merged /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/document/TestNumberTools.java:r899639
   Merged /lucene/java/trunk/src/test/org/apache/lucene/document/TestNumberTools.java:r821888,899627


Property changes on: src\test\org\apache\lucene\index\TestBackwardsCompatibility.java
___________________________________________________________________
Modified: svn:mergeinfo
   Merged /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:r899639
   Merged /lucene/java/trunk/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java:r821888,899627

Index: src/test/org/apache/lucene/index/TestDocumentWriter.java
===================================================================
--- src/test/org/apache/lucene/index/TestDocumentWriter.java	(revision 899639)
+++ src/test/org/apache/lucene/index/TestDocumentWriter.java	(working copy)
@@ -215,6 +215,7 @@
         if (index == tokens.length) {
           return false;
         } else {
+          clearAttributes();
           termAtt.setTermBuffer(tokens[index++]);
           return true;
         }        
Index: src/test/org/apache/lucene/index/TestPayloads.java
===================================================================
--- src/test/org/apache/lucene/index/TestPayloads.java	(revision 899639)
+++ src/test/org/apache/lucene/index/TestPayloads.java	(working copy)
@@ -533,6 +533,7 @@
         public boolean incrementToken() throws IOException {
             if (!first) return false;
             first = false;
+            clearAttributes();
             termAtt.setTermBuffer(term);
             payloadAtt.setPayload(new Payload(payload));
             return true;
Index: src/test/org/apache/lucene/index/TestTermdocPerf.java
===================================================================
--- src/test/org/apache/lucene/index/TestTermdocPerf.java	(revision 899639)
+++ src/test/org/apache/lucene/index/TestTermdocPerf.java	(working copy)
@@ -47,6 +47,7 @@
    public boolean incrementToken() throws IOException {
      num--;
      if (num >= 0) {
+       clearAttributes();
        termAtt.setTermBuffer(value);
        return true;
      }
Index: src/test/org/apache/lucene/index/TestTermVectorsReader.java
===================================================================
--- src/test/org/apache/lucene/index/TestTermVectorsReader.java	(revision 899639)
+++ src/test/org/apache/lucene/index/TestTermVectorsReader.java	(working copy)
@@ -136,6 +136,7 @@
         return false;
       else {
         final TestToken testToken = tokens[tokenUpto++];
+        clearAttributes();
         termAtt.setTermBuffer(testToken.text);
         offsetAtt.setOffset(testToken.startOffset, testToken.endOffset);
         if (tokenUpto > 1) {
Index: src/test/org/apache/lucene/queryParser/TestQueryParser.java
===================================================================
--- src/test/org/apache/lucene/queryParser/TestQueryParser.java	(revision 899639)
+++ src/test/org/apache/lucene/queryParser/TestQueryParser.java	(working copy)
@@ -101,6 +101,7 @@
     public boolean incrementToken() throws IOException {
       if (inPhrase) {
         inPhrase = false;
+        clearAttributes();
         termAtt.setTermBuffer("phrase2");
         offsetAtt.setOffset(savedStart, savedEnd);
         return true;
Index: src/test/org/apache/lucene/search/TestPositionIncrement.java
===================================================================
--- src/test/org/apache/lucene/search/TestPositionIncrement.java	(revision 899639)
+++ src/test/org/apache/lucene/search/TestPositionIncrement.java	(working copy)
@@ -73,6 +73,7 @@
           public boolean incrementToken() {
             if (i == TOKENS.length)
               return false;
+            clearAttributes();
             termAtt.setTermBuffer(TOKENS[i]);
             offsetAtt.setOffset(i,i);
             posIncrAtt.setPositionIncrement(INCREMENTS[i]);
Index: src/test/org/apache/lucene/search/TestTermRangeQuery.java
===================================================================
--- src/test/org/apache/lucene/search/TestTermRangeQuery.java	(revision 899639)
+++ src/test/org/apache/lucene/search/TestTermRangeQuery.java	(working copy)
@@ -252,6 +252,7 @@
         if (done)
           return false;
         else {
+          clearAttributes();
           done = true;
           if (count == 1) {
             termAtt.termBuffer()[0] = buffer[0];

Property changes on: src\test\org\apache\lucene\util\TestAttributeSource.java
___________________________________________________________________
Modified: svn:mergeinfo
   Merged /lucene/java/branches/lucene_3_0/src/test/org/apache/lucene/util/TestAttributeSource.java:r899639
   Merged /lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java:r821888,899627