Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java =================================================================== --- contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java (revision 806538) +++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java (working copy) @@ -156,8 +156,8 @@ */ public TokenStream init(TokenStream tokenStream) throws IOException { position = -1; - termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class); - posIncAtt = (PositionIncrementAttribute) tokenStream.getAttribute(PositionIncrementAttribute.class); + termAtt = (TermAttribute) tokenStream.addAttribute(TermAttribute.class); + posIncAtt = (PositionIncrementAttribute) tokenStream.addAttribute(PositionIncrementAttribute.class); if(!skipInitExtractor) { if(fieldWeightedSpanTerms != null) { fieldWeightedSpanTerms.clear(); Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java =================================================================== --- contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java (revision 806538) +++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermScorer.java (working copy) @@ -95,7 +95,7 @@ * @see org.apache.lucene.search.highlight.Scorer#init(org.apache.lucene.analysis.TokenStream) */ public TokenStream init(TokenStream tokenStream) { - termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class); + termAtt = (TermAttribute) tokenStream.addAttribute(TermAttribute.class); return null; } Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java =================================================================== --- contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java (revision 806538) +++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleFragmenter.java (working copy) @@ -47,7 +47,7 @@ * @see org.apache.lucene.search.highlight.Fragmenter#start(java.lang.String, org.apache.lucene.analysis.TokenStream) */ public void start(String originalText, TokenStream stream) { - offsetAtt = (OffsetAttribute) stream.getAttribute(OffsetAttribute.class); + offsetAtt = (OffsetAttribute) stream.addAttribute(OffsetAttribute.class); currentNumFrags = 1; } Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java =================================================================== --- contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java (revision 806538) +++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/SimpleSpanFragmenter.java (working copy) @@ -101,8 +101,8 @@ position = -1; currentNumFrags = 1; textSize = originalText.length(); - termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class); - posIncAtt = (PositionIncrementAttribute) tokenStream.getAttribute(PositionIncrementAttribute.class); - offsetAtt = (OffsetAttribute) tokenStream.getAttribute(OffsetAttribute.class); + termAtt = (TermAttribute) tokenStream.addAttribute(TermAttribute.class); + posIncAtt = (PositionIncrementAttribute) tokenStream.addAttribute(PositionIncrementAttribute.class); + offsetAtt = (OffsetAttribute) tokenStream.addAttribute(OffsetAttribute.class); } } Index: contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java =================================================================== --- contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java (revision 806538) +++ contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenGroup.java (working copy) @@ -41,8 +41,8 @@ private TermAttribute termAtt; public TokenGroup(TokenStream tokenStream) { - offsetAtt = (OffsetAttribute) tokenStream.getAttribute(OffsetAttribute.class); - termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class); + offsetAtt = (OffsetAttribute) tokenStream.addAttribute(OffsetAttribute.class); + termAtt = (TermAttribute) tokenStream.addAttribute(TermAttribute.class); } void addToken(float score) { Index: contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java =================================================================== --- contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (revision 806538) +++ contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (working copy) @@ -33,8 +33,6 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; -import junit.framework.TestCase; - import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.LowerCaseTokenizer; import org.apache.lucene.analysis.SimpleAnalyzer; @@ -78,6 +76,7 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.util.Version; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.w3c.dom.Element; import org.w3c.dom.NodeList; @@ -85,7 +84,7 @@ * JUnit Test for Highlighter class. * */ -public class HighlighterTest extends TestCase implements Formatter { +public class HighlighterTest extends BaseTokenStreamTestCase implements Formatter { private IndexReader reader; static final String FIELD_NAME = "contents"; private Query query; @@ -1600,10 +1599,8 @@ } } - /* - * @see TestCase#setUp() - */ protected void setUp() throws Exception { + super.setUp(); ramDir = new RAMDirectory(); IndexWriter writer = new IndexWriter(ramDir, new StandardAnalyzer(), true); for (int i = 0; i < texts.length; i++) { @@ -1624,9 +1621,6 @@ } - /* - * @see TestCase#tearDown() - */ protected void tearDown() throws Exception { super.tearDown(); } @@ -1692,9 +1686,9 @@ public SynonymTokenizer(TokenStream realStream, Map synonyms) { this.realStream = realStream; this.synonyms = synonyms; - realTermAtt = (TermAttribute) realStream.getAttribute(TermAttribute.class); - realPosIncrAtt = (PositionIncrementAttribute) realStream.getAttribute(PositionIncrementAttribute.class); - realOffsetAtt = (OffsetAttribute) realStream.getAttribute(OffsetAttribute.class); + realTermAtt = (TermAttribute) realStream.addAttribute(TermAttribute.class); + realPosIncrAtt = (PositionIncrementAttribute) realStream.addAttribute(PositionIncrementAttribute.class); + realOffsetAtt = (OffsetAttribute) realStream.addAttribute(OffsetAttribute.class); termAtt = (TermAttribute) addAttribute(TermAttribute.class); posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class); Index: contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java =================================================================== --- contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (revision 806538) +++ contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (working copy) @@ -33,8 +33,7 @@ import java.util.List; import java.util.Set; -import junit.framework.TestCase; - +import org.apache.lucene.analysis.BaseTokenStreamTestCase; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.SimpleAnalyzer; import org.apache.lucene.analysis.StopAnalyzer; @@ -198,7 +197,7 @@ */ -public class MemoryIndexTest extends TestCase { +public class MemoryIndexTest extends BaseTokenStreamTestCase { private Analyzer analyzer; private boolean fastMode = false; @@ -214,7 +213,8 @@ /* all files will be open relative to this */ public String fileDir; - public void setUp() { + protected void setUp() throws Exception { + super.setUp(); fileDir = System.getProperty("lucene.common.dir", null); } Index: contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java =================================================================== --- contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java (revision 806538) +++ contrib/memory/src/test/org/apache/lucene/index/memory/PatternAnalyzerTest.java (working copy) @@ -31,8 +31,7 @@ import java.util.Set; import java.util.regex.Pattern; -import junit.framework.TestCase; - +import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.analysis.LetterTokenizer; import org.apache.lucene.analysis.LowerCaseFilter; import org.apache.lucene.analysis.StopAnalyzer; @@ -58,8 +57,9 @@ Thus the PatternAnalyzer produces correct output, whereas the WhitespaceAnalyzer silently truncates text, and so the comparison results in assertEquals() don't match up. +TODO: Convert to new TokenStream API! */ -public class PatternAnalyzerTest extends TestCase { +public class PatternAnalyzerTest extends LuceneTestCase { /** Runs the tests and/or benchmark */ public static void main(String[] args) throws Throwable { Index: contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java =================================================================== --- contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java (revision 806538) +++ contrib/memory/src/test/org/apache/lucene/index/memory/TestSynonymTokenFilter.java (working copy) @@ -31,10 +31,9 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute; import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute; import org.apache.lucene.analysis.tokenattributes.TermAttribute; +import org.apache.lucene.analysis.BaseTokenStreamTestCase; -import junit.framework.TestCase; - -public class TestSynonymTokenFilter extends TestCase { +public class TestSynonymTokenFilter extends BaseTokenStreamTestCase { File dataDir = new File(System.getProperty("dataDir", "./bin")); File testFile = new File(dataDir, "org/apache/lucene/index/memory/testSynonyms.txt"); Index: src/java/org/apache/lucene/index/TermsHashPerField.java =================================================================== --- src/java/org/apache/lucene/index/TermsHashPerField.java (revision 806538) +++ src/java/org/apache/lucene/index/TermsHashPerField.java (working copy) @@ -249,7 +249,7 @@ private boolean doNextCall; void start(Fieldable f) { - termAtt = (TermAttribute) fieldState.attributeSource.getAttribute(TermAttribute.class); + termAtt = (TermAttribute) fieldState.attributeSource.addAttribute(TermAttribute.class); consumer.start(f); if (nextPerField != null) { nextPerField.start(f); Index: src/java/org/apache/lucene/search/QueryTermVector.java =================================================================== --- src/java/org/apache/lucene/search/QueryTermVector.java (revision 806538) +++ src/java/org/apache/lucene/search/QueryTermVector.java (working copy) @@ -61,7 +61,7 @@ boolean hasMoreTokens = false; stream.reset(); - TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class); + TermAttribute termAtt = (TermAttribute) stream.addAttribute(TermAttribute.class); hasMoreTokens = stream.incrementToken(); while (hasMoreTokens) { Index: src/java/org/apache/lucene/util/AttributeSource.java =================================================================== --- src/java/org/apache/lucene/util/AttributeSource.java (revision 806538) +++ src/java/org/apache/lucene/util/AttributeSource.java (working copy) @@ -249,7 +249,11 @@ *
Signature for Java 1.5: public <T extends Attribute> T getAttribute(Class<T>)
*
* @throws IllegalArgumentException if this AttributeSource does not contain the
- * Attribute
+ * Attribute. It is recommended to always use {@link #addAttribute} even in consumers
+ * of TokenStreams, because you cannot know if a specific TokenStream really uses
+ * a specific Attribute. {@link #addAttribute} will automatically make the attribute
+ * available. If you want to only use the attribute, if it is available (to optimize
+ * consuming), use {@link #hasAttribute}.
*/
public Attribute getAttribute(Class attClass) {
final Attribute att = (Attribute) this.attributes.get(attClass);
Index: src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java
===================================================================
--- src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 0)
+++ src/test/org/apache/lucene/analysis/BaseTokenStreamTestCase.java (revision 0)
@@ -0,0 +1,67 @@
+package org.apache.lucene.analysis;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util.LuceneTestCase;
+
+/**
+ * Base class for all Lucene unit tests that use TokenStreams.
+ *
+ * This class runs all tests twice, one time with {@link TokenStream#setOnlyUseNewAPI} false
+ * and after that one time with true.
+ */
+public abstract class BaseTokenStreamTestCase extends LuceneTestCase {
+
+ private boolean onlyUseNewAPI = false;
+
+ public BaseTokenStreamTestCase() {
+ super();
+ }
+
+ public BaseTokenStreamTestCase(String name) {
+ super(name);
+ }
+
+ // @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ TokenStream.setOnlyUseNewAPI(onlyUseNewAPI);
+ }
+
+ // @Override
+ public void runBare() throws Throwable {
+ // Do the test with onlyUseNewAPI=false (default)
+ try {
+ onlyUseNewAPI = false;
+ super.runBare();
+ } catch (Throwable e) {
+ System.out.println("Test failure of "+getName()+" occurred with onlyUseNewAPI=false");
+ throw e;
+ }
+
+ // Do the test again with onlyUseNewAPI=true
+ try {
+ onlyUseNewAPI = true;
+ super.runBare();
+ } catch (Throwable e) {
+ System.out.println("Test failure of "+getName()+" occurred with onlyUseNewAPI=true");
+ throw e;
+ }
+ }
+
+}
Property changes on: src\test\org\apache\lucene\analysis\BaseTokenStreamTestCase.java
___________________________________________________________________
Added: svn:keywords
+ Date Author Id Revision HeadURL
Added: svn:eol-style
+ native
Index: src/test/org/apache/lucene/analysis/BaseTokenTestCase.java
===================================================================
--- src/test/org/apache/lucene/analysis/BaseTokenTestCase.java (revision 806538)
+++ src/test/org/apache/lucene/analysis/BaseTokenTestCase.java (working copy)
@@ -25,6 +25,7 @@
import org.apache.lucene.util.LuceneTestCase;
+/* TODO: Convert to new TokenStream API. Token instances must be removed for that to work */
public abstract class BaseTokenTestCase extends LuceneTestCase {
public static String tsToString(TokenStream in) throws IOException {
StringBuffer out = new StringBuffer();
@@ -48,6 +49,12 @@
}
*/
+ protected void setUp() throws Exception {
+ super.setUp();
+ // force this to false, as we use Token and next()
+ TokenStream.setOnlyUseNewAPI(false);
+ }
+
public void assertTokEqual(List/*